From 4877dab2c42005e8f7a63ff6943529990b7c0753 Mon Sep 17 00:00:00 2001 From: Karamaz0V1 Date: Thu, 26 Jul 2018 16:22:48 +0200 Subject: [PATCH] Supervisor working --- Notebooks/YAML Serialization.ipynb | 84 ++++- Notebooks/notebook.tex | 572 ----------------------------- supervisor.py | 51 ++- test.yml | 31 -- 4 files changed, 106 insertions(+), 632 deletions(-) delete mode 100644 Notebooks/notebook.tex delete mode 100644 test.yml diff --git a/Notebooks/YAML Serialization.ipynb b/Notebooks/YAML Serialization.ipynb index b1fa07e..8c8978f 100644 --- a/Notebooks/YAML Serialization.ipynb +++ b/Notebooks/YAML Serialization.ipynb @@ -160,6 +160,7 @@ " def __init__(self):\n", " self._pt = time.process_time()\n", " self._times = OrderedDict()\n", + " self._stime = time.time()\n", " \n", " def time(self, name):\n", " self._times[name + '_process_time'] = time.process_time() - self._pt\n", @@ -168,8 +169,13 @@ " def get_times(self):\n", " return self._times\n", " \n", - "kronos = Kronos()\n", - "start_time = time.time()" + " def get_start_date(self):\n", + " return self._stime\n", + " \n", + " def get_end_date(self):\n", + " return time.time()\n", + " \n", + "kronos = Kronos()" ] }, { @@ -308,7 +314,7 @@ "\n", " expe_report['supervisor'] = os.uname()[1]\n", "\n", - " for timev, datek in zip((start_time, end_time), ('start_date', 'end_date')):\n", + " for timev, datek in zip((kronos.get_start_date(), kronos.get_end_date()), ('start_date', 'end_date')):\n", " expe_report[datek] = datetime.datetime.fromtimestamp(timev).strftime('Le %d/%m/%Y à %H:%M:%S')\n", "\n", " ressources = kronos.get_times()\n", @@ -321,15 +327,6 @@ "expe_report" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**TODO**\n", - "\n", - "améliorer kronos pour le start et le end time (build and get_times)\n" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -535,6 +532,69 @@ "desc.run(**expe['descriptors_param'])" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "A = []" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "A.pop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "A.count()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "continue" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "E = Path('./Enrichment')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "(E / 'Test' / 'aefaef.tif').stem" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "E / ('qwer' + '.tif')" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/Notebooks/notebook.tex b/Notebooks/notebook.tex deleted file mode 100644 index 38e256f..0000000 --- a/Notebooks/notebook.tex +++ /dev/null @@ -1,572 +0,0 @@ - -% Default to the notebook output style - - - - -% Inherit from the specified cell style. - - - - - -\documentclass[11pt]{article} - - - - \usepackage[T1]{fontenc} - % Nicer default font (+ math font) than Computer Modern for most use cases - \usepackage{mathpazo} - - % Basic figure setup, for now with no caption control since it's done - % automatically by Pandoc (which extracts ![](path) syntax from Markdown). - \usepackage{graphicx} - % We will generate all images so they have a width \maxwidth. This means - % that they will get their normal width if they fit onto the page, but - % are scaled down if they would overflow the margins. - \makeatletter - \def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth - \else\Gin@nat@width\fi} - \makeatother - \let\Oldincludegraphics\includegraphics - % Set max figure width to be 80% of text width, for now hardcoded. - \renewcommand{\includegraphics}[1]{\Oldincludegraphics[width=.8\maxwidth]{#1}} - % Ensure that by default, figures have no caption (until we provide a - % proper Figure object with a Caption API and a way to capture that - % in the conversion process - todo). - \usepackage{caption} - \DeclareCaptionLabelFormat{nolabel}{} - \captionsetup{labelformat=nolabel} - - \usepackage{adjustbox} % Used to constrain images to a maximum size - \usepackage{xcolor} % Allow colors to be defined - \usepackage{enumerate} % Needed for markdown enumerations to work - \usepackage{geometry} % Used to adjust the document margins - \usepackage{amsmath} % Equations - \usepackage{amssymb} % Equations - \usepackage{textcomp} % defines textquotesingle - % Hack from http://tex.stackexchange.com/a/47451/13684: - \AtBeginDocument{% - \def\PYZsq{\textquotesingle}% Upright quotes in Pygmentized code - } - \usepackage{upquote} % Upright quotes for verbatim code - \usepackage{eurosym} % defines \euro - \usepackage[mathletters]{ucs} % Extended unicode (utf-8) support - \usepackage[utf8x]{inputenc} % Allow utf-8 characters in the tex document - \usepackage{fancyvrb} % verbatim replacement that allows latex - \usepackage{grffile} % extends the file name processing of package graphics - % to support a larger range - % The hyperref package gives us a pdf with properly built - % internal navigation ('pdf bookmarks' for the table of contents, - % internal cross-reference links, web links for URLs, etc.) - \usepackage{hyperref} - \usepackage{longtable} % longtable support required by pandoc >1.10 - \usepackage{booktabs} % table support for pandoc > 1.12.2 - \usepackage[inline]{enumitem} % IRkernel/repr support (it uses the enumerate* environment) - \usepackage[normalem]{ulem} % ulem is needed to support strikethroughs (\sout) - % normalem makes italics be italics, not underlines - - - - - % Colors for the hyperref package - \definecolor{urlcolor}{rgb}{0,.145,.698} - \definecolor{linkcolor}{rgb}{.71,0.21,0.01} - \definecolor{citecolor}{rgb}{.12,.54,.11} - - % ANSI colors - \definecolor{ansi-black}{HTML}{3E424D} - \definecolor{ansi-black-intense}{HTML}{282C36} - \definecolor{ansi-red}{HTML}{E75C58} - \definecolor{ansi-red-intense}{HTML}{B22B31} - \definecolor{ansi-green}{HTML}{00A250} - \definecolor{ansi-green-intense}{HTML}{007427} - \definecolor{ansi-yellow}{HTML}{DDB62B} - \definecolor{ansi-yellow-intense}{HTML}{B27D12} - \definecolor{ansi-blue}{HTML}{208FFB} - \definecolor{ansi-blue-intense}{HTML}{0065CA} - \definecolor{ansi-magenta}{HTML}{D160C4} - \definecolor{ansi-magenta-intense}{HTML}{A03196} - \definecolor{ansi-cyan}{HTML}{60C6C8} - \definecolor{ansi-cyan-intense}{HTML}{258F8F} - \definecolor{ansi-white}{HTML}{C5C1B4} - \definecolor{ansi-white-intense}{HTML}{A1A6B2} - - % commands and environments needed by pandoc snippets - % extracted from the output of `pandoc -s` - \providecommand{\tightlist}{% - \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} - \DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}} - % Add ',fontsize=\small' for more characters per line - \newenvironment{Shaded}{}{} - \newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}} - \newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}} - \newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}} - \newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}} - \newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}} - \newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} - \newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} - \newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}} - \newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}} - \newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}} - \newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}} - \newcommand{\RegionMarkerTok}[1]{{#1}} - \newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}} - \newcommand{\NormalTok}[1]{{#1}} - - % Additional commands for more recent versions of Pandoc - \newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.53,0.00,0.00}{{#1}}} - \newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} - \newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} - \newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.73,0.40,0.53}{{#1}}} - \newcommand{\ImportTok}[1]{{#1}} - \newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.73,0.13,0.13}{\textit{{#1}}}} - \newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} - \newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} - \newcommand{\VariableTok}[1]{\textcolor[rgb]{0.10,0.09,0.49}{{#1}}} - \newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}} - \newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.40,0.40,0.40}{{#1}}} - \newcommand{\BuiltInTok}[1]{{#1}} - \newcommand{\ExtensionTok}[1]{{#1}} - \newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.74,0.48,0.00}{{#1}}} - \newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.49,0.56,0.16}{{#1}}} - \newcommand{\InformationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} - \newcommand{\WarningTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} - - - % Define a nice break command that doesn't care if a line doesn't already - % exist. - \def\br{\hspace*{\fill} \\* } - % Math Jax compatability definitions - \def\gt{>} - \def\lt{<} - % Document parameters - \title{Triskele Python Demo} - - - - - % Pygments definitions - -\makeatletter -\def\PY@reset{\let\PY@it=\relax \let\PY@bf=\relax% - \let\PY@ul=\relax \let\PY@tc=\relax% - \let\PY@bc=\relax \let\PY@ff=\relax} -\def\PY@tok#1{\csname PY@tok@#1\endcsname} -\def\PY@toks#1+{\ifx\relax#1\empty\else% - \PY@tok{#1}\expandafter\PY@toks\fi} -\def\PY@do#1{\PY@bc{\PY@tc{\PY@ul{% - \PY@it{\PY@bf{\PY@ff{#1}}}}}}} -\def\PY#1#2{\PY@reset\PY@toks#1+\relax+\PY@do{#2}} - -\expandafter\def\csname PY@tok@w\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} -\expandafter\def\csname PY@tok@c\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} -\expandafter\def\csname PY@tok@cp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.74,0.48,0.00}{##1}}} -\expandafter\def\csname PY@tok@k\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@kp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@kt\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.69,0.00,0.25}{##1}}} -\expandafter\def\csname PY@tok@o\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} -\expandafter\def\csname PY@tok@ow\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}} -\expandafter\def\csname PY@tok@nb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@nf\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}} -\expandafter\def\csname PY@tok@nc\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}} -\expandafter\def\csname PY@tok@nn\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}} -\expandafter\def\csname PY@tok@ne\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.82,0.25,0.23}{##1}}} -\expandafter\def\csname PY@tok@nv\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} -\expandafter\def\csname PY@tok@no\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} -\expandafter\def\csname PY@tok@nl\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.63,0.63,0.00}{##1}}} -\expandafter\def\csname PY@tok@ni\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.60,0.60,0.60}{##1}}} -\expandafter\def\csname PY@tok@na\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.49,0.56,0.16}{##1}}} -\expandafter\def\csname PY@tok@nt\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@nd\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}} -\expandafter\def\csname PY@tok@s\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} -\expandafter\def\csname PY@tok@sd\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} -\expandafter\def\csname PY@tok@si\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}} -\expandafter\def\csname PY@tok@se\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.13}{##1}}} -\expandafter\def\csname PY@tok@sr\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}} -\expandafter\def\csname PY@tok@ss\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} -\expandafter\def\csname PY@tok@sx\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@m\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} -\expandafter\def\csname PY@tok@gh\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} -\expandafter\def\csname PY@tok@gu\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} -\expandafter\def\csname PY@tok@gd\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} -\expandafter\def\csname PY@tok@gi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} -\expandafter\def\csname PY@tok@gr\endcsname{\def\PY@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PY@tok@ge\endcsname{\let\PY@it=\textit} -\expandafter\def\csname PY@tok@gs\endcsname{\let\PY@bf=\textbf} -\expandafter\def\csname PY@tok@gp\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} -\expandafter\def\csname PY@tok@go\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PY@tok@gt\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} -\expandafter\def\csname PY@tok@err\endcsname{\def\PY@bc##1{\setlength{\fboxsep}{0pt}\fcolorbox[rgb]{1.00,0.00,0.00}{1,1,1}{\strut ##1}}} -\expandafter\def\csname PY@tok@kc\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@kd\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@kn\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@kr\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@bp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PY@tok@fm\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}} -\expandafter\def\csname PY@tok@vc\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} -\expandafter\def\csname PY@tok@vg\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} -\expandafter\def\csname PY@tok@vi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} -\expandafter\def\csname PY@tok@vm\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}} -\expandafter\def\csname PY@tok@sa\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} -\expandafter\def\csname PY@tok@sb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} -\expandafter\def\csname PY@tok@sc\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} -\expandafter\def\csname PY@tok@dl\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} -\expandafter\def\csname PY@tok@s2\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} -\expandafter\def\csname PY@tok@sh\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} -\expandafter\def\csname PY@tok@s1\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}} -\expandafter\def\csname PY@tok@mb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} -\expandafter\def\csname PY@tok@mf\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} -\expandafter\def\csname PY@tok@mh\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} -\expandafter\def\csname PY@tok@mi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} -\expandafter\def\csname PY@tok@il\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} -\expandafter\def\csname PY@tok@mo\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}} -\expandafter\def\csname PY@tok@ch\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} -\expandafter\def\csname PY@tok@cm\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} -\expandafter\def\csname PY@tok@cpf\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} -\expandafter\def\csname PY@tok@c1\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} -\expandafter\def\csname PY@tok@cs\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}} - -\def\PYZbs{\char`\\} -\def\PYZus{\char`\_} -\def\PYZob{\char`\{} -\def\PYZcb{\char`\}} -\def\PYZca{\char`\^} -\def\PYZam{\char`\&} -\def\PYZlt{\char`\<} -\def\PYZgt{\char`\>} -\def\PYZsh{\char`\#} -\def\PYZpc{\char`\%} -\def\PYZdl{\char`\$} -\def\PYZhy{\char`\-} -\def\PYZsq{\char`\'} -\def\PYZdq{\char`\"} -\def\PYZti{\char`\~} -% for compatibility with earlier versions -\def\PYZat{@} -\def\PYZlb{[} -\def\PYZrb{]} -\makeatother - - - % Exact colors from NB - \definecolor{incolor}{rgb}{0.0, 0.0, 0.5} - \definecolor{outcolor}{rgb}{0.545, 0.0, 0.0} - - - - - % Prevent overflowing lines due to hard-to-break entities - \sloppy - % Setup hyperref package - \hypersetup{ - breaklinks=true, % so long urls are correctly broken across lines - colorlinks=true, - urlcolor=urlcolor, - linkcolor=linkcolor, - citecolor=citecolor, - } - % Slightly bigger margins than the latex defaults - - \geometry{verbose,tmargin=1in,bmargin=1in,lmargin=1in,rmargin=1in} - - - - \begin{document} - - - \maketitle - - - - - \hypertarget{init}{% -\section{Init}\label{init}} - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}1}]:} \PY{k+kn}{import} \PY{n+nn}{numpy} \PY{k}{as} \PY{n+nn}{np} - \PY{k+kn}{import} \PY{n+nn}{matplotlib}\PY{n+nn}{.}\PY{n+nn}{pyplot} \PY{k}{as} \PY{n+nn}{plt} -\end{Verbatim} - - - \hypertarget{import-triskele}{% -\subsection{Import Triskele}\label{import-triskele}} - -I did not install triskele-python on my system folders, so this is a -trick to load the module outside the triskele project. - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}2}]:} \PY{k+kn}{import} \PY{n+nn}{sys} - \PY{n}{sys}\PY{o}{.}\PY{n}{path}\PY{o}{.}\PY{n}{append}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{../triskele/python/}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)} \PY{c+c1}{\PYZsh{} Change with your triskele install path} - \PY{k+kn}{import} \PY{n+nn}{triskele} -\end{Verbatim} - - - \hypertarget{simple-test}{% -\section{Simple test}\label{simple-test}} - -Just to test the python import and the triskele compilation. Create an -array with random values. - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}3}]:} \PY{n}{raster} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{randint}\PY{p}{(}\PY{l+m+mi}{0}\PY{p}{,} \PY{l+m+mi}{255}\PY{p}{,} \PY{p}{(}\PY{l+m+mi}{1000}\PY{p}{,} \PY{l+m+mi}{1000}\PY{p}{)}\PY{p}{)} -\end{Verbatim} - - - Create the instance of triskele object with the array to filter. - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}4}]:} \PY{n}{ap\PYZus{}random} \PY{o}{=} \PY{n}{triskele}\PY{o}{.}\PY{n}{Triskele}\PY{p}{(}\PY{n}{raster}\PY{p}{)} -\end{Verbatim} - - - Filter the raster with this method : - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}5}]:} \PY{n}{filtered\PYZus{}raster} \PY{o}{=} \PY{n}{ap\PYZus{}random}\PY{o}{.}\PY{n}{filter}\PY{p}{(}\PY{p}{)} - \PY{n+nb}{print}\PY{p}{(}\PY{n}{filtered\PYZus{}raster}\PY{p}{)} -\end{Verbatim} - - - \begin{Verbatim}[commandchars=\\\{\}] -STDOUT: -Input:/tmp/infile.tif [1000,1000] (1 chanels of Byte) -Crop topLeft:(0,0) size:[1000,1000] band:[0] -Output:/tmp/outfile.tif -core count:24 - -STDERR: - -*** apGenerator done! - -Leaf Count Mean Min Max - max 1 1000000 1000000 1000000 - -Comp Count Mean Min Max - max 1 347157 347157 347157 - - -Time Sum Count Mean Min Max - build tree 00:00:00.086426 1 00:00:00.086426 00:00:00.086426 00:00:00.086426 - setup 00:00:00.038795 1 00:00:00.038795 00:00:00.038795 00:00:00.038795 - parents 00:00:00.026013 1 00:00:00.026013 00:00:00.026013 00:00:00.026013 - merge 00:00:00.005112 1 00:00:00.005112 00:00:00.005112 00:00:00.005112 - index 00:00:00.006034 1 00:00:00.006034 00:00:00.006034 00:00:00.006034 - compress 00:00:00.008091 1 00:00:00.008091 00:00:00.008091 00:00:00.008091 - children 00:00:00.108123 1 00:00:00.108123 00:00:00.108123 00:00:00.108123 - area 00:00:00.004463 1 00:00:00.004463 00:00:00.004463 00:00:00.004463 - -[[194 125 164 {\ldots} 213 182 168] - [ 25 48 242 {\ldots} 229 16 143] - [ 7 207 212 {\ldots} 112 94 100] - {\ldots} - [227 180 185 {\ldots} 34 10 229] - [129 149 27 {\ldots} 217 66 195] - [145 203 251 {\ldots} 186 116 165]] - - \end{Verbatim} - - If there is no error so far everything is okay. - -Theoretically the filtered output should be a copy of the input. (it's -not, there is something going on along the edges, we should tell -François !) - -\hypertarget{attribute-filter}{% -\subsection{Attribute filter}\label{attribute-filter}} - -With \texttt{filter} method you can ask for specific attributes -filtering. The results are stacked. - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}6}]:} \PY{n}{filtered\PYZus{}rasters} \PY{o}{=} \PY{n}{ap\PYZus{}random}\PY{o}{.}\PY{n}{filter}\PY{p}{(}\PY{n}{area}\PY{o}{=}\PY{p}{[}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{100}\PY{p}{,} \PY{l+m+mf}{1e3}\PY{p}{]}\PY{p}{)} - \PY{n+nb}{print}\PY{p}{(}\PY{n}{filtered\PYZus{}rasters}\PY{o}{.}\PY{n}{shape}\PY{p}{)} -\end{Verbatim} - - - \begin{Verbatim}[commandchars=\\\{\}] -STDOUT: -Input:/tmp/infile.tif [1000,1000] (1 chanels of Byte) -Crop topLeft:(0,0) size:[1000,1000] band:[0] -Output:/tmp/outfile.tif -core count:24 - -STDERR: - -*** apGenerator done! - -Leaf Count Mean Min Max - max 1 1000000 1000000 1000000 - -Comp Count Mean Min Max - max 1 347157 347157 347157 - - -Time Sum Count Mean Min Max - build tree 00:00:00.094336 1 00:00:00.094336 00:00:00.094336 00:00:00.094336 - setup 00:00:00.040030 1 00:00:00.040030 00:00:00.040030 00:00:00.040030 - parents 00:00:00.034337 1 00:00:00.034337 00:00:00.034337 00:00:00.034337 - merge 00:00:00.004679 1 00:00:00.004679 00:00:00.004679 00:00:00.004679 - index 00:00:00.005624 1 00:00:00.005624 00:00:00.005624 00:00:00.005624 - compress 00:00:00.007514 1 00:00:00.007514 00:00:00.007514 00:00:00.007514 - children 00:00:00.113179 1 00:00:00.113179 00:00:00.113179 00:00:00.113179 - area 00:00:00.004390 1 00:00:00.004390 00:00:00.004390 00:00:00.004390 - filtering 00:00:00.007614 1 00:00:00.007614 00:00:00.007614 00:00:00.007614 - -(1000, 1000, 4) - - \end{Verbatim} - - I don't know if you can see the signature with yout IDE : -\texttt{filter(tree=\textquotesingle{}max-tree\textquotesingle{},\ area=None,\ standard\_deviation=None,\ moment\_of\_inertia=None)} - -The parameters are the same than apGenerator from François, I should -write the doc\ldots{} - - \hypertarget{advanced-test}{% -\section{Advanced test}\label{advanced-test}} - -With a real example. - -Load a TIFF file with 4 bands : - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}7}]:} \PY{n}{raster} \PY{o}{=} \PY{n}{triskele}\PY{o}{.}\PY{n}{read}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{../triskele/data/10m.tif}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)} - \PY{n+nb}{print}\PY{p}{(}\PY{n}{raster}\PY{o}{.}\PY{n}{shape}\PY{p}{)} - - \PY{n}{plt}\PY{o}{.}\PY{n}{imshow}\PY{p}{(}\PY{n}{raster}\PY{p}{)} - \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)} -\end{Verbatim} - - - \begin{Verbatim}[commandchars=\\\{\}] -(291, 685, 4) - - \end{Verbatim} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_14_1.png} - \end{center} - { \hspace*{\fill} \\} - - Create the instance with some tweaks. - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}8}]:} \PY{n}{ap\PYZus{}10m} \PY{o}{=} \PY{n}{triskele}\PY{o}{.}\PY{n}{Triskele}\PY{p}{(}\PY{n}{raster}\PY{p}{,} \PY{n}{verbose}\PY{o}{=}\PY{k+kc}{False}\PY{p}{)} \PY{c+c1}{\PYZsh{} Verbose False to hide apGenerator gibbering :p} -\end{Verbatim} - - - Filter as you wish : - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}9}]:} \PY{n}{tree} \PY{o}{=} \PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{tos\PYZhy{}tree}\PY{l+s+s1}{\PYZsq{}} \PY{c+c1}{\PYZsh{} \PYZob{}\PYZsq{}max\PYZhy{}tree\PYZsq{}, \PYZsq{}min\PYZhy{}tree\PYZsq{}, ...\PYZcb{}} - \PY{n}{area} \PY{o}{=} \PY{p}{[}\PY{l+m+mi}{100}\PY{p}{,} \PY{l+m+mf}{1e3}\PY{p}{,} \PY{l+m+mf}{1e4}\PY{p}{]} - \PY{n}{standard\PYZus{}deviation} \PY{o}{=} \PY{k+kc}{None} - \PY{n}{moment\PYZus{}of\PYZus{}inertia} \PY{o}{=} \PY{k+kc}{None} - - \PY{n}{filtered\PYZus{}rasters} \PY{o}{=} \PY{n}{ap\PYZus{}10m}\PY{o}{.}\PY{n}{filter}\PY{p}{(}\PY{n}{tree}\PY{o}{=}\PY{n}{tree}\PY{p}{,} \PY{n}{area}\PY{o}{=}\PY{n}{area}\PY{p}{,} \PY{n}{standard\PYZus{}deviation}\PY{o}{=}\PY{n}{standard\PYZus{}deviation}\PY{p}{,} \PY{n}{moment\PYZus{}of\PYZus{}inertia}\PY{o}{=}\PY{n}{moment\PYZus{}of\PYZus{}inertia}\PY{p}{)} - \PY{n+nb}{print}\PY{p}{(}\PY{n}{filtered\PYZus{}rasters}\PY{o}{.}\PY{n}{shape}\PY{p}{)} -\end{Verbatim} - - - \begin{Verbatim}[commandchars=\\\{\}] -(291, 685, 16) - - \end{Verbatim} - - \hypertarget{display-results}{% -\subsection{Display results}\label{display-results}} - - \begin{Verbatim}[commandchars=\\\{\}] -{\color{incolor}In [{\color{incolor}10}]:} \PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n}{filtered\PYZus{}rasters}\PY{o}{.}\PY{n}{shape}\PY{p}{[}\PY{l+m+mi}{2}\PY{p}{]}\PY{p}{)}\PY{p}{:} - \PY{n}{plt}\PY{o}{.}\PY{n}{imshow}\PY{p}{(}\PY{n}{filtered\PYZus{}rasters}\PY{p}{[}\PY{p}{:}\PY{p}{,}\PY{p}{:}\PY{p}{,}\PY{n}{i}\PY{p}{]}\PY{p}{)} - \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)} -\end{Verbatim} - - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_0.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_1.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_2.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_3.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_4.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_5.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_6.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_7.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_8.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_9.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_10.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_11.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_12.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_13.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_14.png} - \end{center} - { \hspace*{\fill} \\} - - \begin{center} - \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_20_15.png} - \end{center} - { \hspace*{\fill} \\} - - - % Add a bibliography block to the postdoc - - - - \end{document} diff --git a/supervisor.py b/supervisor.py index 9514634..7e43c11 100644 --- a/supervisor.py +++ b/supervisor.py @@ -38,7 +38,10 @@ def setup_yaml(): yaml.add_representer(OrderedDict, represent_dict_order) setup_yaml() -test_dir = Path('./Tests') +enrichment_dir = Path('./Enrichment/') +test_dir = enrichment_dir / 'Tests' +staging_dir = enrichment_dir / 'Staging' +result_dir = enrichment_dir / 'Results' def update_queue(): tmp_queue = list() @@ -60,12 +63,23 @@ def run(expe_file): with open(expe_file) as f: expe = OrderedDict(yaml.safe_load(f)['expe']) - ### Compute hashes - expe_hashes = compute_hashes(expe) - ### Keep track of time kronos = Kronos() + ### Compute hashes + expe_hashes = compute_hashes(expe) + + ### Create output names + oname = '{}_{}'.format(expe_file.stem, expe_hashes['global'][:6]) + oname_yml = oname + '.yml' + oname_tif = oname + '.tif' + + ### Create partial report + expe_report = create_report(kronos) + + ### Stage expe + write_expe_file(staging_dir / oname_yml, expe, expe_hashes, expe_report) + ### Compute descriptors descriptors = compute_descriptors(expe) kronos.time('description') @@ -78,24 +92,25 @@ def run(expe_file): metrics = run_metrics(expe, classification) kronos.time('metrics') - ### Create report + ### Create complete report expe_report = create_report(kronos) ### Name and write prediction - oname = '{}_{}'.format(Path(expe_file).stem, expe_hashes['global'][:6]) - oname_tif = oname + '.tif' - triskele.write(oname_tif, classification) + triskele.write(result_dir / oname_tif, classification) ### Write report and results - oname_yml = oname + '.yml' - with open(oname_yml, 'w') as of: + write_expe_file(result_dir / oname_yml, expe, expe_hashes, expe_report, oname_tif, metrics) + + +def write_expe_file(file, expe, hashes=None, report=None, classification=None, results=None): + with open(file, 'w') as of: yaml.dump(OrderedDict({'expe': expe, - 'expe_hashes': expe_hashes, - 'expe_report': expe_report, - 'expe_classification': oname_tif, - 'expe_results': metrics}), + 'expe_hashes': hashes, + 'expe_report': report, + 'expe_classification': classification, + 'expe_results': results}), of, default_flow_style=False, encoding=None, allow_unicode=True) - + def compute_hashes(expe): glob = hashlib.sha1() @@ -170,7 +185,7 @@ def create_report(kronos): expe_report['supervisor'] = os.uname()[1] for timev, datek in zip((kronos.get_start_date(), kronos.get_end_date()), ('start_date', 'end_date')): - expe_report[datek] = datetime.datetime.fromtimestamp(timev).strftime('Le %d/%m/%Y à %H:%M:%S') + expe_report[datek] = datetime.datetime.fromtimestamp(timev).strftime('Le %d/%m/%Y à %H:%M:%S') if timev is not None else None ressources = kronos.get_times() ressources['ram'] = None @@ -184,10 +199,12 @@ class Kronos(object): self._pt = time.process_time() self._times = OrderedDict() self._stime = time.time() + self._etime = None def time(self, name): self._times[name + '_process_time'] = time.process_time() - self._pt self._pt = time.process_time() + self._etime = time.time() def get_times(self): return self._times @@ -196,7 +213,7 @@ class Kronos(object): return self._stime def get_end_date(self): - return time.time() + return self._etime def watch_folder(): diff --git a/test.yml b/test.yml deleted file mode 100644 index f57c103..0000000 --- a/test.yml +++ /dev/null @@ -1,31 +0,0 @@ -expe: - name: Première expérience - date: 9 juillet 2018 - priority: 1 - ground_truth: '../Data/ground_truth/2018_IEEE_GRSS_DFC_GT_TR.tif' - descriptors_script: - name: Descriptors.dfc_aps - parameters: - treshold: 1e4 - rasters: - - '../Data/phase1_rasters/DEM+B_C123/UH17_GEM051_TR.tif' - - '../Data/phase1_rasters/DEM_C123_3msr/UH17_GEG051_TR.tif' - areas: [100, 1000] - moi: [.5, .9] - cross_validation: - name: CrossValidationGenerator.APsCVG - parameters: - n_test: 2 - classifier: - name: sklearn.ensemble.RandomForestClassifier - parameters: - n_jobs: -1 - random_state: 0 - n_estimators: 50 - min_samples_leaf: 10 -expe_hashes: - ground_truth: XXX - descriptors_script: XXX - cross_validation: XXX - classifier: XXX - global: XXX \ No newline at end of file