Read cache dir from environment

Supervisor and Jurse ready for production

See merge request Florent/minigrida!1

config.json

@@ -1,3 +1,3 @@
 {
-    "process_count": 2
+    "process_count": 1
 }

minigrida/cvgenerators/__init__.py

@@ -0,0 +1 @@
+from .generic import *

minigrida/cvgenerators/generic.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# file generic.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 08 juil. 2020
+"""Abstract
+
+doc.
+"""
+
+from sklearn.model_selection import GroupKFold, StratifiedShuffleSplit
+
+
+class GroupStratifiedShuffleSplit(GroupKFold):
+    def __init__(self, n_splits=5, train_size=None, random_state=None):
+        super().__init__(n_splits)
+        self.train_size = train_size
+        self.random_state = random_state
+    
+    def split(self, X, y=None, groups=None):
+        splits = [s for s in super().split(X, y, groups)]
+        for train, test in splits:
+            sss_train, sss_test = next(StratifiedShuffleSplit(1, train_size=self.train_size, random_state=self.random_state).split(X[train], y[train], groups[train]))
+            
+            yield train[sss_train], test

minigrida/database/helpers.py

@@ -16,7 +16,7 @@ import logging
 
 from .design import Session, Experiment, Project, db
 
-log = logging.getLogger()
+log = logging.getLogger(__name__)
 
 def compute_expe_hash(experiment):
     return hashlib.md5(json.dumps(experiment, sort_keys=True).encode('utf-8')).hexdigest()
@@ -34,7 +34,7 @@ def create_experiment(session_name, protocol, expe, urgency=1):
         e = q.first()
         e.sessions.add(session)
     else:
-        Experiment(sessions=session, protocol=protocol, expe=expe, expe_hash=expe_hash)
+        Experiment(sessions=session, protocol=protocol, expe=expe, expe_hash=expe_hash, urgency=urgency)
 
 @orm.db_session
 def create_project(name):
@@ -58,7 +58,7 @@ def pending_experiments():
     return Experiment.select(lambda x: x.status == 'pending').exists()
 
 
-@orm.db_session(serializable=True, optimistic=False)
+@orm.db_session(optimistic=False)
 def next_experiment():
     # TODO: take session urgency into account
     expe = orm.select(e for e in Experiment
@@ -81,7 +81,7 @@ def update_experiment(expe, **params):
         _update_experiment(expe, **params)
 
 
-@orm.db_session
+@orm.db_session(optimistic=False, retry=3)
 def _update_experiment(expe, **params):
     e = Experiment.get_for_update(id=expe.id)
     for k, v in params.items():

minigrida/descriptors/aps.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# file aps.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 30 mai 2020
+"""Abstract
+
+doc.
+"""
+
+import numpy as np
+import sap
+from multiprocessing import Pool
+
+
+def _attribute_profiles(*kwargs):
+    return sap.attribute_profiles(*kwargs)
+
+
+def run(gt, rasters, coords, remove,
+        attributes, adjacency='4', filtering='direct', dtype=np.float32):
+    X = []
+    y = []
+    groups = []
+    Xn = None
+
+    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
+        # Compute EAP
+        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
+        pool = Pool()
+        eap = pool.starmap(_attribute_profiles, [
+                           (raster, attribute, adjacency, name, filtering)
+                           for (name, raster), attribute
+                           in zip(rastersi.items(), attributes)])
+        pool.close()
+        pool.join()
+        eap = sap.concatenate(eap)
+
+        Xn = [' '.join((a.description['tree']['image_name'],
+                        a.description['attribute'],
+                        *[str(v) for v in p.values()]))
+                       for a in eap for p in a.description['profiles']] if not Xn else Xn
+
+        # Create vectors
+        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
+        y_raw = gti
+
+        # Remove unwanted label X, y
+        lbl = np.ones_like(y_raw, dtype=np.bool)
+        for l in remove if remove else []:
+            lbl &= y_raw != l
+
+        X += [X_raw[lbl]]
+        y += [y_raw[lbl]]
+        groups += [np.repeat(coordsi, lbl.sum())]
+
+    X = np.concatenate(X)
+    y = np.concatenate(y)
+    groups = np.concatenate(groups)
+
+    return X, y, groups, Xn

minigrida/descriptors/daps.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# file daps.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 07 juil. 2020
+
+#!/usr/bin/env python
+# file aps.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 30 mai 2020
+"""Abstract
+
+doc.
+"""
+
+import numpy as np
+import sap
+from multiprocessing import Pool
+
+
+def _diff_attribute_profiles(*kwargs):
+    return sap.attribute_profiles(*kwargs).diff()
+
+
+def run(gt, rasters, coords, remove,
+        attributes, adjacency='4', filtering='direct', dtype=np.float32):
+    X = []
+    y = []
+    groups = []
+    Xn = None
+
+    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
+        # Compute EAP
+        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
+        pool = Pool()
+        eap = pool.starmap(_diff_attribute_profiles, [
+                           (raster, attribute, adjacency, name, filtering)
+                           for (name, raster), attribute
+                           in zip(rastersi.items(), attributes)])
+        pool.close()
+        pool.join()
+        eap = sap.concatenate(eap)
+
+        Xn = [' '.join((a.description['tree']['image_name'],
+                a.description['attribute'],
+                *[sap.profiles._title(p)]))
+               for a in eap for p in a.description['profiles']] if not Xn else Xn
+
+        # Create vectors
+        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
+        y_raw = gti
+
+        # Remove unwanted label X, y
+        lbl = np.ones_like(y_raw, dtype=np.bool)
+        for l in remove if remove else []:
+            lbl &= y_raw != l
+
+        X += [X_raw[lbl]]
+        y += [y_raw[lbl]]
+        groups += [np.repeat(coordsi, lbl.sum())]
+
+    X = np.concatenate(X)
+    y = np.concatenate(y)
+    groups = np.concatenate(groups)
+
+    return X, y, groups, Xn

minigrida/descriptors/dsdaps.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# file dsdaps.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 07 juil. 2020
+"""Abstract
+
+doc.
+"""
+
+import numpy as np
+import sap
+from multiprocessing import Pool
+
+
+def _diff_sd_attribute_profiles(*kwargs):
+    return sap.self_dual_attribute_profiles(*kwargs).diff()
+
+
+def run(gt, rasters, coords, remove,
+        attributes, adjacency='4', filtering='direct', dtype=np.float32):
+    X = []
+    y = []
+    groups = []
+    Xn = None
+
+    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
+        # Compute EAP
+        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
+        pool = Pool()
+        eap = pool.starmap(_diff_sd_attribute_profiles, [
+                           (raster, attribute, adjacency, name, filtering)
+                           for (name, raster), attribute
+                           in zip(rastersi.items(), attributes)])
+        pool.close()
+        pool.join()
+        eap = sap.concatenate(eap)
+
+        Xn = [' '.join((a.description['tree']['image_name'],
+                a.description['attribute'],
+                *[sap.profiles._title(p)]))
+               for a in eap for p in a.description['profiles']] if not Xn else Xn
+
+        # Create vectors
+        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
+        y_raw = gti
+
+        # Remove unwanted label X, y
+        lbl = np.ones_like(y_raw, dtype=np.bool)
+        for l in remove if remove else []:
+            lbl &= y_raw != l
+
+        X += [X_raw[lbl]]
+        y += [y_raw[lbl]]
+        groups += [np.repeat(coordsi, lbl.sum())]
+
+    X = np.concatenate(X)
+    y = np.concatenate(y)
+    groups = np.concatenate(groups)
+
+    return X, y, groups, Xn

minigrida/descriptors/edaps.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# file edaps.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 07 juil. 2020
+"""Abstract
+
+doc.
+"""
+
+import numpy as np
+import sap
+from multiprocessing import Pool
+
+
+def _diff_attribute_profiles(*kwargs):
+    image = kwargs[0]
+    name = kwargs[3]
+    return sap.attribute_profiles(*kwargs).diff() \
+           + sap.Profiles([image[None]], 
+                          [{'tree': {'image_name': name}, 
+                            'attribute': 'altitude', 
+                            'profiles': [{'operation': 'copy'}]}
+                          ])
+
+
+def run(gt, rasters, coords, remove,
+        attributes, adjacency='4', filtering='direct', dtype=np.float32):
+    X = []
+    y = []
+    groups = []
+    Xn = None
+
+    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
+        # Compute EAP
+        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
+        pool = Pool()
+        eap = pool.starmap(_diff_attribute_profiles, [
+                           (raster, attribute, adjacency, name, filtering)
+                           for (name, raster), attribute
+                           in zip(rastersi.items(), attributes)])
+        pool.close()
+        pool.join()
+        eap = sap.concatenate(eap)
+
+        Xn = [' '.join((a.description['tree']['image_name'],
+                a.description['attribute'],
+                *[sap.profiles._title(p)]))
+               for a in eap for p in a.description['profiles']] if not Xn else Xn
+
+        # Create vectors
+        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
+        y_raw = gti
+
+        # Remove unwanted label X, y
+        lbl = np.ones_like(y_raw, dtype=np.bool)
+        for l in remove if remove else []:
+            lbl &= y_raw != l
+
+        X += [X_raw[lbl]]
+        y += [y_raw[lbl]]
+        groups += [np.repeat(coordsi, lbl.sum())]
+
+    X = np.concatenate(X)
+    y = np.concatenate(y)
+    groups = np.concatenate(groups)
+
+    return X, y, groups, Xn

minigrida/descriptors/edsdaps.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# file edsdaps.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 07 juil. 2020
+"""Abstract
+
+doc.
+"""
+
+import numpy as np
+import sap
+from multiprocessing import Pool
+
+
+def _diff_sd_attribute_profiles(*kwargs):
+    image = kwargs[0]
+    name = kwargs[3]
+    return sap.self_dual_attribute_profiles(*kwargs).diff() \
+           + sap.Profiles([image[None]], 
+                          [{'tree': {'image_name': name}, 
+                            'attribute': 'altitude', 
+                            'profiles': [{'operation': 'copy'}]}
+                          ])
+
+
+def run(gt, rasters, coords, remove,
+        attributes, adjacency='4', filtering='direct', dtype=np.float32):
+    X = []
+    y = []
+    groups = []
+    Xn = None
+
+    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
+        # Compute EAP
+        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
+        pool = Pool()
+        eap = pool.starmap(_diff_sd_attribute_profiles, [
+                           (raster, attribute, adjacency, name, filtering)
+                           for (name, raster), attribute
+                           in zip(rastersi.items(), attributes)])
+        pool.close()
+        pool.join()
+        eap = sap.concatenate(eap)
+
+        Xn = [' '.join((a.description['tree']['image_name'],
+                a.description['attribute'],
+                *[sap.profiles._title(p)]))
+               for a in eap for p in a.description['profiles']] if not Xn else Xn
+
+        # Create vectors
+        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
+        y_raw = gti
+
+        # Remove unwanted label X, y
+        lbl = np.ones_like(y_raw, dtype=np.bool)
+        for l in remove if remove else []:
+            lbl &= y_raw != l
+
+        X += [X_raw[lbl]]
+        y += [y_raw[lbl]]
+        groups += [np.repeat(coordsi, lbl.sum())]
+
+    X = np.concatenate(X)
+    y = np.concatenate(y)
+    groups = np.concatenate(groups)
+
+    return X, y, groups, Xn

minigrida/descriptors/fps.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# file fps.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 31 mai 2020
+"""Abstract
+
+doc.
+"""
+
+import numpy as np
+import sap
+from joblib import Memory
+
+memory = Memory(location='cache/', verbose=0)
+
+
+@memory.cache
+def _feature_profiles(**kwargs):
+    return sap.feature_profiles(**kwargs)
+
+
+def _origin_profile(image, name):
+    return sap.Profiles([image[None,:]], [{
+        'attribute': 'Origin',
+        'filtering rule': None,
+        'name': 'copy',
+        'out feature': 'altitude',
+        'profiles': [{'operation': 'copy feature altitude'}],
+        'tree': {'adjacency': None, 'image_hash': None, 'image_name': name}}])
+
+
+def run(gt, rasters, coords, remove, attributes, adjacency=4,
+        feature='same', filtering='direct'):
+    X = []
+    y = []
+    groups = []
+    Xn = None
+
+    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
+        # Compute EAP
+        eap = []
+        for name, raster in rastersi.items():
+            eap += [_origin_profile(raster, name)]
+            eap += [_feature_profiles(image=raster,
+                                      attribute=attributes,
+                                      adjacency=adjacency,
+                                      image_name=name,
+                                      out_feature=feature,
+                                      filtering_rule=filtering)]
+        eap = sap.concatenate(eap)
+
+        Xn = [' '.join((a['tree']['image_name'],
+                        a['attribute'],
+                        'feature',
+                        a['out feature'],
+                        *[str(v) for v in p.values()]))
+                       for a in eap.description for p in a['profiles']] if not Xn else Xn
+
+        # Create vectors
+        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1)
+        y_raw = gti
+
+        # Remove unwanted label X, y
+        lbl = np.ones_like(y_raw, dtype=np.bool)
+        for l in remove if remove else []:
+            lbl &= y_raw != l
+
+        X += [X_raw[lbl]]
+        y += [y_raw[lbl]]
+        groups += [np.repeat(coordsi, lbl.sum())]
+
+    X = np.concatenate(X)
+    y = np.concatenate(y)
+    groups = np.concatenate(groups)
+
+    return X, y, groups, Xn

minigrida/descriptors/max_aps.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# file max_aps.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 31 mai 2020
+"""Abstract
+
+doc.
+"""
+
+import numpy as np
+import sap
+from joblib import Memory
+
+memory = Memory(location='cache/', verbose=0)
+
+
+@memory.cache
+def _max_attribute_profiles(image, attribute, adjacency=4, image_name=None,
+                            filtering_rule='direct'):
+    maxt = sap.MaxTree(image, adjacency, image_name)
+    return sap.create_profiles(maxt, attribute, 'altitude',
+                               filtering_rule, 'max attribute profiles')
+
+
+def run(gt, rasters, coords, remove, attributes, adjacency='4',
+        filtering='direct'):
+    X = []
+    y = []
+    groups = []
+    Xn = None
+
+    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
+        # Compute EAP
+        eap = []
+        for name, raster in rastersi.items():
+            eap += [_max_attribute_profiles(raster, attributes, adjacency, name, filtering)]
+        eap = sap.concatenate(eap)
+
+        Xn = [' '.join((a['tree']['image_name'], 
+                        a['attribute'], 
+                        *[str(v) for v in p.values()])) 
+                       for a in eap.description for p in a['profiles']] if not Xn else Xn
+
+        # Create vectors
+        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1)
+        y_raw = gti
+
+        # Remove unwanted label X, y
+        lbl = np.ones_like(y_raw, dtype=np.bool)
+        for l in remove if remove else []:
+            lbl &= y_raw != l
+
+        X += [X_raw[lbl]]
+        y += [y_raw[lbl]]
+        groups += [np.repeat(coordsi, lbl.sum())]
+
+    X = np.concatenate(X)
+    y = np.concatenate(y)
+    groups = np.concatenate(groups)
+
+    return X, y, groups, Xn

minigrida/descriptors/pixel.py

@@ -11,12 +11,12 @@ doc.
 import numpy as np
 
 
-def run(gt, rasters, remove=None):
+def run(gt, rasters, coords, remove=None):
     X = []
     y = []
     groups = []
 
-    for i, (gti, rastersi) in enumerate(zip(gt, rasters)):
+    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
         # Create vectors
         X_raw = np.moveaxis(np.array(list(rastersi.values())), 0, -1)
         y_raw = gti
@@ -28,10 +28,11 @@ def run(gt, rasters, remove=None):
 
         X += [X_raw[lbl]]
         y += [y_raw[lbl]]
-        groups += [np.repeat(i, lbl.sum())]
+        groups += [np.repeat(coordsi, lbl.sum())]
 
     X = np.concatenate(X)
     y = np.concatenate(y)
     groups = np.concatenate(groups)
+    Xn = rasters[0].keys()
 
-    return X, y, groups
+    return X, y, groups, Xn

minigrida/descriptors/sdaps.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# file sdaps.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 31 mai 2020
+"""Abstract
+
+doc.
+"""
+
+import numpy as np
+import sap
+from multiprocessing import Pool
+
+
+def _self_dual_attribute_profiles(*kwargs):
+    return sap.self_dual_attribute_profiles(*kwargs)
+
+
+def run(gt, rasters, coords, remove,
+        attributes, adjacency='4', filtering='direct', dtype=np.float32):
+    X = []
+    y = []
+    groups = []
+    Xn = None
+
+    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
+        # Compute EAP
+        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
+        pool = Pool()
+        eap = pool.starmap(_self_dual_attribute_profiles, [
+                           (raster, attribute, adjacency, name, filtering)
+                           for (name, raster), attribute
+                           in zip(rastersi.items(), attributes)])
+        pool.close()
+        pool.join()
+        eap = sap.concatenate(eap)
+
+        Xn = [' '.join((a.description['tree']['image_name'],
+                        a.description['attribute'],
+                        *[str(v) for v in p.values()]))
+                       for a in eap for p in a.description['profiles']] if not Xn else Xn
+
+        # Create vectors
+        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
+        y_raw = gti
+
+        # Remove unwanted label X, y
+        lbl = np.ones_like(y_raw, dtype=np.bool)
+        for l in remove if remove else []:
+            lbl &= y_raw != l
+
+        X += [X_raw[lbl]]
+        y += [y_raw[lbl]]
+        groups += [np.repeat(coordsi, lbl.sum())]
+
+    X = np.concatenate(X)
+    y = np.concatenate(y)
+    groups = np.concatenate(groups)
+
+    return X, y, groups, Xn

minigrida/loader/coord_tiles.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# file coord_tiles.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 29 mai 2020
+"""Abstract
+
+doc.
+"""
+
+from pathlib import Path
+import logging
+import rasterio as rio
+
+log = logging.getLogger(__name__)
+
+
+def run(rasters_path, rasters_name, gt_suffix='gt.tif'):
+    gt_names = list(Path(rasters_path).glob('*' + gt_suffix))
+    log.info('Found {} ground truth tiles.'.format(len(gt_names)))
+
+    gt = []
+    rasters = []
+    coords = []
+
+    for gtn in gt_names:
+        gt += [load_tif(gtn)]
+
+        rasters += [{Path(n).stem:
+                     load_tif(gtn.as_posix().replace(gt_suffix, '') + n)
+                     for n in rasters_name}]
+
+        coords += ['_'.join(gtn.stem.split('_')[:2])]
+
+    return gt, rasters, coords
+
+
+def load_tif(path):
+    return rio.open(str(path)).read()[0]

minigrida/protocols/__init__.py

@@ -10,3 +10,4 @@
 
 #from .jurse import Jurse
 from .jurse2 import Jurse2
+from .jurse3 import Jurse3

minigrida/protocols/jurse2.py

@@ -21,6 +21,7 @@ class Jurse2(Protocol):
 
     def __init__(self, expe):
         super().__init__(expe, self.__class__.__name__)
+        self._results = {}
 
     def _run(self):
         self._log.info('Load data')
@@ -44,9 +45,7 @@ class Jurse2(Protocol):
         self._log.info('Run metrics')
         metrics = self._run_metrics(classification, descriptors)
 
-        results = {}
+        self._results['metrics'] = metrics
-        results['metrics'] = metrics
-        self._results = results
 
     def _load_data(self):
         data_loader = self._expe['data_loader']
@@ -65,7 +64,7 @@ class Jurse2(Protocol):
         return att
 
     def _compute_classification(self, descriptors):
-        X, y, groups = descriptors
+        X, y, groups, Xn = descriptors
 
         # CrossVal and ML
         cv = self._expe['cross_validation']
@@ -76,6 +75,7 @@ class Jurse2(Protocol):
 
         y_pred = np.zeros_like(y)
 
+        cl_feature_importances = []
         cvi = cross_val(**cv['parameters'])
         for train_index, test_index in cvi.split(X, y, groups):
             cli = classifier(**cl['parameters'])
@@ -86,13 +86,21 @@ class Jurse2(Protocol):
             self._log.info('  - predict')
             y_pred[test_index] = cli.predict(X[test_index])
 
+            cl_feature_importances += [cli.feature_importances_.copy()]
+
+        cl_feature_importances = np.array(cl_feature_importances)
+        self._results['features'] = {
+                'name': list(Xn),
+                'importance': cl_feature_importances.tolist()
+                }
+
         return y_pred
 
     def _get_results(self):
         return self._results
 
     def _run_metrics(self, classification, descriptors):
-        X, y_true, groups = descriptors
+        X, y_true, groups, Xn = descriptors
         y_pred = classification
 
         self._log.info('  - Scores')

minigrida/protocols/jurse3.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# file jurse3.py
+# author Florent Guiotte <florent.guiotte@irisa.fr>
+# version 0.0
+# date 17 sept. 2021
+
+import os
+import importlib
+
+from joblib import Memory
+from pathlib import Path
+
+from . import Jurse2
+
+ENV_KEY = 'MINIGRIDA_CACHE'
+DEFAULT_CACHE = './cache'
+
+CACHE = os.environ[ENV_KEY] \
+            if ENV_KEY in os.environ \
+            else DEFAULT_CACHE
+
+
+class Jurse3(Jurse2):
+    """Jurse2 protocol with cache
+
+    Same as Jurse2 but enable caching results to speed up
+    hyperparameters tunning.
+    """
+    def __init__(self, expe):
+        super().__init__(expe)
+
+        self.memory = Memory(CACHE if Path(CACHE).exists else None, verbose=0)
+
+    def _compute_descriptors(self, data):
+        script = self._expe['descriptors_script']
+
+        desc = importlib.import_module(script['name'])
+        run = self.memory.cache(desc.run)
+        att = run(*data, **script['parameters'])
+
+        return att

minigrida/supervisor.py

@@ -19,13 +19,29 @@ from protocols.protocol import TestError
 import database
 from multiprocessing import Process
 import json
+import argparse
 
 host = os.uname()[1]
 log = logging.getLogger('Supervisor [{}]'.format(host))
 
+parser = argparse.ArgumentParser(description='Run minigrida supervisor')
+parser.add_argument('--conf',
+                    metavar='config',
+                    default='config.json',
+                    type=str,
+                    help='the path to the supervisor\'s config file.')
+parser.add_argument('--cred',
+                    metavar='credentials',
+                    default='credentials.json',
+                    type=str,
+                    help='the path to the DB credentials file.')
+args = parser.parse_args()
 
 def run(expe, hostpid=host):
-    database.update_experiment(expe, worker=hostpid, start_date=datetime.now())
+    database.update_experiment(expe,
+                               worker=hostpid,
+                               status='running',
+                               start_date=datetime.now())
 
     # Load protocol
     log.info('Load protocol {}'.format(expe.protocol))
@@ -53,6 +69,9 @@ def run(expe, hostpid=host):
                                aa=test.aa,
                                k=test.k,
                                report=test.get_results(),
+                               ressources={
+                                   'process_time': test.get_process_time()
+                                   },
                                status='complete')
 
     # End of test
@@ -64,7 +83,7 @@ def main(pid=None):
     log.name = 'Supervisor [{}]'.format(hostpid)
 
     log.info('Connecting to database')
-    database.connect('credentials.json')
+    database.connect(args.cred)
 
     while(True):
         if not database.pending_experiments():
@@ -88,7 +107,7 @@ if __name__ == '__main__':
     logger.setup_logging()
     log.info('Starting supervisor')
     try:
-        with open('config.json') as f:
+        with open(args.conf) as f:
             config = json.load(f)
             process_count = config['process_count']
     except Exception as e:
@@ -97,3 +116,4 @@ if __name__ == '__main__':
 
     for i in range(process_count):
         Process(target=main, args=(i,)).start()
+        time.sleep(1)

requirements.txt

@@ -2,3 +2,4 @@ sklearn
 sap>=0.2.2
 pony
 psycopg2
+pyyml

setup.py

@@ -17,4 +17,8 @@ setup(name='minigrida',
       author_email='florent.guiotte@uhb.fr',
       url='https://git.guiotte.fr/Florent/minigrida',
       packages=['minigrida'],#'cvgenerators', 'descriptors', 'protocols', 'database'],
+      install_requires=[
+          'pony',
+          'psycopg2-binary',
+          ]
       )