config.json

@@ -1,3 +1,3 @@
 {
-    "process_count": 1
+    "process_count": 2
 }

minigrida/cvgenerators/__init__.py

@@ -1 +0,0 @@
-from .generic import *

minigrida/cvgenerators/generic.py

@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-# file generic.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 08 juil. 2020
-"""Abstract
-
-doc.
-"""
-
-from sklearn.model_selection import GroupKFold, StratifiedShuffleSplit
-
-
-class GroupStratifiedShuffleSplit(GroupKFold):
-    def __init__(self, n_splits=5, train_size=None, random_state=None):
-        super().__init__(n_splits)
-        self.train_size = train_size
-        self.random_state = random_state
-    
-    def split(self, X, y=None, groups=None):
-        splits = [s for s in super().split(X, y, groups)]
-        for train, test in splits:
-            sss_train, sss_test = next(StratifiedShuffleSplit(1, train_size=self.train_size, random_state=self.random_state).split(X[train], y[train], groups[train]))
-            
-            yield train[sss_train], test

minigrida/database/helpers.py

@@ -16,7 +16,7 @@ import logging
 
 from .design import Session, Experiment, Project, db
 
-log = logging.getLogger(__name__)
+log = logging.getLogger()
 
 def compute_expe_hash(experiment):
     return hashlib.md5(json.dumps(experiment, sort_keys=True).encode('utf-8')).hexdigest()
@@ -34,7 +34,7 @@ def create_experiment(session_name, protocol, expe, urgency=1):
         e = q.first()
         e.sessions.add(session)
     else:
-        Experiment(sessions=session, protocol=protocol, expe=expe, expe_hash=expe_hash, urgency=urgency)
+        Experiment(sessions=session, protocol=protocol, expe=expe, expe_hash=expe_hash)
 
 @orm.db_session
 def create_project(name):
@@ -58,7 +58,7 @@ def pending_experiments():
     return Experiment.select(lambda x: x.status == 'pending').exists()
 
 
-@orm.db_session(optimistic=False)
+@orm.db_session(serializable=True, optimistic=False)
 def next_experiment():
     # TODO: take session urgency into account
     expe = orm.select(e for e in Experiment
@@ -81,7 +81,7 @@ def update_experiment(expe, **params):
         _update_experiment(expe, **params)
 
 
-@orm.db_session(optimistic=False, retry=3)
+@orm.db_session
 def _update_experiment(expe, **params):
     e = Experiment.get_for_update(id=expe.id)
     for k, v in params.items():

minigrida/descriptors/aps.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-# file aps.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 30 mai 2020
-"""Abstract
-
-doc.
-"""
-
-import numpy as np
-import sap
-from multiprocessing import Pool
-
-
-def _attribute_profiles(*kwargs):
-    return sap.attribute_profiles(*kwargs)
-
-
-def run(gt, rasters, coords, remove,
-        attributes, adjacency='4', filtering='direct', dtype=np.float32):
-    X = []
-    y = []
-    groups = []
-    Xn = None
-
-    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
-        # Compute EAP
-        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
-        pool = Pool()
-        eap = pool.starmap(_attribute_profiles, [
-                           (raster, attribute, adjacency, name, filtering)
-                           for (name, raster), attribute
-                           in zip(rastersi.items(), attributes)])
-        pool.close()
-        pool.join()
-        eap = sap.concatenate(eap)
-
-        Xn = [' '.join((a.description['tree']['image_name'],
-                        a.description['attribute'],
-                        *[str(v) for v in p.values()]))
-                       for a in eap for p in a.description['profiles']] if not Xn else Xn
-
-        # Create vectors
-        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
-        y_raw = gti
-
-        # Remove unwanted label X, y
-        lbl = np.ones_like(y_raw, dtype=np.bool)
-        for l in remove if remove else []:
-            lbl &= y_raw != l
-
-        X += [X_raw[lbl]]
-        y += [y_raw[lbl]]
-        groups += [np.repeat(coordsi, lbl.sum())]
-
-    X = np.concatenate(X)
-    y = np.concatenate(y)
-    groups = np.concatenate(groups)
-
-    return X, y, groups, Xn

minigrida/descriptors/daps.py

@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-# file daps.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 07 juil. 2020
-
-#!/usr/bin/env python
-# file aps.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 30 mai 2020
-"""Abstract
-
-doc.
-"""
-
-import numpy as np
-import sap
-from multiprocessing import Pool
-
-
-def _diff_attribute_profiles(*kwargs):
-    return sap.attribute_profiles(*kwargs).diff()
-
-
-def run(gt, rasters, coords, remove,
-        attributes, adjacency='4', filtering='direct', dtype=np.float32):
-    X = []
-    y = []
-    groups = []
-    Xn = None
-
-    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
-        # Compute EAP
-        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
-        pool = Pool()
-        eap = pool.starmap(_diff_attribute_profiles, [
-                           (raster, attribute, adjacency, name, filtering)
-                           for (name, raster), attribute
-                           in zip(rastersi.items(), attributes)])
-        pool.close()
-        pool.join()
-        eap = sap.concatenate(eap)
-
-        Xn = [' '.join((a.description['tree']['image_name'],
-                a.description['attribute'],
-                *[sap.profiles._title(p)]))
-               for a in eap for p in a.description['profiles']] if not Xn else Xn
-
-        # Create vectors
-        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
-        y_raw = gti
-
-        # Remove unwanted label X, y
-        lbl = np.ones_like(y_raw, dtype=np.bool)
-        for l in remove if remove else []:
-            lbl &= y_raw != l
-
-        X += [X_raw[lbl]]
-        y += [y_raw[lbl]]
-        groups += [np.repeat(coordsi, lbl.sum())]
-
-    X = np.concatenate(X)
-    y = np.concatenate(y)
-    groups = np.concatenate(groups)
-
-    return X, y, groups, Xn

minigrida/descriptors/dsdaps.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-# file dsdaps.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 07 juil. 2020
-"""Abstract
-
-doc.
-"""
-
-import numpy as np
-import sap
-from multiprocessing import Pool
-
-
-def _diff_sd_attribute_profiles(*kwargs):
-    return sap.self_dual_attribute_profiles(*kwargs).diff()
-
-
-def run(gt, rasters, coords, remove,
-        attributes, adjacency='4', filtering='direct', dtype=np.float32):
-    X = []
-    y = []
-    groups = []
-    Xn = None
-
-    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
-        # Compute EAP
-        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
-        pool = Pool()
-        eap = pool.starmap(_diff_sd_attribute_profiles, [
-                           (raster, attribute, adjacency, name, filtering)
-                           for (name, raster), attribute
-                           in zip(rastersi.items(), attributes)])
-        pool.close()
-        pool.join()
-        eap = sap.concatenate(eap)
-
-        Xn = [' '.join((a.description['tree']['image_name'],
-                a.description['attribute'],
-                *[sap.profiles._title(p)]))
-               for a in eap for p in a.description['profiles']] if not Xn else Xn
-
-        # Create vectors
-        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
-        y_raw = gti
-
-        # Remove unwanted label X, y
-        lbl = np.ones_like(y_raw, dtype=np.bool)
-        for l in remove if remove else []:
-            lbl &= y_raw != l
-
-        X += [X_raw[lbl]]
-        y += [y_raw[lbl]]
-        groups += [np.repeat(coordsi, lbl.sum())]
-
-    X = np.concatenate(X)
-    y = np.concatenate(y)
-    groups = np.concatenate(groups)
-
-    return X, y, groups, Xn

minigrida/descriptors/edaps.py

@@ -1,68 +0,0 @@
-#!/usr/bin/env python
-# file edaps.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 07 juil. 2020
-"""Abstract
-
-doc.
-"""
-
-import numpy as np
-import sap
-from multiprocessing import Pool
-
-
-def _diff_attribute_profiles(*kwargs):
-    image = kwargs[0]
-    name = kwargs[3]
-    return sap.attribute_profiles(*kwargs).diff() \
-           + sap.Profiles([image[None]], 
-                          [{'tree': {'image_name': name}, 
-                            'attribute': 'altitude', 
-                            'profiles': [{'operation': 'copy'}]}
-                          ])
-
-
-def run(gt, rasters, coords, remove,
-        attributes, adjacency='4', filtering='direct', dtype=np.float32):
-    X = []
-    y = []
-    groups = []
-    Xn = None
-
-    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
-        # Compute EAP
-        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
-        pool = Pool()
-        eap = pool.starmap(_diff_attribute_profiles, [
-                           (raster, attribute, adjacency, name, filtering)
-                           for (name, raster), attribute
-                           in zip(rastersi.items(), attributes)])
-        pool.close()
-        pool.join()
-        eap = sap.concatenate(eap)
-
-        Xn = [' '.join((a.description['tree']['image_name'],
-                a.description['attribute'],
-                *[sap.profiles._title(p)]))
-               for a in eap for p in a.description['profiles']] if not Xn else Xn
-
-        # Create vectors
-        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
-        y_raw = gti
-
-        # Remove unwanted label X, y
-        lbl = np.ones_like(y_raw, dtype=np.bool)
-        for l in remove if remove else []:
-            lbl &= y_raw != l
-
-        X += [X_raw[lbl]]
-        y += [y_raw[lbl]]
-        groups += [np.repeat(coordsi, lbl.sum())]
-
-    X = np.concatenate(X)
-    y = np.concatenate(y)
-    groups = np.concatenate(groups)
-
-    return X, y, groups, Xn

minigrida/descriptors/edsdaps.py

@@ -1,68 +0,0 @@
-#!/usr/bin/env python
-# file edsdaps.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 07 juil. 2020
-"""Abstract
-
-doc.
-"""
-
-import numpy as np
-import sap
-from multiprocessing import Pool
-
-
-def _diff_sd_attribute_profiles(*kwargs):
-    image = kwargs[0]
-    name = kwargs[3]
-    return sap.self_dual_attribute_profiles(*kwargs).diff() \
-           + sap.Profiles([image[None]], 
-                          [{'tree': {'image_name': name}, 
-                            'attribute': 'altitude', 
-                            'profiles': [{'operation': 'copy'}]}
-                          ])
-
-
-def run(gt, rasters, coords, remove,
-        attributes, adjacency='4', filtering='direct', dtype=np.float32):
-    X = []
-    y = []
-    groups = []
-    Xn = None
-
-    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
-        # Compute EAP
-        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
-        pool = Pool()
-        eap = pool.starmap(_diff_sd_attribute_profiles, [
-                           (raster, attribute, adjacency, name, filtering)
-                           for (name, raster), attribute
-                           in zip(rastersi.items(), attributes)])
-        pool.close()
-        pool.join()
-        eap = sap.concatenate(eap)
-
-        Xn = [' '.join((a.description['tree']['image_name'],
-                a.description['attribute'],
-                *[sap.profiles._title(p)]))
-               for a in eap for p in a.description['profiles']] if not Xn else Xn
-
-        # Create vectors
-        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
-        y_raw = gti
-
-        # Remove unwanted label X, y
-        lbl = np.ones_like(y_raw, dtype=np.bool)
-        for l in remove if remove else []:
-            lbl &= y_raw != l
-
-        X += [X_raw[lbl]]
-        y += [y_raw[lbl]]
-        groups += [np.repeat(coordsi, lbl.sum())]
-
-    X = np.concatenate(X)
-    y = np.concatenate(y)
-    groups = np.concatenate(groups)
-
-    return X, y, groups, Xn

minigrida/descriptors/fps.py

@@ -1,77 +0,0 @@
-#!/usr/bin/env python
-# file fps.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 31 mai 2020
-"""Abstract
-
-doc.
-"""
-
-import numpy as np
-import sap
-from joblib import Memory
-
-memory = Memory(location='cache/', verbose=0)
-
-
-@memory.cache
-def _feature_profiles(**kwargs):
-    return sap.feature_profiles(**kwargs)
-
-
-def _origin_profile(image, name):
-    return sap.Profiles([image[None,:]], [{
-        'attribute': 'Origin',
-        'filtering rule': None,
-        'name': 'copy',
-        'out feature': 'altitude',
-        'profiles': [{'operation': 'copy feature altitude'}],
-        'tree': {'adjacency': None, 'image_hash': None, 'image_name': name}}])
-
-
-def run(gt, rasters, coords, remove, attributes, adjacency=4,
-        feature='same', filtering='direct'):
-    X = []
-    y = []
-    groups = []
-    Xn = None
-
-    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
-        # Compute EAP
-        eap = []
-        for name, raster in rastersi.items():
-            eap += [_origin_profile(raster, name)]
-            eap += [_feature_profiles(image=raster,
-                                      attribute=attributes,
-                                      adjacency=adjacency,
-                                      image_name=name,
-                                      out_feature=feature,
-                                      filtering_rule=filtering)]
-        eap = sap.concatenate(eap)
-
-        Xn = [' '.join((a['tree']['image_name'],
-                        a['attribute'],
-                        'feature',
-                        a['out feature'],
-                        *[str(v) for v in p.values()]))
-                       for a in eap.description for p in a['profiles']] if not Xn else Xn
-
-        # Create vectors
-        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1)
-        y_raw = gti
-
-        # Remove unwanted label X, y
-        lbl = np.ones_like(y_raw, dtype=np.bool)
-        for l in remove if remove else []:
-            lbl &= y_raw != l
-
-        X += [X_raw[lbl]]
-        y += [y_raw[lbl]]
-        groups += [np.repeat(coordsi, lbl.sum())]
-
-    X = np.concatenate(X)
-    y = np.concatenate(y)
-    groups = np.concatenate(groups)
-
-    return X, y, groups, Xn

minigrida/descriptors/max_aps.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python
-# file max_aps.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 31 mai 2020
-"""Abstract
-
-doc.
-"""
-
-import numpy as np
-import sap
-from joblib import Memory
-
-memory = Memory(location='cache/', verbose=0)
-
-
-@memory.cache
-def _max_attribute_profiles(image, attribute, adjacency=4, image_name=None,
-                            filtering_rule='direct'):
-    maxt = sap.MaxTree(image, adjacency, image_name)
-    return sap.create_profiles(maxt, attribute, 'altitude',
-                               filtering_rule, 'max attribute profiles')
-
-
-def run(gt, rasters, coords, remove, attributes, adjacency='4',
-        filtering='direct'):
-    X = []
-    y = []
-    groups = []
-    Xn = None
-
-    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
-        # Compute EAP
-        eap = []
-        for name, raster in rastersi.items():
-            eap += [_max_attribute_profiles(raster, attributes, adjacency, name, filtering)]
-        eap = sap.concatenate(eap)
-
-        Xn = [' '.join((a['tree']['image_name'], 
-                        a['attribute'], 
-                        *[str(v) for v in p.values()])) 
-                       for a in eap.description for p in a['profiles']] if not Xn else Xn
-
-        # Create vectors
-        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1)
-        y_raw = gti
-
-        # Remove unwanted label X, y
-        lbl = np.ones_like(y_raw, dtype=np.bool)
-        for l in remove if remove else []:
-            lbl &= y_raw != l
-
-        X += [X_raw[lbl]]
-        y += [y_raw[lbl]]
-        groups += [np.repeat(coordsi, lbl.sum())]
-
-    X = np.concatenate(X)
-    y = np.concatenate(y)
-    groups = np.concatenate(groups)
-
-    return X, y, groups, Xn

minigrida/descriptors/pixel.py

@@ -11,12 +11,12 @@ doc.
 import numpy as np
 
 
-def run(gt, rasters, coords, remove=None):
+def run(gt, rasters, remove=None):
     X = []
     y = []
     groups = []
 
-    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
+    for i, (gti, rastersi) in enumerate(zip(gt, rasters)):
         # Create vectors
         X_raw = np.moveaxis(np.array(list(rastersi.values())), 0, -1)
         y_raw = gti
@@ -28,11 +28,10 @@ def run(gt, rasters, coords, remove=None):
 
         X += [X_raw[lbl]]
         y += [y_raw[lbl]]
-        groups += [np.repeat(coordsi, lbl.sum())]
+        groups += [np.repeat(i, lbl.sum())]
 
     X = np.concatenate(X)
     y = np.concatenate(y)
     groups = np.concatenate(groups)
-    Xn = rasters[0].keys()
 
-    return X, y, groups, Xn
+    return X, y, groups

minigrida/descriptors/sdaps.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-# file sdaps.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 31 mai 2020
-"""Abstract
-
-doc.
-"""
-
-import numpy as np
-import sap
-from multiprocessing import Pool
-
-
-def _self_dual_attribute_profiles(*kwargs):
-    return sap.self_dual_attribute_profiles(*kwargs)
-
-
-def run(gt, rasters, coords, remove,
-        attributes, adjacency='4', filtering='direct', dtype=np.float32):
-    X = []
-    y = []
-    groups = []
-    Xn = None
-
-    for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)):
-        # Compute EAP
-        attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes
-        pool = Pool()
-        eap = pool.starmap(_self_dual_attribute_profiles, [
-                           (raster, attribute, adjacency, name, filtering)
-                           for (name, raster), attribute
-                           in zip(rastersi.items(), attributes)])
-        pool.close()
-        pool.join()
-        eap = sap.concatenate(eap)
-
-        Xn = [' '.join((a.description['tree']['image_name'],
-                        a.description['attribute'],
-                        *[str(v) for v in p.values()]))
-                       for a in eap for p in a.description['profiles']] if not Xn else Xn
-
-        # Create vectors
-        X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype)
-        y_raw = gti
-
-        # Remove unwanted label X, y
-        lbl = np.ones_like(y_raw, dtype=np.bool)
-        for l in remove if remove else []:
-            lbl &= y_raw != l
-
-        X += [X_raw[lbl]]
-        y += [y_raw[lbl]]
-        groups += [np.repeat(coordsi, lbl.sum())]
-
-    X = np.concatenate(X)
-    y = np.concatenate(y)
-    groups = np.concatenate(groups)
-
-    return X, y, groups, Xn

minigrida/loader/coord_tiles.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# file coord_tiles.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 29 mai 2020
-"""Abstract
-
-doc.
-"""
-
-from pathlib import Path
-import logging
-import rasterio as rio
-
-log = logging.getLogger(__name__)
-
-
-def run(rasters_path, rasters_name, gt_suffix='gt.tif'):
-    gt_names = list(Path(rasters_path).glob('*' + gt_suffix))
-    log.info('Found {} ground truth tiles.'.format(len(gt_names)))
-
-    gt = []
-    rasters = []
-    coords = []
-
-    for gtn in gt_names:
-        gt += [load_tif(gtn)]
-
-        rasters += [{Path(n).stem:
-                     load_tif(gtn.as_posix().replace(gt_suffix, '') + n)
-                     for n in rasters_name}]
-
-        coords += ['_'.join(gtn.stem.split('_')[:2])]
-
-    return gt, rasters, coords
-
-
-def load_tif(path):
-    return rio.open(str(path)).read()[0]

minigrida/protocols/__init__.py

@@ -10,4 +10,3 @@
 
 #from .jurse import Jurse
 from .jurse2 import Jurse2
-from .jurse3 import Jurse3

minigrida/protocols/jurse2.py

@@ -21,7 +21,6 @@ class Jurse2(Protocol):
 
     def __init__(self, expe):
         super().__init__(expe, self.__class__.__name__)
-        self._results = {}
 
     def _run(self):
         self._log.info('Load data')
@@ -45,7 +44,9 @@ class Jurse2(Protocol):
         self._log.info('Run metrics')
         metrics = self._run_metrics(classification, descriptors)
 
-        self._results['metrics'] = metrics
+        results = {}
+        results['metrics'] = metrics
+        self._results = results
 
     def _load_data(self):
         data_loader = self._expe['data_loader']
@@ -64,7 +65,7 @@ class Jurse2(Protocol):
         return att
 
     def _compute_classification(self, descriptors):
-        X, y, groups, Xn = descriptors
+        X, y, groups = descriptors
 
         # CrossVal and ML
         cv = self._expe['cross_validation']
@@ -75,7 +76,6 @@ class Jurse2(Protocol):
 
         y_pred = np.zeros_like(y)
 
-        cl_feature_importances = []
         cvi = cross_val(**cv['parameters'])
         for train_index, test_index in cvi.split(X, y, groups):
             cli = classifier(**cl['parameters'])
@@ -86,21 +86,13 @@ class Jurse2(Protocol):
             self._log.info('  - predict')
             y_pred[test_index] = cli.predict(X[test_index])
 
-            cl_feature_importances += [cli.feature_importances_.copy()]
-
-        cl_feature_importances = np.array(cl_feature_importances)
-        self._results['features'] = {
-                'name': list(Xn),
-                'importance': cl_feature_importances.tolist()
-                }
-
         return y_pred
 
     def _get_results(self):
         return self._results
 
     def _run_metrics(self, classification, descriptors):
-        X, y_true, groups, Xn = descriptors
+        X, y_true, groups = descriptors
         y_pred = classification
 
         self._log.info('  - Scores')

minigrida/protocols/jurse3.py

@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-# file jurse3.py
-# author Florent Guiotte <florent.guiotte@irisa.fr>
-# version 0.0
-# date 17 sept. 2021
-
-import os
-import importlib
-
-from joblib import Memory
-from pathlib import Path
-
-from . import Jurse2
-
-ENV_KEY = 'MINIGRIDA_CACHE'
-DEFAULT_CACHE = './cache'
-
-CACHE = os.environ[ENV_KEY] \
-            if ENV_KEY in os.environ \
-            else DEFAULT_CACHE
-
-
-class Jurse3(Jurse2):
-    """Jurse2 protocol with cache
-
-    Same as Jurse2 but enable caching results to speed up
-    hyperparameters tunning.
-    """
-    def __init__(self, expe):
-        super().__init__(expe)
-
-        self.memory = Memory(CACHE if Path(CACHE).exists else None, verbose=0)
-
-    def _compute_descriptors(self, data):
-        script = self._expe['descriptors_script']
-
-        desc = importlib.import_module(script['name'])
-        run = self.memory.cache(desc.run)
-        att = run(*data, **script['parameters'])
-
-        return att

minigrida/supervisor.py

@@ -19,29 +19,13 @@ from protocols.protocol import TestError
 import database
 from multiprocessing import Process
 import json
-import argparse
 
 host = os.uname()[1]
 log = logging.getLogger('Supervisor [{}]'.format(host))
 
-parser = argparse.ArgumentParser(description='Run minigrida supervisor')
-parser.add_argument('--conf',
-                    metavar='config',
-                    default='config.json',
-                    type=str,
-                    help='the path to the supervisor\'s config file.')
-parser.add_argument('--cred',
-                    metavar='credentials',
-                    default='credentials.json',
-                    type=str,
-                    help='the path to the DB credentials file.')
-args = parser.parse_args()
 
 def run(expe, hostpid=host):
-    database.update_experiment(expe,
-                               worker=hostpid,
-                               status='running',
-                               start_date=datetime.now())
+    database.update_experiment(expe, worker=hostpid, start_date=datetime.now())
 
     # Load protocol
     log.info('Load protocol {}'.format(expe.protocol))
@@ -69,9 +53,6 @@ def run(expe, hostpid=host):
                                aa=test.aa,
                                k=test.k,
                                report=test.get_results(),
-                               ressources={
-                                   'process_time': test.get_process_time()
-                                   },
                                status='complete')
 
     # End of test
@@ -83,7 +64,7 @@ def main(pid=None):
     log.name = 'Supervisor [{}]'.format(hostpid)
 
     log.info('Connecting to database')
-    database.connect(args.cred)
+    database.connect('credentials.json')
 
     while(True):
         if not database.pending_experiments():
@@ -107,7 +88,7 @@ if __name__ == '__main__':
     logger.setup_logging()
     log.info('Starting supervisor')
     try:
-        with open(args.conf) as f:
+        with open('config.json') as f:
             config = json.load(f)
             process_count = config['process_count']
     except Exception as e:
@@ -116,4 +97,3 @@ if __name__ == '__main__':
 
     for i in range(process_count):
         Process(target=main, args=(i,)).start()
-        time.sleep(1)

requirements.txt

@@ -2,4 +2,3 @@ sklearn
 sap>=0.2.2
 pony
 psycopg2
-pyyml

setup.py

@@ -17,8 +17,4 @@ setup(name='minigrida',
       author_email='florent.guiotte@uhb.fr',
       url='https://git.guiotte.fr/Florent/minigrida',
       packages=['minigrida'],#'cvgenerators', 'descriptors', 'protocols', 'database'],
-      install_requires=[
-          'pony',
-          'psycopg2-binary',
-          ]
       )