From a70c782931d00025a4f0b3414862b5de79923122 Mon Sep 17 00:00:00 2001 From: Karamaz0V1 Date: Thu, 11 Jun 2020 11:45:41 +0200 Subject: [PATCH] Pool for descriptors --- minigrida/descriptors/aps.py | 20 +++++++++++--------- minigrida/descriptors/sdaps.py | 26 ++++++++++++++------------ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/minigrida/descriptors/aps.py b/minigrida/descriptors/aps.py index 4521a74..8d7d94b 100644 --- a/minigrida/descriptors/aps.py +++ b/minigrida/descriptors/aps.py @@ -10,17 +10,15 @@ doc. import numpy as np import sap -from joblib import Memory - -memory = Memory(location='cache/', verbose=0) +from multiprocessing import Pool -@memory.cache def _attribute_profiles(*kwargs): return sap.attribute_profiles(*kwargs) -def run(gt, rasters, coords, remove, attributes, adjacency='4', filtering='direct'): +def run(gt, rasters, coords, remove, + attributes, adjacency='4', filtering='direct', dtype=np.float32): X = [] y = [] groups = [] @@ -29,9 +27,13 @@ def run(gt, rasters, coords, remove, attributes, adjacency='4', filtering='direc for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)): # Compute EAP attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes - eap = [] - for (name, raster), attribute in zip(rastersi.items(), attributes): - eap += [_attribute_profiles(raster, attribute, adjacency, name, filtering)] + pool = Pool() + eap = pool.starmap(_attribute_profiles, [ + (raster, attribute, adjacency, name, filtering) + for (name, raster), attribute + in zip(rastersi.items(), attributes)]) + pool.close() + pool.join() eap = sap.concatenate(eap) Xn = [' '.join((a['tree']['image_name'], @@ -40,7 +42,7 @@ def run(gt, rasters, coords, remove, attributes, adjacency='4', filtering='direc for a in eap.description for p in a['profiles']] if not Xn else Xn # Create vectors - X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1) + X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype) y_raw = gti # Remove unwanted label X, y diff --git a/minigrida/descriptors/sdaps.py b/minigrida/descriptors/sdaps.py index 0e1854a..28d0893 100644 --- a/minigrida/descriptors/sdaps.py +++ b/minigrida/descriptors/sdaps.py @@ -10,17 +10,15 @@ doc. import numpy as np import sap -from joblib import Memory - -memory = Memory(location='cache/', verbose=0) +from multiprocessing import Pool -@memory.cache def _self_dual_attribute_profiles(*kwargs): return sap.self_dual_attribute_profiles(*kwargs) -def run(gt, rasters, coords, remove, attributes, adjacency='4', filtering='direct'): +def run(gt, rasters, coords, remove, + attributes, adjacency='4', filtering='direct', dtype=np.float32): X = [] y = [] groups = [] @@ -29,18 +27,22 @@ def run(gt, rasters, coords, remove, attributes, adjacency='4', filtering='direc for i, (gti, rastersi, coordsi) in enumerate(zip(gt, rasters, coords)): # Compute EAP attributes = [attributes] * len(rastersi) if isinstance(attributes, dict) else attributes - eap = [] - for (name, raster), attribute in zip(rastersi.items(), attributes): - eap += [_self_dual_attribute_profiles(raster, attribute, adjacency, name, filtering)] + pool = Pool() + eap = pool.starmap(_self_dual_attribute_profiles, [ + (raster, attribute, adjacency, name, filtering) + for (name, raster), attribute + in zip(rastersi.items(), attributes)]) + pool.close() + pool.join() eap = sap.concatenate(eap) - Xn = [' '.join((a['tree']['image_name'], - a['attribute'], - *[str(v) for v in p.values()])) + Xn = [' '.join((a['tree']['image_name'], + a['attribute'], + *[str(v) for v in p.values()])) for a in eap.description for p in a['profiles']] if not Xn else Xn # Create vectors - X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1) + X_raw = np.moveaxis(np.array(list(eap.vectorize())), 0, -1).astype(dtype) y_raw = gti # Remove unwanted label X, y