Fix treshold 👌

Notebooks/Node Treshold.ipynb

@@ -50,6 +50,13 @@
     "disp_test.run()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Metadata copy bug"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -64,7 +71,7 @@
     "in_test = ld2dap.LoadTIFF(rasters)\n",
     "tresh_test.input = in_test\n",
     "tresh2_test.input = tresh_test\n",
-    "disp_test.input = in_test\n",
+    "disp_test.input = tresh_test\n",
     "o.input = in_test\n",
     "\n",
     "\n",
@@ -72,13 +79,10 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "A = list()\n",
-    "A.copy()"
+    "### New treshold"
    ]
   },
   {
@@ -87,7 +91,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "print(o.metadata[0])"
+    "o.data.shape"
    ]
   },
   {
@@ -96,7 +100,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "[x.copy() for x in o.metadata]"
+    "o.data.max(axis=(0,1)).shape, o.data.max(axis=(0,1))"
    ]
   },
   {
@@ -105,8 +109,21 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "mdesc = o.metadata[0].desc\n",
-    "mdesc_copy = o.metadata[0].desc.copy()"
+    "t = 1e3\n",
+    "\n",
+    "o.run()\n",
+    "f = o.data > t\n",
+    "o.data[f] = np.nan\n",
+    "m = np.nanmax(o.data, axis=(0,1))\n",
+    "r = f.sum(axis=(0,1))\n",
+    "\n",
+    "display(m, r)\n",
+    "\n",
+    "o.data = np.rollaxis(o.data, 2)\n",
+    "f = np.rollaxis(f, 2)\n",
+    "\n",
+    "o.data[f] = np.repeat(m, r)\n",
+    "o.data = np.rollaxis(o.data, 0, 3)"
    ]
   },
   {
@@ -115,7 +132,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "o.metadata[0].desc.app"
+    "o.data.max(axis=(0,1))"
    ]
   },
   {
@@ -124,7 +141,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "hex(id(mdesc)), hex(id(o.metadata[0].desc)), hex(id(mdesc_copy))"
+    "A = np.zeros((10,20,2))\n",
+    "display(A.shape)\n",
+    "\n",
+    "A = np.rollaxis(A, 2)\n",
+    "display(A.shape)\n",
+    "\n",
+    "A = np.rollaxis(A,0,3)\n",
+    "A.shape\n"
    ]
   },
   {
@@ -133,7 +157,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "mdesc_copy.append('BDQ')"
+    "A.shape"
    ]
   },
   {
@@ -142,7 +166,67 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "mdesc_copy"
+    "A = np.zeros((10,10,2))\n",
+    "A[:,:,1] = 1\n",
+    "\n",
+    "F = np.random.random((10,10,2)) < .2\n",
+    "\n",
+    "r = F.sum(axis=(0,1))\n",
+    "m = np.array([10, 20])\n",
+    "\n",
+    "A = np.rollaxis(A, 2)\n",
+    "F = np.rollaxis(F, 2)\n",
+    "\n",
+    "A[F] = np.repeat(m, r)\n",
+    "\n",
+    "A = np.rollaxis(A, 0, 3)\n",
+    "A\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "want = f.sum(axis=(0,1))\n",
+    "want"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "np.repeat(np.nanmax(o.data, axis=(0,1)), f.sum(axis=(0,1)))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "f.any()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "o.data[f] = 0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "np.tile(m, (2,1))"
    ]
   }
  ],

ld2dap/Treshold.py

@@ -9,24 +9,36 @@
 # TODO details
 
 from ld2dap.core import Filter
+import numpy as np
+
 
 class Treshold(Filter):
     def __init__(self, treshold, max_value=None):
         super().__init__(self.__class__.__name__)
-        self.treshold  = treshold
-        self.max_value = max_value #if max_value is not None else treshold
+        self.treshold = treshold
+        self.max_value = max_value  # if max_value is not None else treshold
 
     def _process(self, data, metadata):
-        # TODO: UPGRADE RASTER DEPENDANCE
+        # TODO: UPGRADE STACK DEPENDANCE
+        # This filter each raster independently
+        self.logger.info('Filtering')
+
+        treshold_filter = data > self.treshold
+        repeat_count = treshold_filter.sum(axis=(0, 1))
+        data[treshold_filter] = np.nan
+
         if self.max_value is None:
-            self.max_value = data[data < self.treshold].max()
+            self.max_value = np.nanmax(data, axis=(0, 1))
+
+        data = np.rollaxis(data, 2)
+        treshold_filter = np.rollaxis(treshold_filter, 2)
+
+        data[treshold_filter] = np.repeat(self.max_value, repeat_count)
+        data = np.rollaxis(data, 0, 3)
 
         for stack in metadata:
             for d, s in zip(stack.desc, stack.symb):
                 d.append('treshold {}'.format(self.treshold))
-                #s.append('T_{{{}}}'.format(self.treshold))
+                # s.append('T_{{{}}}'.format(self.treshold))
 
-            ## TODO: TMP FIX
-            #data[:,:,stack.begin][data[:,:,stack.begin] > self.treshold] = data[:,:,stack.begin][data[:,:,stack.begin]].max()
-        
-        return data * (data < self.treshold) + self.max_value * (data >= self.treshold), metadata
+        return data, metadata