{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "sys.path.append(\"..\")\n",
    "import rasterizer\n",
    "import raster_assistant as ra\n",
    "\n",
    "sys.path.append('../triskele/python/')\n",
    "import triskele\n",
    "\n",
    "figsize = np.array((16, 3)) * 1.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tresholds for Custom Raster from DFC LiDAR data\n",
    "\n",
    "Compare our results with the DFC rasters and set the tresholds for the raster factory.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load DFC raster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfc_raster = triskele.read('../Data/phase1_rasters/Intensity_C1/UH17_GI1F051_TR.tif')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(dfc_raster)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The raster from DFC dataset are noised with high value noise. We need to filter high values. We empirically set the treshold to 1e4."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.clip(dfc_raster, dfc_raster.min(), 1e4, out=dfc_raster)\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(dfc_raster)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set filtering and clipping treshold to process rasters from LiDAR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data without filtering or clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C1 = ra.bulk_load('../Data/lidar/C1', 'C1', filter_treshold=0, clip_treshold=0, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we process the raster with the same resolution and a nearest interpolation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_c0 = ra.rasterize_cache('intensity', C1, .5, 'nearest', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_c0)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also have high value noise, but far better than the DFC noise."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data without filtering and minimal clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C1 = ra.bulk_load('../Data/lidar/C1', 'C1', filter_treshold=0, clip_treshold=0.01, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we process the raster with the same resolution and a nearest interpolation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_c0_01 = ra.rasterize_cache('intensity', C1, .5, 'nearest', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_c0_01)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clipping does not remove unwanted high value noise."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data with minimal filtering and no clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C1 = ra.bulk_load('../Data/lidar/C1', 'C1', filter_treshold=0.01, clip_treshold=0, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we process the raster with the same resolution and a nearest interpolation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_01_c0 = ra.rasterize_cache('intensity', C1, .5, 'nearest', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_01_c0)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filtering remove high value noise, but the tone mapping is bad (too dark)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data with filtering and no clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C1 = ra.bulk_load('../Data/lidar/C1', 'C1', filter_treshold=0.1, clip_treshold=0, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we process the raster with the same resolution and a nearest interpolation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_1_c0 = ra.rasterize_cache('intensity', C1, .5, 'nearest', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_1_c0)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tone mapping is correct, but interpolation artifacts appears where too much points are removed from filtering (e.g. in the stadium)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data without filtering and with clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C1 = ra.bulk_load('../Data/lidar/C1', 'C1', filter_treshold=0, clip_treshold=0.1, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we process the raster with the same resolution and a nearest interpolation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_c0_1 = ra.rasterize_cache('intensity', C1, .5, 'nearest', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_c0_1)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tone map is correct, no interpolation artifact but high noise sparkle the result."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data with minimal filtering and minimal clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C1 = ra.bulk_load('../Data/lidar/C1', 'C1', filter_treshold=0.01, clip_treshold=0.01, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we process the raster with the same resolution and a nearest interpolation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_01_c0_01 = ra.rasterize_cache('intensity', C1, .5, 'nearest', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_01_c0_01)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tone map is not correct."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data with minimal filtering and normal clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C1 = ra.bulk_load('../Data/lidar/C1', 'C1', filter_treshold=0.01, clip_treshold=0.1, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we process the raster with the same resolution and a nearest interpolation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_01_c0_1 = ra.rasterize_cache('intensity', C1, .5, 'nearest', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_01_c0_1)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tone map is correct, no interpolation artifact and low high noise in the result. We will now on choose "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare interpolation method\n",
    "\n",
    "### Nearest neighbour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_01_c0_1_nearest = ra.rasterize_cache('intensity', C1, .5, 'nearest', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_01_c0_1_nearest)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear interpolation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_01_c0_1 = ra.rasterize_cache('intensity', C1, .5, 'linear', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_01_c0_1)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cubic interpolation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_01_c0_1 = ra.rasterize_cache('intensity', C1, .5, 'cubic', False, cache_dir='../Res/enrichment_rasters')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_01_c0_1)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cubic interpolation seems to create negative values, maybe at the same spots of the DFC high noise ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize)\n",
    "plt.imshow((raster_f0_01_c0_1 < 0) * 1.)\n",
    "plt.colorbar()\n",
    "plt.title('Cubic low noise')\n",
    "plt.show()\n",
    "\n",
    "dfc_raster_raw = triskele.read('../Data/phase1_rasters/Intensity_C1/UH17_GI1F051_TR.tif')\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow((dfc_raster_raw > 1e4) * 1.)\n",
    "plt.colorbar()\n",
    "plt.title('DFC high noise')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(np.logical_and((dfc_raster_raw > 1e4), (raster_f0_01_c0_1 < 0)) * 1)\n",
    "plt.colorbar()\n",
    "plt.title('DFC high noise and Cubic low noise')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow((dfc_raster_raw > 1e4) * 1 - (raster_f0_01_c0_1 < 0) * 1)\n",
    "plt.colorbar()\n",
    "plt.title('DFC high noise minus Cubic low noise')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Numerous common noise pixel between DFC noise and our cubic interpolation.\n",
    "\n",
    "Let's try with our high noise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize)\n",
    "plt.imshow((raster_f0_01_c0_1 > raster_f0_01_c0_1_nearest.max()) * 1.)\n",
    "plt.colorbar()\n",
    "plt.title('Cubic high noise')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow((dfc_raster_raw > 1e4) * 1.)\n",
    "plt.colorbar()\n",
    "plt.title('DFC high noise')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(np.logical_and((dfc_raster_raw > 1e4), (raster_f0_01_c0_1 > raster_f0_01_c0_1_nearest.max())) * 1)\n",
    "plt.colorbar()\n",
    "plt.title('DFC high noise and Cubic low noise')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow((dfc_raster_raw > 1e4) * 1 - (raster_f0_01_c0_1 > raster_f0_01_c0_1_nearest.max()) * 1)\n",
    "plt.colorbar()\n",
    "plt.title('DFC high noise minus Cubic low noise')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Very low correlation between our raster and the DFC high noise.\n",
    "\n",
    "### Filter low and high interpolated values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster_f0_01_c0_1_postprocess = np.clip(raster_f0_01_c0_1, C1.intensity.min(), C1.intensity.max())\n",
    "\n",
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster_f0_01_c0_1_postprocess)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TMP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = ra.rasterize_cache('intensity', C1, .5, 'cubic-clip', False, cache_dir='../Res/enrichment_rasters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imsave('../Res/postprocess_b.png', raster_f0_01_c0_1_postprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imsave('../Res/dfc_c1.png', dfc_raster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster = ra.rasterize_cache('intensity', C1, 1., 'nearest', False, cache_dir='../Res/enrichment_rasters')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Raster Validation\n",
    "\n",
    "### Rasterize some data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raster = rasterizer.rasterize(C1.spatial, C1.intensity, 0.5, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=figsize)\n",
    "plt.imshow(raster, origin='upper')\n",
    "plt.show()\n",
    "plt.imsave('../Res/raster_validation.png', raster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(raster.flatten(), bins=1000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write TIFF file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "triskele.write('../Res/validation.tiff', raster)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare with DFC dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfc = triskele.read('../Data/phase1_rasters/Intensity_C1/UH17_GI1F051_TR.tif')\n",
    "our = triskele.read('../Res/validation.tiff')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filter DFC with same parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ra.auto_filter(dfc, treshold=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfc.shape, our.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfc.dtype, our.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(dfc.flatten(), bins=1000, label='DFC')\n",
    "plt.hist(our.flatten(), bins=1000, label='Our', alpha=.8)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display Rasters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f, axs = plt.subplots(2, figsize=figsize * 2)\n",
    "\n",
    "axs[0].imshow(dfc)\n",
    "axs[0].set_title('DFC')\n",
    "axs[1].imshow(our)\n",
    "axs[1].set_title('Our')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Raster Pack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C123.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multiprocessing import Pool, Process, Queue\n",
    "import multiprocessing as mp\n",
    "#mp.set_start_method('spawn')\n",
    "\n",
    "def rasterize_cache_mp(data_var, field, res, method, reverse, cache):\n",
    "    if data_var == 'C1':\n",
    "        ra.rasterize_cache(C1, field, res, method, reverse, cache)\n",
    "    if data_var == 'C2':\n",
    "        ra.rasterize_cache(C2, field, res, method, reverse, cache)\n",
    "    if data_var == 'C3':\n",
    "        ra.rasterize_cache(C3, field, res, method, reverse, cache)\n",
    "    if data_var == 'C123':\n",
    "        ra.rasterize_cache(C123, field, res, method, reverse, cache)\n",
    "    \n",
    "pool = Pool(processes=5)\n",
    "\n",
    "job_args = list()\n",
    "\n",
    "for res in (0.5, 1., 2., 3., 5., 10., .1):\n",
    "    for reverse in (False, True):\n",
    "        for inter in ('linear', 'nearest'):\n",
    "            for field in ('z', 'intensity', 'num_returns'):\n",
    "                for data in ('C1', 'C2', 'C3', 'C123'):\n",
    "                    job_args.append([data, field, res, inter, reverse, '../Res/HVR/'])\n",
    "                    \n",
    "for i in pool.starmap(rasterize_cache_mp, job_args):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}