From 83a92476273635382a761f02f67540ab6879e76a Mon Sep 17 00:00:00 2001 From: Karamaz0V1 Date: Mon, 17 Sep 2018 22:19:37 +0200 Subject: [PATCH 1/6] Add docker virtualization --- Dockerfile | 40 ++++++++++++++++++++++++++++++++++++++++ cvgenerators/jurse.py | 1 - docker-compose.yml | 8 ++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dc145f4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +#FROM base/archlinux +FROM arch-python + +WORKDIR /app + +#ADD . /app + +# Update +#RUN pacman -Syu --no-comfirm + +# minigrida deps +#RUN pacman -S --noconfirm python-gdal python-yaml + +# triskele deps (build) +RUN pacman -S --noconfirm git python-pip make gcc intel-tbb boost + +# LD2DAPs deps +RUN pacman -S --noconfirm python-pandas python-scikit-learn + +# !#$*(&@(&(%* +RUN pacman -S --noconfirm python-matplotlib + +# Dependencies +# TODO: prod pip install +# TODO: remove source +## Triskele +RUN git clone https://git.normalized.xyz/Florent/Triskele.git \ + && cd Triskele \ + && make -f MakefileNoOTB apGenerator \ + && cd python \ + && pip install -e . + +## LD2DAPs +RUN git clone https://git.normalized.xyz/Florent/LD2DAPs.git \ + && cd LD2DAPs \ + && pip install -e . + +RUN mkdir Enrichment Data + +CMD ["python", "supervisor.py"] diff --git a/cvgenerators/jurse.py b/cvgenerators/jurse.py index 1f6948e..c68eb20 100644 --- a/cvgenerators/jurse.py +++ b/cvgenerators/jurse.py @@ -9,7 +9,6 @@ # TODO details import numpy as np -import ipdb class Split: """Geographic split cross validation generator. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..956ef97 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3' +services: + minigrida: + build: . + image: minigrida + volumes: + - ../TMP_thecomedian_Enrichment:/app/Enrichment + - ../TMP_thecomedian_Data:/app/Data From b2de4a484618b20b6dcb067b33b23105815bd992 Mon Sep 17 00:00:00 2001 From: Karamaz0V1 Date: Tue, 18 Sep 2018 19:15:03 +0200 Subject: [PATCH 2/6] Add nfs volumes --- docker-compose-nfs.yml | 30 ++++++++++++++++++++++++++++++ docker-compose.yml | 19 ++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 docker-compose-nfs.yml diff --git a/docker-compose-nfs.yml b/docker-compose-nfs.yml new file mode 100644 index 0000000..1d3c80f --- /dev/null +++ b/docker-compose-nfs.yml @@ -0,0 +1,30 @@ +version: '3.7' + +# NFS server require `nfs` and `nfsd` module. You can add them in +# `/etc/modules-load.d/nfs.conf` for automatic handling during boot. + +services: + minigrida-storage: + image: erichough/nfs-server + environment: + NFS_EXPORT_0: '/nfs *(rw,sync,crossmnt,fsid=0)' + NFS_EXPORT_1: '/nfs/music *(rw,sync)' + NFS_EXPORT_2: '/nfs/home *(rw,sync,nohide)' + + #NFS_EXPORT_0: '/nfs *(rw,all_squash,anonuid=1000,anongid=974,fsid=0)' + #NFS_EXPORT_0: '/nfs *(rw,sync,crossmnt,fsid=0)' + #NFS_EXPORT_1: '/nfs/enrichment *(rw,sync,all_squash,fsid=0,anonuid=99,anongid=99)' + #NFS_EXPORT_2: '/nfs/data *(rw,sync,all_squash,fsid=0,anonuid=99,anongid=99)' + volumes: + - ./nfs_data:/nfs + #- data:/nfs/data + #- enrichment:/nfs/enrichment + ports: + - 2049:2049 + cap_add: + - SYS_ADMIN + +volumes: + data: + enrichment: + nfs: diff --git a/docker-compose.yml b/docker-compose.yml index 956ef97..597d7f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,21 @@ -version: '3' +version: '3.7' + services: minigrida: build: . image: minigrida volumes: - - ../TMP_thecomedian_Enrichment:/app/Enrichment - - ../TMP_thecomedian_Data:/app/Data + - enrichment:/app/Enrichment + - data:/app/Data + +volumes: + data: + driver_opts: + type: "nfs" + o: "addr=172.22.160.165,nolock,soft,rw" + device: ":/srv/nfs/data" + enrichment: + driver_opts: + type: "nfs" + o: "addr=172.22.160.165,nolock,soft,rw" + device: ":/srv/nfs/enrichment" From 7ce3fe0cc73a8c320c6b77febbd47903f045ac83 Mon Sep 17 00:00:00 2001 From: Karamaz0V1 Date: Wed, 19 Sep 2018 15:33:35 +0200 Subject: [PATCH 3/6] Add compose for registry --- Dockerfile | 10 +++++----- docker-compose-registry.yml | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 docker-compose-registry.yml diff --git a/Dockerfile b/Dockerfile index dc145f4..e303bef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,15 @@ -#FROM base/archlinux -FROM arch-python +FROM base/archlinux +#FROM arch-python WORKDIR /app -#ADD . /app +ADD . /app # Update -#RUN pacman -Syu --no-comfirm +RUN pacman -Syu --noconfirm # minigrida deps -#RUN pacman -S --noconfirm python-gdal python-yaml +RUN pacman -S --noconfirm python-gdal python-yaml # triskele deps (build) RUN pacman -S --noconfirm git python-pip make gcc intel-tbb boost diff --git a/docker-compose-registry.yml b/docker-compose-registry.yml new file mode 100644 index 0000000..9d8f06d --- /dev/null +++ b/docker-compose-registry.yml @@ -0,0 +1,14 @@ +version: '3.7' + +services: + registry: + image: registry + restart: always + volumes: + - storage:/var/lib/registry + ports: + - 5000:5000 + +volumes: + storage: + From d660b1de4ec4690ca82f4abb50a2ed049c2a57d2 Mon Sep 17 00:00:00 2001 From: Karamaz0V1 Date: Wed, 19 Sep 2018 15:48:00 +0200 Subject: [PATCH 4/6] Try global deploy --- docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 597d7f6..75b5c7e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,19 +3,21 @@ version: '3.7' services: minigrida: build: . - image: minigrida + image: metropolis:5000/minigrida volumes: - enrichment:/app/Enrichment - data:/app/Data + deploy: + mode: global volumes: data: driver_opts: type: "nfs" - o: "addr=172.22.160.165,nolock,soft,rw" + o: "addr=metropolis,nolock,soft,rw" device: ":/srv/nfs/data" enrichment: driver_opts: type: "nfs" - o: "addr=172.22.160.165,nolock,soft,rw" + o: "addr=metropolis,nolock,soft,rw" device: ":/srv/nfs/enrichment" From 959f55acf89fd15aae29b22fc778b941be9fbab0 Mon Sep 17 00:00:00 2001 From: Karamaz0V1 Date: Wed, 19 Sep 2018 18:21:37 +0200 Subject: [PATCH 5/6] Prepare for production --- Dockerfile | 8 ++++---- docker-compose.yml | 2 ++ supervisor.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e303bef..33c36ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,7 @@ FROM base/archlinux -#FROM arch-python WORKDIR /app -ADD . /app - # Update RUN pacman -Syu --noconfirm @@ -26,7 +23,7 @@ RUN pacman -S --noconfirm python-matplotlib ## Triskele RUN git clone https://git.normalized.xyz/Florent/Triskele.git \ && cd Triskele \ - && make -f MakefileNoOTB apGenerator \ + && make -f MakefileNoOTB apGenerator -j 8\ && cd python \ && pip install -e . @@ -37,4 +34,7 @@ RUN git clone https://git.normalized.xyz/Florent/LD2DAPs.git \ RUN mkdir Enrichment Data +## Minigrida scripts +ADD . /app + CMD ["python", "supervisor.py"] diff --git a/docker-compose.yml b/docker-compose.yml index 75b5c7e..f45778d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,8 @@ services: minigrida: build: . image: metropolis:5000/minigrida + environment: + - HOST volumes: - enrichment:/app/Enrichment - data:/app/Data diff --git a/supervisor.py b/supervisor.py index bada05e..48d4089 100644 --- a/supervisor.py +++ b/supervisor.py @@ -122,7 +122,7 @@ def run(expe_file): def create_report(experience, stime=None, etime=None): expe_report = OrderedDict() - expe_report['supervisor'] = os.uname()[1] + expe_report['supervisor'] = os.environ["HOST"] # Dates for datek, timev in zip(('start_date', 'end_date'), (stime, etime)): From 71c8595e2a59da4ca3b132313c2a8fedbf407ed4 Mon Sep 17 00:00:00 2001 From: Karamaz0V1 Date: Thu, 20 Sep 2018 09:48:07 +0200 Subject: [PATCH 6/6] Fix hostname --- supervisor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/supervisor.py b/supervisor.py index 48d4089..4b7c000 100644 --- a/supervisor.py +++ b/supervisor.py @@ -122,7 +122,8 @@ def run(expe_file): def create_report(experience, stime=None, etime=None): expe_report = OrderedDict() - expe_report['supervisor'] = os.environ["HOST"] + host = os.getenv("HOST") + expe_report['supervisor'] = host if host is not None else os.uname()[1] # Dates for datek, timev in zip(('start_date', 'end_date'), (stime, etime)):