From b363c63d44eef05a06f8f7ff7c471ac4392cc4e9 Mon Sep 17 00:00:00 2001
From: "Suren A. Chilingaryan" <csa@ipecompute2.ands.kit.edu>
Date: Wed, 11 Nov 2020 05:29:12 +0100
Subject: Support for ufo & jupyter notebooks
---
build/buildah-astra.sh | 29 +++------------------------
build/buildah-base.sh | 42 ++++++++++++++++++++++++++++++++++++++++
build/buildah-ccpi.sh | 15 ++------------
build/buildah-remote.sh | 20 +++++++++++++++++++
build/buildah-tomo.sh | 11 +++++++++++
build/buildah-ufo.sh | 11 +++++++++++
build/buildah.sh | 3 +++
build/extract.sh | 6 +++++-
build/pre-setup/requirements.txt | 1 +
build/setup/build/cmake.sh | 2 +-
build/setup/build/python.sh | 2 +-
build/setup/repos.sh | 6 ++++++
build/update.sh | 9 ++++++---
run/config.sh | 8 ++++++++
run/devel.sh | 8 ++++----
run/run-devel.sh | 5 -----
run/run-jupyter.sh | 13 +++++++++++++
run/run-ssh.sh | 10 +++++-----
run/run.sh | 5 +++--
run/update-from-devel.sh | 8 +++++---
20 files changed, 150 insertions(+), 64 deletions(-)
create mode 100644 build/buildah-base.sh
create mode 100644 build/buildah-remote.sh
create mode 100644 build/buildah-tomo.sh
create mode 100644 build/buildah-ufo.sh
create mode 100644 run/config.sh
delete mode 100644 run/run-devel.sh
create mode 100644 run/run-jupyter.sh
diff --git a/build/buildah-astra.sh b/build/buildah-astra.sh
index fa69db6..1282e28 100644
--- a/build/buildah-astra.sh
+++ b/build/buildah-astra.sh
@@ -2,32 +2,9 @@
set -o errexit
-container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04)
+container=$(buildah from pycuda:latest)
-buildah config --label maintainer="Suren A. Chilingaryan <csa@suren.me>" $container
-buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" $container
-
-buildah run $container sh -c 'echo "ssh:x:101:" >> /etc/group'
-buildah run $container sh -c 'echo "sshd:x:101:101:sshd:/dev/null:/sbin/nologin" >> /etc/passwd'
-buildah run $container sh -c 'echo "messagebus:x:102:" >> /etc/group'
-buildah run $container sh -c 'echo "messagebus:x:102:102:messagebus:/dev/null:/sbin/nologin" >> /etc/passwd'
-buildah run $container sh -c 'echo "rdma:x:103:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-journal:x:104:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-network:x:105:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-network:x:105:105:systemd-network:/dev/null:/sbin/nologin" >> /etc/passwd'
-buildah run $container sh -c 'echo "systemd-resolve:x:106:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-resolve:x:106:106:systemd-resolve:/dev/null:/sbin/nologin" >> /etc/passwd'
-
-buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git openssh-server'
-buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3'
-buildah run $container sh -c 'apt-get install -y cmake autoconf automake libtool libboost-all-dev'
-
-buildah run $container sh -c 'update-alternatives --remove python /usr/bin/python2'
-buildah run $container sh -c 'update-alternatives --install /usr/bin/python python /usr/bin/python3 10'
-
-buildah copy $container pre-setup /root/pre-setup
-buildah run $container pip3 install -r /root/pre-setup/requirements.txt
-
-buildah copy $container setup /root/setup
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
buildah run $container bash /root/setup/repos.sh "astra"
+
buildah commit --format docker $container astra:latest
diff --git a/build/buildah-base.sh b/build/buildah-base.sh
new file mode 100644
index 0000000..b656515
--- /dev/null
+++ b/build/buildah-base.sh
@@ -0,0 +1,42 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04)
+
+buildah config --label maintainer="Suren A. Chilingaryan <csa@suren.me>" $container
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" $container
+
+buildah run $container ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime
+buildah run $container sh -c 'echo "Europe/London" > /etc/timezone'
+
+buildah run $container sh -c 'echo "ssh:x:101:" >> /etc/group'
+buildah run $container sh -c 'echo "sshd:x:101:101:sshd:/dev/null:/sbin/nologin" >> /etc/passwd'
+buildah run $container sh -c 'echo "messagebus:x:102:" >> /etc/group'
+buildah run $container sh -c 'echo "messagebus:x:102:102:messagebus:/dev/null:/sbin/nologin" >> /etc/passwd'
+buildah run $container sh -c 'echo "rdma:x:103:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-journal:x:104:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-network:x:105:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-network:x:105:105:systemd-network:/dev/null:/sbin/nologin" >> /etc/passwd'
+buildah run $container sh -c 'echo "systemd-resolve:x:106:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-resolve:x:106:106:systemd-resolve:/dev/null:/sbin/nologin" >> /etc/passwd'
+
+buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git openssh-server'
+buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3'
+buildah run $container sh -c 'apt-get install -y cmake make autoconf automake libtool gcc g++ libboost-all-dev'
+buildah run $container sh -c 'apt-get install -y ca-certificates pkg-config gobject-introspection liblapack-dev libjpeg-dev libtiff-dev libglib2.0-dev libjson-glib-dev libopenmpi-dev libhdf5-dev libclfft-dev libgsl-dev libgirepository1.0-dev fftw3-dev zlib1g-dev'
+buildah run $container sh -c 'apt-get install -y python3-dev python3-numpy python-gobject python-tifffile'
+buildah run $container sh -c 'apt-get install -y ocl-icd-opencl-dev clinfo'
+buildah run $container sh -c 'apt-get install -y jupyter-notebook'
+
+buildah run $container sh -c 'update-alternatives --remove python /usr/bin/python2'
+buildah run $container sh -c 'update-alternatives --install /usr/bin/python python /usr/bin/python3 10'
+
+buildah run $container mkdir -p /etc/OpenCL/vendors
+buildah run $container sh -c 'echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd'
+
+buildah copy $container pre-setup /root/pre-setup
+buildah run $container pip3 install -r /root/pre-setup/requirements.txt
+
+buildah copy $container setup /root/setup
+buildah commit --format docker $container pycuda:latest
diff --git a/build/buildah-ccpi.sh b/build/buildah-ccpi.sh
index 1bc6b10..3d006fc 100644
--- a/build/buildah-ccpi.sh
+++ b/build/buildah-ccpi.sh
@@ -4,18 +4,7 @@ set -o errexit
container=$(buildah from astra:latest)
-buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=19.10 --env HOME=/ccpi/data $container
-buildah config --port 2222 $container
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=19.10 --env HOME=/ccpi/data --env LD_LIBRARY_PATH="/ccpi/repos/CCPi-Regularisation-Toolkit/src/Core:/ccpi/repos/CCPi-Framework/src/Core" $container
+buildah run $container bash /root/setup/repos.sh "ccpi"
-buildah run $container bash /root/setup/repos.sh "all"
-
-buildah run $container sh -c "echo '$(id -ng):x:$(id -g):' >> /etc/group"
-#buildah run $container sh -c "echo '$(id -nu):x:$(id -u):$(id -g):$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
-buildah run $container sh -c "echo '$(id -nu):x:0:0:$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
-
-buildah run $container sh -c 'service ssh start'
-buildah run $container sh -c "echo '$(id -nu):test' | chpasswd"
buildah commit --format docker $container ccpi:latest
-
-echo $container
-
diff --git a/build/buildah-remote.sh b/build/buildah-remote.sh
new file mode 100644
index 0000000..1016d19
--- /dev/null
+++ b/build/buildah-remote.sh
@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+image="$1"
+[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX>"; exit 1; }
+
+container=$(buildah from ${image}:latest)
+buildah config --port 2222/tcp --port 8888/tcp $container
+
+buildah run $container sh -c "echo '$(id -ng):x:$(id -g):' >> /etc/group"
+#buildah run $container sh -c "echo '$(id -nu):x:$(id -u):$(id -g):$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
+buildah run $container sh -c "echo '$(id -nu):x:0:0:$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
+
+buildah run $container sh -c 'service ssh start'
+buildah run $container sh -c "echo '$(id -nu):test' | chpasswd"
+
+buildah commit --format docker $container ${image}-remote:latest
+
+echo "$container build from ${image} and saved as ${image}-remote"
diff --git a/build/buildah-tomo.sh b/build/buildah-tomo.sh
new file mode 100644
index 0000000..ed9287e
--- /dev/null
+++ b/build/buildah-tomo.sh
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+container=$(buildah from ccpi:latest)
+
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
+
+buildah run $container bash /root/setup/repos.sh "ufo"
+
+buildah commit --format docker $container tomo:latest
diff --git a/build/buildah-ufo.sh b/build/buildah-ufo.sh
new file mode 100644
index 0000000..ee8e328
--- /dev/null
+++ b/build/buildah-ufo.sh
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+container=$(buildah from pycuda:latest)
+
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
+
+buildah run $container bash /root/setup/repos.sh "ufo"
+
+buildah commit --format docker $container ufo:latest
diff --git a/build/buildah.sh b/build/buildah.sh
index 2f4f283..8e84457 100644
--- a/build/buildah.sh
+++ b/build/buildah.sh
@@ -1,4 +1,7 @@
set -o errexit
+bash buildah-base.sh
bash buildah-astra.sh
bash buildah-ccpi.sh
+bash buildah-tomo.sh
+bash buildah-remote.sh tomo
diff --git a/build/extract.sh b/build/extract.sh
index 5321db8..17c6e3f 100644
--- a/build/extract.sh
+++ b/build/extract.sh
@@ -2,7 +2,11 @@
set -o errexit
-container=$(buildah from localhost/ccpi)
+image="$1"
+[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX>"; exit 1; }
+
+
+container=$(buildah from localhost/${image})
path=$(buildah mount $container)
mkdir -p ../repos/
cp -ra "$path/ccpi/repos"/* ../repos/
diff --git a/build/pre-setup/requirements.txt b/build/pre-setup/requirements.txt
index 87e3dca..e7b8d3b 100644
--- a/build/pre-setup/requirements.txt
+++ b/build/pre-setup/requirements.txt
@@ -4,3 +4,4 @@ scipy
matplotlib
h5py
pillow
+pycuda
diff --git a/build/setup/build/cmake.sh b/build/setup/build/cmake.sh
index 09beccf..1065907 100644
--- a/build/setup/build/cmake.sh
+++ b/build/setup/build/cmake.sh
@@ -4,6 +4,6 @@ set -o errexit
pypath=$(python3 -c 'import site; print(site.getsitepackages()[0])')
#rm CMakeCache.txt
-cmake -D "PYTHON_DEST_DIR:PATH=$pypath" .
+cmake -DCMAKE_INSTALL_PREFIX=/usr -D "PYTHON_DEST_DIR:PATH=$pypath" .
make
make install
diff --git a/build/setup/build/python.sh b/build/setup/build/python.sh
index de08a2c..3694a98 100644
--- a/build/setup/build/python.sh
+++ b/build/setup/build/python.sh
@@ -3,7 +3,7 @@
set -o errexit
(
- cd Wrappers/Python
+ [ -d "Wrappers/Python" ] && cd "Wrappers/Python"
python3 setup.py install
)
diff --git a/build/setup/repos.sh b/build/setup/repos.sh
index 92c1a2f..0c7481a 100644
--- a/build/setup/repos.sh
+++ b/build/setup/repos.sh
@@ -20,6 +20,12 @@ repos_ccpi=( \
"https://github.com/vais-ral/CCPi-Regularisation-Toolkit.git master ccpi.sh" \
)
+repos_ufo=( \
+ "https://github.com/ufo-kit/ufo-core.git master cmake.sh" \
+ "https://github.com/ufo-kit/ufo-filters.git master cmake.sh" \
+ "https://github.com/ufo-kit/tofu master python.sh" \
+)
+
repos_all=("${repos_astra[@]}" "${repos_ccpi[@]}")
declare -n repos="repos_$repo_group"
diff --git a/build/update.sh b/build/update.sh
index 0201a4d..a3dc9ff 100644
--- a/build/update.sh
+++ b/build/update.sh
@@ -2,15 +2,18 @@
set -o errexit
+image="$1"
+[ -n "$image" ] || { echo "Usage: $0 <astra|ufo|ccpi|ccpi-remote|tomo|tomo-remote|imageX>"; exit 1; }
+
#container=cuda-working-container-15
-container=$(buildah from localhost/ccpi)
+container=$(buildah from localhost/${image})
#buildah run $container bash
#exit
-buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data $container
+buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data --env LD_LIBRARY_PATH="/ccpi/repos/CCPi-Regularisation-Toolkit/src/Core:/ccpi/repos/CCPi-Framework/src/Core" $container
buildah copy $container setup /root/setup
buildah run $container bash /root/setup/repos.sh "all" "/root/setup"
#buildah commit --format docker $container ccpi:latest
-echo "buildah commit --format docker $container ccpi:latest"
+echo "buildah commit --format docker $container ${image}:latest"
diff --git a/run/config.sh b/run/config.sh
new file mode 100644
index 0000000..048ecfb
--- /dev/null
+++ b/run/config.sh
@@ -0,0 +1,8 @@
+port="$(($(id -u)%100 + 100))"
+ssh_port="${port}22"
+web_port="${port}80"
+
+image="$1"
+[ -n "$image" ] || image="tomo-remote" # tomo-remote:latest tomo-remote:devel (also ccpi, ufo, tomo)
+
+echo "Image: $image, Ports: $ssh_port, $web_port"
diff --git a/run/devel.sh b/run/devel.sh
index ad8fa15..a15f203 100644
--- a/run/devel.sh
+++ b/run/devel.sh
@@ -2,10 +2,10 @@
set -o errexit
+. config.sh
+
if [ ! -d ../repos ]; then
- buildah unshare bash ../build/extract.sh
+ buildah unshare bash ../build/extract.sh ${image}
fi
-port="$(($(id -u) - 900))22"
-
-podman run --name "ccpi-devel" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p $port:2222/tcp -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/ccpi bash -c "bash /root/setup/provision.sh; bash"
+podman run --name "ccpi-devel" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p $ssh_port:2222/tcp -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/${image} bash -c "bash /root/setup/provision.sh; bash"
diff --git a/run/run-devel.sh b/run/run-devel.sh
deleted file mode 100644
index 2adb95c..0000000
--- a/run/run-devel.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#! /bin/bash
-
-set -o errexit
-
-podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data localhost/ccpi:devel bash
diff --git a/run/run-jupyter.sh b/run/run-jupyter.sh
new file mode 100644
index 0000000..f596a2b
--- /dev/null
+++ b/run/run-jupyter.sh
@@ -0,0 +1,13 @@
+#! /bin/bash
+
+set -o errexit
+
+. config.sh
+
+echo "Running jupyter on the port $web_port"
+echo "Create ssh tunnel and use browser to connect http://localhost:$web_port"
+echo " ssh -L $web_port:localhost:$web_port $(hostname)"
+echo "Generate password file '/ccpi/data/.jupyter/jupyter_notebook_config.json' with running in the container the following command"
+echo " jupyter notebook password"
+
+podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p ${web_port}:8888/tcp -v ~/ccpi/data:/ccpi/data localhost/${image} jupyter notebook --allow-root --no-browser --ip 0.0.0.0 --port=8888 --notebook-dir=/ccpi/data/ --debug
\ No newline at end of file
diff --git a/run/run-ssh.sh b/run/run-ssh.sh
index 950024a..c467334 100644
--- a/run/run-ssh.sh
+++ b/run/run-ssh.sh
@@ -2,12 +2,12 @@
set -o errexit
-port="$(($(id -u) - 900))22"
+. config.sh
-echo "Running on the port $port"
+echo "Running ssh on the port $ssh_port"
echo "Create ssh tunnel and connect:"
-echo " ssh -L $port:localhost:$port $(hostname)"
+echo " ssh -L $ssh_port:localhost:$ssh_port $(hostname)"
echo " and connect locally "
-echo " ssh -p $port localhost"
+echo " ssh -p $ssh_port localhost"
-podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p ${port}:2222/tcp -v ~/ccpi/data:/ccpi/data localhost/ccpi /usr/sbin/sshd -D -p 2222
+podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p ${ssh_port}:2222/tcp -v ~/ccpi/data:/ccpi/data localhost/${image} /usr/sbin/sshd -D -p 2222
diff --git a/run/run.sh b/run/run.sh
index 0010e1d..bdfade3 100644
--- a/run/run.sh
+++ b/run/run.sh
@@ -2,5 +2,6 @@
set -o errexit
-port="$(($(id -u) - 900))22"
-podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p $port:2222/tcp -v ~/ccpi/data:/ccpi/data localhost/ccpi bash
+. config.sh
+
+podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p ${web_port}:8888/tcp -p $ssh_port:2222/tcp -v ~/ccpi/data:/ccpi/data localhost/${image} bash
diff --git a/run/update-from-devel.sh b/run/update-from-devel.sh
index 19aee6b..b2a8b25 100644
--- a/run/update-from-devel.sh
+++ b/run/update-from-devel.sh
@@ -2,11 +2,13 @@
set -o errexit
+. config.sh
+
if [ ! -d ../repos ]; then
- buildah unshare bash ../build/extract.sh
+ buildah unshare bash ../build/extract.sh ${image}
fi
-podman run --name "ccpi-devel" -it --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/ccpi bash /root/setup/provision.sh
+podman run --name "ccpi-devel" -it --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/${image} bash /root/setup/provision.sh
podman container cp ../repos/ ccpi-devel:/ccpi/
-podman container commit ccpi-devel ccpi:devel
+podman container commit ccpi-devel ${image}:devel
podman rm ccpi-devel
--
cgit v1.2.3