From 1642ceef7cb622e530b30232536b31bb43ac7ed9 Mon Sep 17 00:00:00 2001 From: Willem Jan Palenstijn Date: Mon, 30 Oct 2017 16:29:04 +0100 Subject: Add libastra/linux-64 conda package variants for cudatoolkit Also upgrade the Docker-based linux release scripts to match. --- python/conda/astra-toolbox/build.sh | 8 ++- python/conda/astra-toolbox/conda_build_config.yaml | 9 +++ python/conda/astra-toolbox/meta.yaml | 8 +-- python/conda/libastra/build.sh | 31 ++++++++-- python/conda/libastra/linux_build_config.yaml | 6 ++ python/conda/libastra/meta.yaml | 11 ++-- python/conda/linux_release/buildenv/Dockerfile | 66 +++++++++++++++++----- python/conda/linux_release/buildenv/build.sh | 13 +++++ python/conda/linux_release/builder/Dockerfile | 17 ------ python/conda/linux_release/release.sh | 16 +++--- 10 files changed, 130 insertions(+), 55 deletions(-) create mode 100644 python/conda/astra-toolbox/conda_build_config.yaml create mode 100644 python/conda/libastra/linux_build_config.yaml create mode 100644 python/conda/linux_release/buildenv/build.sh delete mode 100644 python/conda/linux_release/builder/Dockerfile (limited to 'python') diff --git a/python/conda/astra-toolbox/build.sh b/python/conda/astra-toolbox/build.sh index 951fd88..0468037 100644 --- a/python/conda/astra-toolbox/build.sh +++ b/python/conda/astra-toolbox/build.sh @@ -1,4 +1,10 @@ #!/bin/sh +case `uname` in + Darwin*) + CC="gcc -stdlib=libstdc++" + ;; +esac + cd $SRC_DIR/python/ -CPPFLAGS="-DASTRA_CUDA -DASTRA_PYTHON $CPPFLAGS -I$SRC_DIR/ -I$SRC_DIR/include -I$CUDA_ROOT/include" CC=$CC python ./builder.py build install +CPPFLAGS="-DASTRA_CUDA -DASTRA_PYTHON $CPPFLAGS -I$SRC_DIR/ -I$SRC_DIR/include" CC=$CC python ./builder.py build install diff --git a/python/conda/astra-toolbox/conda_build_config.yaml b/python/conda/astra-toolbox/conda_build_config.yaml new file mode 100644 index 0000000..bd38ac6 --- /dev/null +++ b/python/conda/astra-toolbox/conda_build_config.yaml @@ -0,0 +1,9 @@ +python: + - 2.7 + - 3.5 + - 3.6 + +numpy: + - 1.11 + - 1.12 + - 1.13 diff --git a/python/conda/astra-toolbox/meta.yaml b/python/conda/astra-toolbox/meta.yaml index 942397e..f3a39be 100644 --- a/python/conda/astra-toolbox/meta.yaml +++ b/python/conda/astra-toolbox/meta.yaml @@ -9,13 +9,11 @@ source: build: number: 0 script_env: - - CC # [not win] - - CUDA_ROOT # [not win] + - CC # [linux] test: imports: - astra - requires: # To avoid large downloads just for testing after build phase - nomkl # [not win] @@ -25,14 +23,14 @@ requirements: - python - cython >=0.13 - nomkl # [not win] - - numpy + - numpy {{ numpy }} - scipy - six - libastra ==1.8 run: - python - - numpy x.x + - {{ pin_compatible('numpy', max_pin='x.x') }} - scipy - six - libastra ==1.8 diff --git a/python/conda/libastra/build.sh b/python/conda/libastra/build.sh index 304c053..aa9a4ce 100644 --- a/python/conda/libastra/build.sh +++ b/python/conda/libastra/build.sh @@ -1,5 +1,19 @@ #!/bin/sh +case `uname` in + Darwin*) + CUDA_ROOT=/usr/local/cuda + CC=gcc + CXX=g++ + ;; + Linux*) + [ -n "$cudatoolkit" ] || exit 1 + CUDA_ROOT=/usr/local/cuda-$cudatoolkit + ;; +esac + +[ -x "$CUDA_ROOT" ] || exit 1 + cd $SRC_DIR/build/linux $SRC_DIR/build/linux/autogen.sh @@ -12,7 +26,11 @@ rm -f $CONDA_PREFIX/test.out $SRC_DIR/build/linux/configure --with-install-type=prefix --with-cuda=$CUDA_ROOT --prefix=$CONDA_PREFIX NVCCFLAGS="-ccbin $CC -I$CONDA_PREFIX/include $EXTRA_NVCCFLAGS" CC=$CC CXX=$CXX CPPFLAGS="-I$CONDA_PREFIX/include" -make install-libraries +# Clean, because we may be re-using this source tree when building +# multiple variants of this conda package. +make clean + +make -j $CPU_COUNT install-libraries test -d $CUDA_ROOT/lib64 && LIBPATH="$CUDA_ROOT/lib64" || LIBPATH="$CUDA_ROOT/lib" @@ -22,8 +40,13 @@ case `uname` in cp -P $LIBPATH/libcudart.*.dylib $CONDA_PREFIX/lib cp -P $LIBPATH/libcufft.*.dylib $CONDA_PREFIX/lib ;; - *) - cp -P $LIBPATH/libcudart.so.* $CONDA_PREFIX/lib - cp -P $LIBPATH/libcufft.so.* $CONDA_PREFIX/lib + Linux*) + if [ "$cudatoolkit" = "7.0" ]; then + # For some reason conda-build adds these symlinks automatically for + # cudatoolkit-5.5 and 6.0, but not 7.0. For 7.5 these symlinks are not + # necessary, and for 8.0 the cudatoolkit packages includes them. + ln -T -s libcudart.so.7.0.28 $CONDA_PREFIX/lib/libcudart.so.7.0 + ln -T -s libcufft.so.7.0.35 $CONDA_PREFIX/lib/libcufft.so.7.0 + fi ;; esac diff --git a/python/conda/libastra/linux_build_config.yaml b/python/conda/libastra/linux_build_config.yaml new file mode 100644 index 0000000..c5d2319 --- /dev/null +++ b/python/conda/libastra/linux_build_config.yaml @@ -0,0 +1,6 @@ +cudatoolkit: + - 5.5 + - 6.0 + - 7.0 + - 7.5 + - 8.0 diff --git a/python/conda/libastra/meta.yaml b/python/conda/libastra/meta.yaml index 68cf47a..6176f30 100644 --- a/python/conda/libastra/meta.yaml +++ b/python/conda/libastra/meta.yaml @@ -9,9 +9,8 @@ source: build: number: 0 script_env: - - CC # [not win] - - CXX # [not win] - - CUDA_ROOT # [not win] + - CC # [linux] + - CXX # [linux] requirements: build: @@ -20,15 +19,13 @@ requirements: - automake # [osx] - autoconf # [osx] - libtool # [osx] + - cudatoolkit {{ cudatoolkit }} # [linux] run: - vs2015_runtime # [win] + - cudatoolkit {{ cudatoolkit }} # [linux] about: home: http://www.astra-toolbox.com license: GPLv3 summary: 'The ASTRA Toolbox is a Python toolbox of high-performance GPU primitives for 2D and 3D tomography.' - -# See -# http://docs.continuum.io/conda/build.html for -# more information about meta.yaml diff --git a/python/conda/linux_release/buildenv/Dockerfile b/python/conda/linux_release/buildenv/Dockerfile index ce0b63b..17e9c5b 100644 --- a/python/conda/linux_release/buildenv/Dockerfile +++ b/python/conda/linux_release/buildenv/Dockerfile @@ -1,15 +1,55 @@ -FROM debian:7 -ENV PATH /root/miniconda3/bin:$PATH +FROM debian:7 AS BUILDBASE ENV DEBIAN_FRONTEND noninteractive -# http://developer.download.nvidia.com/compute/cuda/5_5/rel/installers/cuda_5.5.22_linux_64.run -ADD cuda_5.5.22_linux_64.run /root/ -# https://repo.continuum.io/miniconda/Miniconda3-4.3.27.1-Linux-x86_64.sh -ADD Miniconda3-4.3.27.1-Linux-x86_64.sh /root/ -RUN apt-get update -RUN apt-get install -y perl-modules build-essential autoconf libtool automake libboost-dev git -RUN /bin/bash /root/Miniconda3-4.3.27.1-Linux-x86_64.sh -b -RUN /bin/bash /root/cuda_5.5.22_linux_64.run -toolkit -silent +RUN apt-get update && apt-get install -y perl-modules build-essential autoconf libtool automake libboost-dev git && rm -rf /var/lib/apt/lists/* + +FROM BUILDBASE AS CUDA80 +RUN touch /root/cuda80 +COPY cuda_8.0.61_375.26_linux-run /root +RUN /bin/bash /root/cuda_8.0.61_375.26_linux-run --toolkit --silent && \ + rm -f /root/cuda_8.0.61_375.26_linux-run +COPY cuda_8.0.61.2_linux-run /root +RUN /bin/bash /root/cuda_8.0.61.2_linux-run --silent --accept-eula && \ + rm -f /root/cuda_8.0.61.2_linux-run + +FROM BUILDBASE AS CUDA75 +RUN touch /root/cuda75 +COPY cuda_7.5.18_linux.run /root +RUN /bin/bash /root/cuda_7.5.18_linux.run --toolkit --silent && \ + rm -f /root/cuda_7.5.18_linux.run + +FROM BUILDBASE AS CUDA70 +RUN touch /root/cuda70 +COPY cuda_7.0.28_linux.run /root +RUN /bin/bash /root/cuda_7.0.28_linux.run -toolkit -silent && \ + rm -f /root/cuda_7.0.28_linux.run + +COPY cufft_patch_linux.tar.gz /root +RUN cd /usr/local/cuda-7.0 && \ + tar xf /root/cufft_patch_linux.tar.gz && \ + rm -f /root/cufft_patch_linux.tar.gz + +FROM BUILDBASE AS CUDA60 +RUN touch /root/cuda60 +COPY cuda_6.0.37_linux_64.run /root +RUN /bin/bash /root/cuda_6.0.37_linux_64.run -toolkit -silent && \ + rm -f /root/cuda_6.0.37_linux_64.run + +FROM BUILDBASE AS CUDA55 +RUN touch /root/cuda55 +COPY cuda_5.5.22_linux_64.run /root +RUN /bin/bash /root/cuda_5.5.22_linux_64.run -toolkit -silent && \ + rm /root/cuda_5.5.22_linux_64.run + +FROM BUILDBASE +RUN touch /root/cuda +COPY --from=CUDA80 /usr/local/cuda-8.0 /usr/local/cuda-8.0 +COPY --from=CUDA75 /usr/local/cuda-7.5 /usr/local/cuda-7.5 +COPY --from=CUDA70 /usr/local/cuda-7.0 /usr/local/cuda-7.0 +COPY --from=CUDA60 /usr/local/cuda-6.0 /usr/local/cuda-6.0 +COPY --from=CUDA55 /usr/local/cuda-5.5 /usr/local/cuda-5.5 + +ENV PATH /root/miniconda3/bin:$PATH +COPY Miniconda3-4.3.27.1-Linux-x86_64.sh /root/ +RUN /bin/bash /root/Miniconda3-4.3.27.1-Linux-x86_64.sh -b && \ + rm -f /root/Miniconda3* RUN conda install -y conda-build -ENV CUDA_ROOT /usr/local/cuda -ENV CC gcc -ENV CXX g++ diff --git a/python/conda/linux_release/buildenv/build.sh b/python/conda/linux_release/buildenv/build.sh new file mode 100644 index 0000000..1cc0830 --- /dev/null +++ b/python/conda/linux_release/buildenv/build.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +export CC=gcc +export CXX=g++ + +cd /root +git clone --depth 1 --branch master https://github.com/astra-toolbox/astra-toolbox +[ $# -eq 0 ] || perl -pi -e "s/^(\s*number:\s*)[0-9]+$/\${1}$1/" astra-toolbox/python/conda/libastra/meta.yaml astra-toolbox/python/conda/astra-toolbox/meta.yaml + +conda-build -m astra-toolbox/python/conda/libastra/linux_build_config.yaml astra-toolbox/python/conda/libastra +conda-build astra-toolbox/python/conda/astra-toolbox + +cp /root/miniconda3/conda-bld/linux-64/*astra* /out diff --git a/python/conda/linux_release/builder/Dockerfile b/python/conda/linux_release/builder/Dockerfile deleted file mode 100644 index 2404609..0000000 --- a/python/conda/linux_release/builder/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM astra-build-env -ARG BUILD_NUMBER= -WORKDIR /root -RUN git clone --depth 1 https://github.com/astra-toolbox/astra-toolbox -RUN [ -z $BUILD_NUMBER ] || perl -pi -e "s/^(\s*number:\s*)[0-9]+$/\${1}$BUILD_NUMBER/" astra-toolbox/python/conda/libastra/meta.yaml astra-toolbox/python/conda/astra-toolbox/meta.yaml -RUN conda-build --python=3.5 astra-toolbox/python/conda/libastra -RUN conda-build --python 2.7 --numpy 1.8 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 2.7 --numpy 1.9 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 2.7 --numpy 1.10 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 2.7 --numpy 1.11 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 2.7 --numpy 1.12 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 3.5 --numpy 1.9 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 3.5 --numpy 1.10 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 3.5 --numpy 1.11 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 3.5 --numpy 1.12 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 3.6 --numpy 1.11 astra-toolbox/python/conda/astra-toolbox -RUN conda-build --python 3.6 --numpy 1.12 astra-toolbox/python/conda/astra-toolbox diff --git a/python/conda/linux_release/release.sh b/python/conda/linux_release/release.sh index 089cc7c..91c13e4 100644 --- a/python/conda/linux_release/release.sh +++ b/python/conda/linux_release/release.sh @@ -2,19 +2,19 @@ D=`mktemp -d` -[ -f buildenv/cuda_5.5.22_linux_64.run ] || (cd buildenv; wget http://developer.download.nvidia.com/compute/cuda/5_5/rel/installers/cuda_5.5.22_linux_64.run ) -[ -f buildenv/Miniconda3-4.3.27.1-Linux-x86_64.sh ] || (cd buildenv; wget https://repo.continuum.io/miniconda/Miniconda3-4.3.27.1-Linux-x86_64.sh ) +for F in https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh http://developer.download.nvidia.com/compute/cuda/5_5/rel/installers/cuda_5.5.22_linux_64.run http://developer.download.nvidia.com/compute/cuda/6_0/rel/installers/cuda_6.0.37_linux_64.run http://developer.download.nvidia.com/compute/cuda/7_0/Prod/local_installers/cuda_7.0.28_linux.run http://developer.download.nvidia.com/compute/cuda/7_0/Prod/cufft_update/cufft_patch_linux.tar.gz http://developer.download.nvidia.com/compute/cuda/7.5/Prod/local_installers/cuda_7.5.18_linux.run https://developer.nvidia.com/compute/cuda/8.0/Prod2/local_installers/cuda_8.0.61_375.26_linux-run https://developer.nvidia.com/compute/cuda/8.0/Prod2/patches/2/cuda_8.0.61.2_linux-run; do + [ -f buildenv/`basename $F` ] || (cd buildenv; wget $F ) +done docker build -t astra-build-env buildenv -#docker build --no-cache --build-arg=BUILD_NUMBER=0 -t astra-builder builder -docker build --no-cache -t astra-builder builder -docker run --name astra-build-cnt -v $D:/out:z astra-builder /bin/bash -c "cp /root/miniconda3/conda-bld/linux-64/*astra* /out" +cp buildenv/build.sh $D + +docker run -v $D:/out:z astra-build-env /bin/bash /out/build.sh + +rm -f $D/build.sh mkdir -p pkgs mv $D/* pkgs rmdir $D -docker rm astra-build-cnt -docker rmi astra-builder - -- cgit v1.2.3