diff --git a/.gitlab-ci.Dockerfile.conda b/.gitlab-ci.Dockerfile.conda
index 44dc6ee5f53cccbdabedb8171dd1beb2e2f2dc03..071959c5cff7a633d11977d032e471c8464ffb9f 100644
--- a/.gitlab-ci.Dockerfile.conda
+++ b/.gitlab-ci.Dockerfile.conda
@@ -8,12 +8,14 @@ ARG CONDA_ENV
# Build stage
FROM $CI_REGISTRY_IMAGE/dependencies/conda-$CONDA_ENV:$CI_COMMIT_REF_NAME AS build
+ARG CI_REGISTRY_IMAGE
+ARG CI_COMMIT_REF_NAME
ARG CONDA_ENV
# Labeling/packaging stuff:
LABEL name="GstLAL Runtime Package, conda" \
maintainer="Patrick Godwin <patrick.godwin@ligo.org>" \
- date="2021-03-18"
+ date="2021-10-14"
# Copy source to container
COPY gstlal /gstlal
@@ -22,11 +24,6 @@ COPY gstlal-calibration /gstlal-calibration
COPY gstlal-inspiral /gstlal-inspiral
COPY gstlal-burst /gstlal-burst
-# Export environment variables needed before compile:
-ENV MKLROOT $CONDA_PREFIX
-ENV MKL_INTERFACE_LAYER LP64
-ENV MKL_THREADING_LAYER SEQUENTIAL
-
# Make RUN commands use bash:
SHELL ["/bin/bash", "-c"]
@@ -98,19 +95,30 @@ RUN . /opt/conda/etc/profile.d/conda.sh && \
RUN rm -rf gstlal-inspiral
# Run stage
-ARG CI_REGISTRY_IMAGE
-ARG CI_COMMIT_REF_NAME
+FROM $CI_REGISTRY_IMAGE/dependencies/conda-$CONDA_ENV:$CI_COMMIT_REF_NAME AS run
+
ARG CONDA_ENV
-FROM $CI_REGISTRY_IMAGE/dependencies/conda-$CONDA_ENV:$CI_COMMIT_REF_NAME AS run
+# Replace created environment with the base environment
+# Also grab conda global config settings
+COPY --from=build /opt/conda /opt/conda
+COPY --from=build /opt/conda/.condarc /opt/conda
-# Copy build artifacts
-COPY --from=build $CONDA_PREFIX $CONDA_PREFIX
+# Set up entrypoint
+COPY .gitlab-ci.conda_entrypoint.sh /bin/entrypoint.sh
+RUN chmod +x /bin/entrypoint.sh
-# Export environment variables:
-ENV PKG_CONFIG_PATH $CONDA_PREFIX/envs/gstlal-$CONDA_ENV/lib/pkgconfig
-ENV GST_PLUGIN_PATH $CONDA_PREFIX/envs/gstlal-$CONDA_ENV/lib/gstreamer-1.0
+# Export environment variables
+ENV PKG_CONFIG_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/pkgconfig
+ENV GST_PLUGIN_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/gstreamer-1.0
ENV GSTLAL_FIR_WHITEN 0
ENV TMPDIR /tmp
-ENTRYPOINT bash
+# Give entrypoint knowledge about environment to source
+ENV CONDA_ENV $CONDA_ENV
+
+# Setup for interactive shell sessions (without polluting $HOME)
+RUN echo ". /opt/conda/etc/profile.d/conda.sh && conda activate gstlal-${CONDA_ENV}" >> /root/.bashrc
+
+ENTRYPOINT ["/bin/entrypoint.sh"]
+CMD ["/bin/bash"]
diff --git a/.gitlab-ci.Dockerfile.conda-deps b/.gitlab-ci.Dockerfile.conda-deps
new file mode 100644
index 0000000000000000000000000000000000000000..77b3dec4fc1d0b29be6988588fad4d42dfb7f7b6
--- /dev/null
+++ b/.gitlab-ci.Dockerfile.conda-deps
@@ -0,0 +1,55 @@
+# NOTE: this Dockerfile is adapted from https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile
+# removing tini as an entrypoint as it doesn't play nice with Singularity
+
+ARG CONDA_ENV
+
+FROM debian:bullseye-slim
+
+ARG CONDA_ENV
+ARG MINIFORGE_NAME=Mambaforge
+ARG MINIFORGE_VERSION=4.10.3-7
+
+ENV CONDA_DIR=/opt/conda
+ENV PATH=${CONDA_DIR}/bin:${PATH}
+
+# 1. Install just enough for conda to work
+# 2. Keep $HOME clean (no .wget-hsts file), since HSTS isn't useful in this context
+# 3. Install miniforge from GitHub releases
+# 4. Apply some cleanup tips from https://jcrist.github.io/conda-docker-tips.html
+# Particularly, we remove pyc and a files. The default install has no js, we can skip that
+# 5. Activate base by default when running as any *non-root* user as well
+# Good security practice requires running most workloads as non-root
+# This makes sure any non-root users created also have base activated
+# for their interactive shells.
+# 6. Activate base by default when running as root as well
+# The root user is already created, so won't pick up changes to /etc/skel
+RUN apt-get update > /dev/null && \
+ apt-get install --no-install-recommends --yes \
+ wget bzip2 ca-certificates \
+ git > /dev/null && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/* && \
+ wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/${MINIFORGE_NAME}-${MINIFORGE_VERSION}-Linux-$(uname -m).sh -O /tmp/miniforge.sh && \
+ /bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \
+ rm /tmp/miniforge.sh && \
+ conda clean -tipsy && \
+ find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \
+ find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \
+ conda clean -afy
+
+COPY gstlal/share/conda/envs/lock/gstlal-$CONDA_ENV-linux-64.lock .
+
+SHELL ["/bin/bash", "-c"]
+
+RUN conda config --set always_yes yes
+RUN conda config --add channels conda-forge
+RUN conda update -n base conda && \
+ conda clean -af
+RUN conda create -n gstlal-$CONDA_ENV --file gstlal-$CONDA_ENV-linux-64.lock --force && \
+ conda clean -af
+RUN rm -f gstlal-$CONDA_ENV-linux-64.lock
+
+ENV PKG_CONFIG_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/pkgconfig
+ENV GST_PLUGIN_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/gstreamer-1.0
+
+ENTRYPOINT ["bin/bash"]
diff --git a/.gitlab-ci.conda_entrypoint.sh b/.gitlab-ci.conda_entrypoint.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c9b806f184b4c85847f7b5f3715000746d074ab1
--- /dev/null
+++ b/.gitlab-ci.conda_entrypoint.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+. /opt/conda/etc/profile.d/conda.sh
+conda activate gstlal-${CONDA_ENV}
+
+exec "$@"
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e002f74e7aa0eb58ae982b6e401f610bf48294f7..b3df4252ab77e13e8f56bf67bdfdd575083cf6b2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -60,29 +60,13 @@ stages:
before_script: [ ]
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - |
- cat <<EOF > Dockerfile
- FROM igwn/base:conda
- COPY gstlal/share/conda/envs/lock/gstlal-$CONDA_ENV-linux-64.lock .
- SHELL ["/bin/bash", "-c"]
- RUN conda config --set always_yes yes
- RUN conda config --add channels conda-forge
- RUN conda update -n base conda && \
- conda clean -af
- RUN conda create -n gstlal-$CONDA_ENV --file gstlal-$CONDA_ENV-linux-64.lock --force && \
- conda clean -af
- RUN rm -f gstlal-$CONDA_ENV-linux-64.lock
- ENV PKG_CONFIG_PATH $CONDA_PREFIX/lib/pkgconfig
- ENV GST_PLUGIN_PATH $CONDA_PREFIX/lib/gstreamer-1.0
- ENTRYPOINT bash
- EOF
- - docker build -t $IMAGE_TAG .
+ - >
+ docker build --pull
+ --build-arg CONDA_ENV=$CONDA_ENV
+ -t $IMAGE_TAG
+ --file .gitlab-ci.Dockerfile.conda-deps
+ .
- docker push $IMAGE_TAG
- # NOTE: waiting on https://gitlab.com/gitlab-org/gitlab/-/issues/30680, otherwise
- # job dependencies don't work correctly for downstream stages
- #only:
- # changes:
- # - gstlal-inspiral/share/conda/dev-environment.yml
needs: [ ]
only:
- schedules
@@ -520,14 +504,6 @@ test:gstlal:conda:
- docker:conda:dev
before_script: [ ]
script:
- - source /opt/conda/etc/profile.d/conda.sh
- - conda activate gstlal-dev
- - echo $CONDA_PREFIX
- - ls $CONDA_PREFIX/lib/gstreamer-1.0
- - export GSTLAL_FIR_WHITEN=0
- - gst-inspect-1.0
-
- # Run doctests
- cd gstlal
- python3 -m pytest -c pytest.ini -m "not requires_gstlal_ugly"
only:
@@ -543,14 +519,6 @@ test:gstlal-only-ugly:conda:
- docker:conda:dev
before_script: [ ]
script:
- - source /opt/conda/etc/profile.d/conda.sh
- - conda activate gstlal-dev
- - echo $CONDA_PREFIX
- - ls $CONDA_PREFIX/lib/gstreamer-1.0
- - export GSTLAL_FIR_WHITEN=0
- - gst-inspect-1.0
-
- # Run doctests
- cd gstlal
- python3 -m pytest -c pytest.ini -m "requires_gstlal_ugly"
only:
@@ -566,12 +534,6 @@ test:gstlal-inspiral:conda:
- docker:conda:dev
before_script: [ ]
script:
- - source /opt/conda/etc/profile.d/conda.sh
- - conda activate gstlal-dev
- - export GSTLAL_FIR_WHITEN=0
- - gst-inspect-1.0
-
- # Run doctests
- cd gstlal-inspiral
- python3 -m pytest -c pytest.ini
only:
@@ -587,12 +549,6 @@ test:gstlal-ugly:conda:
- docker:conda:dev
before_script: [ ]
script:
- - source /opt/conda/etc/profile.d/conda.sh
- - conda activate gstlal-dev
- - export GSTLAL_FIR_WHITEN=0
- - gst-inspect-1.0
-
- # Run doctests
- cd gstlal-ugly
- python3 -m pytest -c pytest.ini
only:
@@ -608,10 +564,6 @@ test:gstlal-burst:conda:
- docker:conda:dev
before_script: [ ]
script:
- - source /opt/conda/etc/profile.d/conda.sh
- - conda activate gstlal-dev
- - export GSTLAL_FIR_WHITEN=0
- - gst-inspect-1.0
- cd gstlal-burst
- python3 -m pytest -c pytest.ini
only:
@@ -627,10 +579,6 @@ test:gstlal-calibration:conda:
- docker:conda:dev
before_script: [ ]
script:
- - source /opt/conda/etc/profile.d/conda.sh
- - conda activate gstlal-dev
- - export GSTLAL_FIR_WHITEN=0
- - gst-inspect-1.0
- cd gstlal-calibration
- python3 -m pytest -c pytest.ini
only:
@@ -651,10 +599,6 @@ test:offline:conda:
script:
# Set up directory structure and copy over built-dependencies from container:
- mkdir public
- # Install RPMs and set up the test environment:
- - source /opt/conda/etc/profile.d/conda.sh
- - conda activate gstlal-dev
- - gst-inspect-1.0
# Export variables for the offline tutorial
- export LAL_PATH=$CONDA_PREFIX
@@ -666,7 +610,7 @@ test:offline:conda:
- cd gstlal-inspiral/tests
- # Run the makefile:
+ # Run tests
- make -f Makefile.offline_tutorial_test ENABLE_PLOTTING=0
only: