build conda container without needing to run `conda activate <env>`

* add entrypoint to conda dockerfile for interactive sessions

.gitlab-ci.Dockerfile.conda

@@ -8,12 +8,14 @@ ARG CONDA_ENV
 # Build stage
 FROM $CI_REGISTRY_IMAGE/dependencies/conda-$CONDA_ENV:$CI_COMMIT_REF_NAME AS build
 
+ARG CI_REGISTRY_IMAGE
+ARG CI_COMMIT_REF_NAME
 ARG CONDA_ENV
 
 # Labeling/packaging stuff:
 LABEL name="GstLAL Runtime Package, conda" \
       maintainer="Patrick Godwin <patrick.godwin@ligo.org>" \
-      date="2021-03-18"
+      date="2021-10-14"
 
 # Copy source to container
 COPY gstlal /gstlal
@@ -22,11 +24,6 @@ COPY gstlal-calibration /gstlal-calibration
 COPY gstlal-inspiral /gstlal-inspiral
 COPY gstlal-burst /gstlal-burst
 
-# Export environment variables needed before compile:
-ENV MKLROOT $CONDA_PREFIX
-ENV MKL_INTERFACE_LAYER LP64
-ENV MKL_THREADING_LAYER SEQUENTIAL
-
 # Make RUN commands use bash:
 SHELL ["/bin/bash", "-c"]
 
@@ -98,19 +95,30 @@ RUN . /opt/conda/etc/profile.d/conda.sh && \
 RUN rm -rf gstlal-inspiral
 
 # Run stage
-ARG CI_REGISTRY_IMAGE
-ARG CI_COMMIT_REF_NAME
+FROM $CI_REGISTRY_IMAGE/dependencies/conda-$CONDA_ENV:$CI_COMMIT_REF_NAME AS run
+
 ARG CONDA_ENV
 
-FROM $CI_REGISTRY_IMAGE/dependencies/conda-$CONDA_ENV:$CI_COMMIT_REF_NAME AS run
+# Replace created environment with the base environment
+# Also grab conda global config settings
+COPY --from=build /opt/conda /opt/conda
+COPY --from=build /opt/conda/.condarc /opt/conda
 
-# Copy build artifacts
-COPY --from=build $CONDA_PREFIX $CONDA_PREFIX
+# Set up entrypoint
+COPY .gitlab-ci.conda_entrypoint.sh /bin/entrypoint.sh
+RUN chmod +x /bin/entrypoint.sh
 
-# Export environment variables: 
-ENV PKG_CONFIG_PATH $CONDA_PREFIX/envs/gstlal-$CONDA_ENV/lib/pkgconfig
-ENV GST_PLUGIN_PATH $CONDA_PREFIX/envs/gstlal-$CONDA_ENV/lib/gstreamer-1.0
+# Export environment variables
+ENV PKG_CONFIG_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/pkgconfig
+ENV GST_PLUGIN_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/gstreamer-1.0
 ENV GSTLAL_FIR_WHITEN 0
 ENV TMPDIR /tmp
 
-ENTRYPOINT bash
+# Give entrypoint knowledge about environment to source
+ENV CONDA_ENV $CONDA_ENV
+
+# Setup for interactive shell sessions (without polluting $HOME)
+RUN echo ". /opt/conda/etc/profile.d/conda.sh && conda activate gstlal-${CONDA_ENV}" >> /root/.bashrc
+
+ENTRYPOINT ["/bin/entrypoint.sh"]
+CMD ["/bin/bash"]

.gitlab-ci.Dockerfile.conda-deps

+# NOTE: this Dockerfile is adapted from https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile
+# removing tini as an entrypoint as it doesn't play nice with Singularity
+
+ARG CONDA_ENV
+
+FROM debian:bullseye-slim
+
+ARG CONDA_ENV
+ARG MINIFORGE_NAME=Mambaforge
+ARG MINIFORGE_VERSION=4.10.3-7
+
+ENV CONDA_DIR=/opt/conda
+ENV PATH=${CONDA_DIR}/bin:${PATH}
+
+# 1. Install just enough for conda to work
+# 2. Keep $HOME clean (no .wget-hsts file), since HSTS isn't useful in this context
+# 3. Install miniforge from GitHub releases
+# 4. Apply some cleanup tips from https://jcrist.github.io/conda-docker-tips.html
+#    Particularly, we remove pyc and a files. The default install has no js, we can skip that
+# 5. Activate base by default when running as any *non-root* user as well
+#    Good security practice requires running most workloads as non-root
+#    This makes sure any non-root users created also have base activated
+#    for their interactive shells.
+# 6. Activate base by default when running as root as well
+#    The root user is already created, so won't pick up changes to /etc/skel
+RUN apt-get update > /dev/null && \
+    apt-get install --no-install-recommends --yes \
+        wget bzip2 ca-certificates \
+        git > /dev/null && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/${MINIFORGE_NAME}-${MINIFORGE_VERSION}-Linux-$(uname -m).sh -O /tmp/miniforge.sh && \
+    /bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \
+    rm /tmp/miniforge.sh && \
+    conda clean -tipsy && \
+    find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \
+    find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \
+    conda clean -afy
+
+COPY gstlal/share/conda/envs/lock/gstlal-$CONDA_ENV-linux-64.lock .
+
+SHELL ["/bin/bash", "-c"]
+
+RUN conda config --set always_yes yes
+RUN conda config --add channels conda-forge
+RUN conda update -n base conda && \
+    conda clean -af
+RUN conda create -n gstlal-$CONDA_ENV --file gstlal-$CONDA_ENV-linux-64.lock --force && \
+    conda clean -af
+RUN rm -f gstlal-$CONDA_ENV-linux-64.lock
+
+ENV PKG_CONFIG_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/pkgconfig
+ENV GST_PLUGIN_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/gstreamer-1.0
+
+ENTRYPOINT ["bin/bash"]

.gitlab-ci.conda_entrypoint.sh

+#!/usr/bin/env bash
+
+. /opt/conda/etc/profile.d/conda.sh
+conda activate gstlal-${CONDA_ENV}
+
+exec "$@"

.gitlab-ci.yml

@@ -60,29 +60,13 @@ stages:
   before_script: [ ]
   script:
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - |
-      cat <<EOF > Dockerfile
-      FROM igwn/base:conda
-      COPY gstlal/share/conda/envs/lock/gstlal-$CONDA_ENV-linux-64.lock .
-      SHELL ["/bin/bash", "-c"]
-      RUN conda config --set always_yes yes
-      RUN conda config --add channels conda-forge
-      RUN conda update -n base conda && \
-          conda clean -af
-      RUN conda create -n gstlal-$CONDA_ENV --file gstlal-$CONDA_ENV-linux-64.lock --force && \
-          conda clean -af
-      RUN rm -f gstlal-$CONDA_ENV-linux-64.lock
-      ENV PKG_CONFIG_PATH $CONDA_PREFIX/lib/pkgconfig
-      ENV GST_PLUGIN_PATH $CONDA_PREFIX/lib/gstreamer-1.0
-      ENTRYPOINT bash
-      EOF
-    - docker build -t $IMAGE_TAG .
+    - >
+      docker build --pull
+      --build-arg CONDA_ENV=$CONDA_ENV
+      -t $IMAGE_TAG
+      --file .gitlab-ci.Dockerfile.conda-deps
+      .
     - docker push $IMAGE_TAG
-  # NOTE: waiting on https://gitlab.com/gitlab-org/gitlab/-/issues/30680, otherwise
-  # job dependencies don't work correctly for downstream stages
-  #only:
-  #  changes:
-  #    - gstlal-inspiral/share/conda/dev-environment.yml
   needs: [ ]
   only:
     - schedules
@@ -520,14 +504,6 @@ test:gstlal:conda:
     - docker:conda:dev
   before_script: [ ]
   script:
-    - source /opt/conda/etc/profile.d/conda.sh
-    - conda activate gstlal-dev
-    - echo $CONDA_PREFIX
-    - ls $CONDA_PREFIX/lib/gstreamer-1.0
-    - export GSTLAL_FIR_WHITEN=0
-    - gst-inspect-1.0
-
-    # Run doctests
     - cd gstlal
     - python3 -m pytest -c pytest.ini -m "not requires_gstlal_ugly"
   only:
@@ -543,14 +519,6 @@ test:gstlal-only-ugly:conda:
     - docker:conda:dev
   before_script: [ ]
   script:
-    - source /opt/conda/etc/profile.d/conda.sh
-    - conda activate gstlal-dev
-    - echo $CONDA_PREFIX
-    - ls $CONDA_PREFIX/lib/gstreamer-1.0
-    - export GSTLAL_FIR_WHITEN=0
-    - gst-inspect-1.0
-
-    # Run doctests
     - cd gstlal
     - python3 -m pytest -c pytest.ini -m "requires_gstlal_ugly"
   only:
@@ -566,12 +534,6 @@ test:gstlal-inspiral:conda:
     - docker:conda:dev
   before_script: [ ]
   script:
-    - source /opt/conda/etc/profile.d/conda.sh
-    - conda activate gstlal-dev
-    - export GSTLAL_FIR_WHITEN=0
-    - gst-inspect-1.0
-
-    # Run doctests
     - cd gstlal-inspiral
     - python3 -m pytest -c pytest.ini
   only:
@@ -587,12 +549,6 @@ test:gstlal-ugly:conda:
     - docker:conda:dev
   before_script: [ ]
   script:
-    - source /opt/conda/etc/profile.d/conda.sh
-    - conda activate gstlal-dev
-    - export GSTLAL_FIR_WHITEN=0
-    - gst-inspect-1.0
-
-    # Run doctests
     - cd gstlal-ugly
     - python3 -m pytest -c pytest.ini
   only:
@@ -608,10 +564,6 @@ test:gstlal-burst:conda:
     - docker:conda:dev
   before_script: [ ]
   script:
-    - source /opt/conda/etc/profile.d/conda.sh
-    - conda activate gstlal-dev
-    - export GSTLAL_FIR_WHITEN=0
-    - gst-inspect-1.0
     - cd gstlal-burst
     - python3 -m pytest -c pytest.ini
   only:
@@ -627,10 +579,6 @@ test:gstlal-calibration:conda:
     - docker:conda:dev
   before_script: [ ]
   script:
-    - source /opt/conda/etc/profile.d/conda.sh
-    - conda activate gstlal-dev
-    - export GSTLAL_FIR_WHITEN=0
-    - gst-inspect-1.0
     - cd gstlal-calibration
     - python3 -m pytest -c pytest.ini
   only:
@@ -651,10 +599,6 @@ test:offline:conda:
   script:
     # Set up directory structure and copy over built-dependencies from container:
     - mkdir public
-    # Install RPMs and set up the test environment:
-    - source /opt/conda/etc/profile.d/conda.sh
-    - conda activate gstlal-dev
-    - gst-inspect-1.0
 
     # Export variables for the offline tutorial
     - export LAL_PATH=$CONDA_PREFIX
@@ -666,7 +610,7 @@ test:offline:conda:
 
     - cd gstlal-inspiral/tests
 
-    # Run the makefile:
+    # Run tests
     - make -f Makefile.offline_tutorial_test ENABLE_PLOTTING=0
 
   only: