From 90dabd833f34f6c629df0146f38a0cf2bb9a38f5 Mon Sep 17 00:00:00 2001
From: James Kennington <james.kennington@ligo.org>
Date: Thu, 16 Feb 2023 03:07:02 +0000
Subject: [PATCH] Remove conda stages of CI build
---
.gitlab-ci.Dockerfile.conda | 114 ----------------
.gitlab-ci.Dockerfile.conda-deps | 55 --------
.gitlab-ci.conda_entrypoint.sh | 6 -
.gitlab-ci.yml | 215 +------------------------------
doc/sphinx-bindoc | 4 +-
5 files changed, 7 insertions(+), 387 deletions(-)
delete mode 100644 .gitlab-ci.Dockerfile.conda
delete mode 100644 .gitlab-ci.Dockerfile.conda-deps
delete mode 100755 .gitlab-ci.conda_entrypoint.sh
diff --git a/.gitlab-ci.Dockerfile.conda b/.gitlab-ci.Dockerfile.conda
deleted file mode 100644
index 2ff525a626..0000000000
--- a/.gitlab-ci.Dockerfile.conda
+++ /dev/null
@@ -1,114 +0,0 @@
-# This is a Dockerfile to build a conda GstLAL
-# runtime container.
-
-ARG CI_REGISTRY_IMAGE
-ARG CI_COMMIT_REF_NAME
-ARG CONDA_ENV
-
-# Build stage
-FROM $CI_REGISTRY_IMAGE/dependencies/conda-$CONDA_ENV:$CI_COMMIT_REF_NAME AS build
-
-ARG CI_REGISTRY_IMAGE
-ARG CI_COMMIT_REF_NAME
-ARG CONDA_ENV
-
-# Labeling/packaging stuff:
-LABEL name="GstLAL Runtime Package, conda" \
- maintainer="Patrick Godwin <patrick.godwin@ligo.org>" \
- date="2021-10-14"
-
-# Copy source to container
-COPY gstlal /gstlal
-COPY gstlal-ugly /gstlal-ugly
-COPY gstlal-inspiral /gstlal-inspiral
-COPY gstlal-burst /gstlal-burst
-
-# Make RUN commands use bash:
-SHELL ["/bin/bash", "-c"]
-
-# Install gstlal
-RUN . /opt/conda/etc/profile.d/conda.sh && \
- conda activate gstlal-$CONDA_ENV && \
- export PREFIX="$CONDA_PREFIX" && \
- export CONDA_BUILD="1" && \
- conda activate gstlal-$CONDA_ENV && \
- cd /gstlal && \
- ./00init.sh && \
- ./configure --prefix=$CONDA_PREFIX --without-doxygen && \
- make -j$NUM_CORES && \
- make install -j$NUM_CORES
-RUN rm -rf gstlal
-
-# Install gstlal-ugly
-RUN . /opt/conda/etc/profile.d/conda.sh && \
- conda activate gstlal-$CONDA_ENV && \
- export PREFIX="$CONDA_PREFIX" && \
- export CONDA_BUILD="1" && \
- conda activate gstlal-$CONDA_ENV && \
- cd /gstlal-ugly && \
- ./00init.sh && \
- ./configure --prefix=$CONDA_PREFIX --without-doxygen && \
- make -j$NUM_CORES && \
- make install -j$NUM_CORES
-RUN rm -rf gstlal-ugly
-
-# Install gstlal-burst
-RUN . /opt/conda/etc/profile.d/conda.sh && \
- conda activate gstlal-$CONDA_ENV && \
- export PREFIX="$CONDA_PREFIX" && \
- export CONDA_BUILD="1" && \
- conda activate gstlal-$CONDA_ENV && \
- cd /gstlal-burst && \
- ./00init.sh && \
- ./configure --prefix=$CONDA_PREFIX --without-doxygen && \
- make -j$NUM_CORES && \
- make install -j$NUM_CORES
-RUN rm -rf gstlal-burst
-
-# Install gstlal-inspiral
-RUN . /opt/conda/etc/profile.d/conda.sh && \
- conda activate gstlal-$CONDA_ENV && \
- export PREFIX="$CONDA_PREFIX" && \
- export CONDA_BUILD="1" && \
- conda activate gstlal-$CONDA_ENV && \
- cd /gstlal-inspiral && \
- ./00init.sh && \
- ./configure --prefix=$CONDA_PREFIX --without-doxygen --disable-massmodel && \
- make -j$NUM_CORES && \
- make install -j$NUM_CORES
-RUN rm -rf gstlal-inspiral
-
-# Fix shebangs on executables
-RUN . /opt/conda/etc/profile.d/conda.sh && \
- conda activate gstlal-$CONDA_ENV && \
- sed -i.tmp "s@/usr/bin/env python3@$CONDA_PREFIX/bin/python@g" ${CONDA_PREFIX}/bin/gstlal_*
-
-# Run stage
-FROM $CI_REGISTRY_IMAGE/dependencies/conda-$CONDA_ENV:$CI_COMMIT_REF_NAME AS run
-
-ARG CONDA_ENV
-
-# Replace created environment with the base environment
-# Also grab conda global config settings
-COPY --from=build /opt/conda /opt/conda
-COPY --from=build /opt/conda/.condarc /opt/conda
-
-# Set up entrypoint
-COPY .gitlab-ci.conda_entrypoint.sh /bin/entrypoint.sh
-RUN chmod +x /bin/entrypoint.sh
-
-# Export environment variables
-ENV PATH="/opt/conda/envs/gstlal-$CONDA_ENV/bin:${PATH}"
-ENV PKG_CONFIG_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/pkgconfig
-ENV GST_PLUGIN_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/gstreamer-1.0
-ENV GSTLAL_FIR_WHITEN 0
-ENV TMPDIR /tmp
-
-# Give entrypoint knowledge about environment to source
-ENV CONDA_ENV $CONDA_ENV
-
-# Setup for interactive shell sessions (without polluting $HOME)
-RUN echo ". /opt/conda/etc/profile.d/conda.sh && conda activate gstlal-${CONDA_ENV}" >> /root/.bashrc
-
-ENTRYPOINT ["/bin/entrypoint.sh"]
-CMD ["/bin/bash"]
diff --git a/.gitlab-ci.Dockerfile.conda-deps b/.gitlab-ci.Dockerfile.conda-deps
deleted file mode 100644
index 77b3dec4fc..0000000000
--- a/.gitlab-ci.Dockerfile.conda-deps
+++ /dev/null
@@ -1,55 +0,0 @@
-# NOTE: this Dockerfile is adapted from https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile
-# removing tini as an entrypoint as it doesn't play nice with Singularity
-
-ARG CONDA_ENV
-
-FROM debian:bullseye-slim
-
-ARG CONDA_ENV
-ARG MINIFORGE_NAME=Mambaforge
-ARG MINIFORGE_VERSION=4.10.3-7
-
-ENV CONDA_DIR=/opt/conda
-ENV PATH=${CONDA_DIR}/bin:${PATH}
-
-# 1. Install just enough for conda to work
-# 2. Keep $HOME clean (no .wget-hsts file), since HSTS isn't useful in this context
-# 3. Install miniforge from GitHub releases
-# 4. Apply some cleanup tips from https://jcrist.github.io/conda-docker-tips.html
-# Particularly, we remove pyc and a files. The default install has no js, we can skip that
-# 5. Activate base by default when running as any *non-root* user as well
-# Good security practice requires running most workloads as non-root
-# This makes sure any non-root users created also have base activated
-# for their interactive shells.
-# 6. Activate base by default when running as root as well
-# The root user is already created, so won't pick up changes to /etc/skel
-RUN apt-get update > /dev/null && \
- apt-get install --no-install-recommends --yes \
- wget bzip2 ca-certificates \
- git > /dev/null && \
- apt-get clean && \
- rm -rf /var/lib/apt/lists/* && \
- wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/${MINIFORGE_NAME}-${MINIFORGE_VERSION}-Linux-$(uname -m).sh -O /tmp/miniforge.sh && \
- /bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \
- rm /tmp/miniforge.sh && \
- conda clean -tipsy && \
- find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \
- find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \
- conda clean -afy
-
-COPY gstlal/share/conda/envs/lock/gstlal-$CONDA_ENV-linux-64.lock .
-
-SHELL ["/bin/bash", "-c"]
-
-RUN conda config --set always_yes yes
-RUN conda config --add channels conda-forge
-RUN conda update -n base conda && \
- conda clean -af
-RUN conda create -n gstlal-$CONDA_ENV --file gstlal-$CONDA_ENV-linux-64.lock --force && \
- conda clean -af
-RUN rm -f gstlal-$CONDA_ENV-linux-64.lock
-
-ENV PKG_CONFIG_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/pkgconfig
-ENV GST_PLUGIN_PATH /opt/conda/envs/gstlal-$CONDA_ENV/lib/gstreamer-1.0
-
-ENTRYPOINT ["bin/bash"]
diff --git a/.gitlab-ci.conda_entrypoint.sh b/.gitlab-ci.conda_entrypoint.sh
deleted file mode 100755
index c9b806f184..0000000000
--- a/.gitlab-ci.conda_entrypoint.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-. /opt/conda/etc/profile.d/conda.sh
-conda activate gstlal-${CONDA_ENV}
-
-exec "$@"
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 88f3cc924a..c16dfe3920 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -39,38 +39,6 @@ stages:
- docs
- nightly-pages
-# conda containers
-.dependencies: &dependencies
- stage: deps
- variables: &conda-deps-vars
- GIT_STRATEGY: fetch
- IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_JOB_NAME:$CI_COMMIT_REF_NAME
- before_script: [ ]
- script:
- - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - >
- docker build --pull
- --build-arg CONDA_ENV=$CONDA_ENV
- -t $IMAGE_TAG
- --file .gitlab-ci.Dockerfile.conda-deps
- .
- - docker push $IMAGE_TAG
- needs: [ ]
- only:
- - schedules
- - pushes
-
-dependencies/conda-dev:
- <<: *dependencies
- variables:
- <<: *conda-deps-vars
- CONDA_ENV: dev
-
-dependencies/conda-prod:
- <<: *dependencies
- variables:
- <<: *conda-deps-vars
- CONDA_ENV: prod
#
# build rpms
@@ -89,7 +57,7 @@ dependencies/conda-prod:
- ./configure --enable-gtk-doc $EXTRA_CONFIG_FLAGS
- make
- make dist
- # Install dependencies
+ # Install dependencies
- yum-builddep -y ${CI_JOB_NAME#level?:rpm:}.spec
- rpmbuild -tb -D "_topdir ${CI_PROJECT_DIR}/rpmbuild" -D 'debug_package %{nil}' -D 'build_cflags ${CFLAGS}' -D 'build_ldflags ${LDFLAGS}' ${CI_JOB_NAME#level?:rpm:}-*.tar.gz
artifacts:
@@ -196,46 +164,6 @@ docker:release:gstlal-burst:rl8:
only:
- /gstlal-burst-[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+/
-.docker:conda: &docker-conda
- interruptible: true
- stage: docker
- variables: &docker-conda-vars
- GIT_STRATEGY: fetch
- NUM_CORES: 4
- before_script: [ ]
- script:
- - IMAGE_TAG=$CI_REGISTRY_IMAGE/conda-$CONDA_ENV:$CI_COMMIT_REF_NAME
- - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - >
- docker build --pull
- --build-arg CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE
- --build-arg CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME
- --build-arg CONDA_ENV=$CONDA_ENV
- -t $IMAGE_TAG
- --file .gitlab-ci.Dockerfile.conda
- .
- - docker push $IMAGE_TAG
- needs:
- - dependencies:conda:prod
- only:
- - schedules
- - pushes
-
-docker:conda:dev:
- <<: *docker-conda
- variables:
- <<: *docker-conda-vars
- CONDA_ENV: dev
- needs:
- - dependencies/conda-dev
-
-docker:conda:prod:
- <<: *docker-conda
- variables:
- <<: *docker-conda-vars
- CONDA_ENV: prod
- needs:
- - dependencies/conda-prod
latest_image:
interruptible: true
@@ -404,155 +332,22 @@ test:offline:rl8:
allow_failure: true
-# test stages (conda)
-
-test:gstlal:conda:
- interruptible: true
- image: $CI_REGISTRY_IMAGE/conda-dev:$CI_COMMIT_REF_NAME
- stage: test-gstlal
- needs:
- - docker:conda:dev
- before_script: [ ]
- script:
- - cd gstlal
- - python3 -m pytest -c pytest.ini -m "not requires_full_build" --junitxml=report.xml
- only:
- - schedules
- - pushes
- allow_failure: true
- artifacts:
- when: always
- reports:
- junit: gstlal/report.xml
-
-test:gstlal-full-build:conda:
- interruptible: true
- image: $CI_REGISTRY_IMAGE/conda-dev:$CI_COMMIT_REF_NAME
- stage: test-gstlal-full-build
- needs:
- - docker:conda:dev
- before_script: [ ]
- script:
- - cd gstlal
- - python3 -m pytest -c pytest.ini -m "requires_full_build" --junitxml=report-full-build.xml
- only:
- - schedules
- - pushes
- allow_failure: true
- artifacts:
- when: always
- reports:
- junit: gstlal/report-full-build.xml
-
-test:gstlal-inspiral:conda:
- interruptible: true
- image: $CI_REGISTRY_IMAGE/conda-dev:$CI_COMMIT_REF_NAME
- stage: test-inspiral
- needs:
- - docker:conda:dev
- before_script: [ ]
- script:
- - cd gstlal-inspiral
- - python3 -m pytest -c pytest.ini --junitxml=report-inspiral.xml
- only:
- - schedules
- - pushes
- allow_failure: true
- artifacts:
- when: always
- reports:
- junit: gstlal/report-inspiral.xml
-
-test:gstlal-ugly:conda:
- interruptible: true
- image: $CI_REGISTRY_IMAGE/conda-dev:$CI_COMMIT_REF_NAME
- stage: test-gstlal-ugly
- needs:
- - docker:conda:dev
- before_script: [ ]
- script:
- - cd gstlal-ugly
- - python3 -m pytest -c pytest.ini --junitxml=report-ugly.xml
- only:
- - schedules
- - pushes
- allow_failure: true
- artifacts:
- when: always
- reports:
- junit: gstlal/report-ugly.xml
-
-test:gstlal-burst:conda:
- interruptible: true
- image: $CI_REGISTRY_IMAGE/conda-dev:$CI_COMMIT_REF_NAME
- stage: test-burst
- needs:
- - docker:conda:dev
- before_script: [ ]
- script:
- - cd gstlal-burst
- - python3 -m pytest -c pytest.ini --junitxml=report-burst.xml
- only:
- - schedules
- - pushes
- allow_failure: true
- artifacts:
- when: always
- reports:
- junit: gstlal/report-burst.xml
-
-test:offline:conda:
- interruptible: true
- image: $CI_REGISTRY_IMAGE/conda-dev:$CI_COMMIT_REF_NAME
- stage: test-offline
- needs:
- - docker:conda:dev
- - test:gstlal:conda
- - test:gstlal-inspiral:conda
- - test:gstlal-burst:conda
- before_script: [ ]
- script:
- # Set up directory structure and copy over built-dependencies from container:
- - mkdir public
-
- # Export variables for the offline tutorial
- - export LAL_PATH=$CONDA_PREFIX
- - export USER=gstlal_CI_test
-
- # Get the necessary ROM data:
- - git clone https://git.ligo.org/alexander.pace/gstlal-testing-data.git ${GSTLAL_DIR}/gstlal-testing-data
- - export LAL_DATA_PATH=${GSTLAL_DIR}/gstlal-testing-data/
-
- - cd gstlal-inspiral/tests
-
- # Run tests
- - make -f Makefile.offline_tutorial_test ENABLE_PLOTTING=0
-
- only:
- - schedules
- - pushes
- allow_failure: true
-
# Documentation
docs:
interruptible: true
- image: $CI_REGISTRY_IMAGE/conda-dev:$CI_COMMIT_REF_NAME
+ image: $DOCKER_BRANCH
stage: docs
before_script: [ ]
script:
- |
- export DEBIAN_FRONTEND=noninteractive
- export TZ=America/New_York
- apt-get --allow-releaseinfo-change update -y
- apt-get install -y dvipng texlive-latex-base texlive-latex-extra
- source /opt/conda/etc/profile.d/conda.sh
- conda activate gstlal-dev
+ yum install -y dvipng texlive-pdftex
+ python3 -m pip install sphinx myst-parser sphinx-rtd-theme
mkdir -p docs/
cd doc; make html IS_CI=1
cd ..; cp -rf doc/_build/* docs/
needs:
- - docker:conda:dev
+ - docker:rl8
artifacts:
paths:
- docs
diff --git a/doc/sphinx-bindoc b/doc/sphinx-bindoc
index 227aa3bbd2..29d73ddcb9 100755
--- a/doc/sphinx-bindoc
+++ b/doc/sphinx-bindoc
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright 2018 Chad Hanna
#
import sys
@@ -11,7 +11,7 @@ def process_source(prog, outfile):
if not line.startswith("###"):
continue
outfile.write(line.replace("### ", "").replace("###",""))
-
+
if len(sys.argv) == 1:
print("USAGE: sphinx-bindoc <output directory> <input directory> [patterns to exclude]")
--
GitLab