Use conda-flow for conda envs in gstlal

.gitlab-ci.yml

@@ -60,15 +60,15 @@ stages:
     - |
       cat <<EOF > Dockerfile
       FROM igwn/base:conda
-      COPY gstlal/share/conda/envs/qualified-linux-64.lock .
+      COPY gstlal/share/conda/envs/lock/gstlal-dev-linux-64.lock .
       SHELL ["/bin/bash", "-c"]
       RUN conda config --set always_yes yes
       RUN conda config --add channels conda-forge
       RUN conda update -n base conda && \
           conda clean -af
-      RUN conda create -n build-env --file qualified-linux-64.lock --force && \
+      RUN conda create -n build-env --file gstlal-dev-linux-64.lock --force && \
           conda clean -af
-      RUN rm -f qualified-linux-64.lock
+      RUN rm -f gstlal-dev-linux-64.lock
       ENV PKG_CONFIG_PATH $CONDA_PREFIX/lib/pkgconfig
       ENV GST_PLUGIN_PATH $CONDA_PREFIX/lib/gstreamer-1.0
       ENTRYPOINT bash

gstlal/share/conda/conda-flow.yml

+paths:
+  env_dir: './envs'
+  base_subpath: 'base'
+  lock_subpath: 'lock'
+  create_if_not_exists: True
+  template: '{name}-{platform}.{kind}'

gstlal/share/conda/envs/README.md

@@ -2,14 +2,7 @@
 
 Since solving a dependency problem can be NP-hard, and could therefore take considerable time to complete, we store our
 production conda environments as fully-specified (or "qualified")
-lock files, which include version numbers and all packages. These will be called "qualified-*.lock"
+lock files, which include version numbers and all packages. These will be called "{name}-{os}.lock"
 with other details in the name to identify different purposes.
 
-## Using the *qualified* environments
-
-For example, on MacOS, look for gstlal/share/conda/envs/qualified-osx-64.lock
-
-## Modifying the conda environment
-
-To modify the conda environment, do not edit the qualified env directly. Rather, make whatever changes are required to
-the corresponding *base* env yaml file, and then run the qualify script.
+This workflow now leverages conda-flow, see: https://git.ligo.org/james.kennington/conda-flow

gstlal/share/conda/qualify_env.sh

-# Script for setting up a development environment on MacOS for GstLAL
-
-# Parse arguments
-while getopts n:b: flag; do
-  case "${flag}" in
-  n) SUB_ENV=${OPTARG} ;;
-  b) BASE_ENV=${OPTARG} ;;
-  esac
-done
-if [[ -z "${BASE_ENV}" ]]; then
-  BASE_ENV="base"
-fi
-echo "Sub env: $SUB_ENV"
-echo "Base env: $BASE_ENV"
-
-# Discover repository paths
-echo "Discovering repository root paths"
-SCRIPT_PATH="$(
-  cd -- "$(dirname "$0")" >/dev/null 2>&1
-  pwd -P
-)"
-CONDA_PATH=$(
-  builtin cd $SCRIPT_PATH
-  pwd
-)
-SHARE_PATH=$(
-  builtin cd $CONDA_PATH
-  cd ..
-  pwd
-)
-GSTLAL_PATH=$(
-  builtin cd $SHARE_PATH
-  cd ..
-  pwd
-)
-REPO_PATH=$(
-  builtin cd $GSTLAL_PATH
-  cd ..
-  pwd
-)
-PARENT_PATH=$(
-  builtin cd $REPO_PATH
-  cd ..
-  pwd
-)
-echo "Repository root discovered: $REPO_PATH"
-
-# Determine Operating System
-echo "Detecting OS"
-if [[ "$OSTYPE" == "linux-gnu"* ]]; then
-  SYS_OS="linux"
-  ENV_BASE_ROOT=$CONDA_PATH/envs/base-linux-64
-elif [[ "$OSTYPE" == "darwin"* ]]; then
-  SYS_OS="macos"
-  ENV_BASE_ROOT=$CONDA_PATH/envs/base-osx-64
-elif [[ "$OSTYPE" == "cygwin" ]]; then
-  SYS_OS="cygwin"
-  echo "Cygwin currently unsupported"
-  exit
-elif [[ "$OSTYPE" == "msys" ]]; then
-  SYS_OS="win"
-  echo "Windows currently unsupported"
-  exit
-else
-  echo "Unsupported OS: $OSTYPE"
-  exit
-fi
-ENV_LOCK_ROOT="$CONDA_PATH/envs/qualified-{platform}"
-echo "OS detected: $SYS_OS"
-
-# Check if non-vanilla env specified by name
-if [[ -z $SUB_ENV ]]; then
-  echo "No sub env passed"
-  ENV_BASE_PATH="$ENV_BASE_ROOT.yml"
-  ENV_LOCK_PATH="$ENV_LOCK_ROOT.lock"
-else
-  echo "Sub env: $SUB_ENV"
-  ENV_BASE_PATH="$ENV_BASE_ROOT-$SUB_ENV.yml"
-  ENV_LOCK_PATH="$ENV_LOCK_ROOT-$SUB_ENV.lock"
-fi
-
-# Check: conda installed
-if ! command -v conda &>/dev/null; then
-  echo "Conda not installed. Please install conda or source set_env.sh"
-  exit
-fi
-
-echo "Sourcing bash and zsh profiles"
-source $HOME/.bash_profile
-source $HOME/.zshrc
-source $HOME/miniconda/bin/activate
-conda deactivate
-conda activate $BASE_ENV
-
-# Check if conda-lock installed
-if ! command -v conda-lock &>/dev/null; then
-  echo "Conda-lock not installed. Please install conda-lock in the $BASE_ENV env using: conda install -c conda-forge conda-lock"
-  exit
-fi
-
-# Create lock file
-echo "Creating qualified env file"
-if [[ ($SYS_OS == "macos") ]]; then
-  conda-lock -f $ENV_BASE_PATH -p osx-64 --filename-template $ENV_LOCK_PATH
-elif [[ ($SYS_OS == "linux") ]]; then
-  conda-lock -f $ENV_BASE_PATH -p linux-64 --filename-template $ENV_LOCK_PATH
-fi
-
-echo "Qualification complete"

gstlal/share/conda/set_env.sh

@@ -7,15 +7,13 @@ echo "Setting GstLAL Development Environment"
 # Parse arguments
 while getopts n:b: flag; do
   case "${flag}" in
-  n) SUB_ENV=${OPTARG} ;;
-  b) BASE_ENV=${OPTARG} ;;
+  n) ENV_NAME=${OPTARG} ;;
   esac
 done
-if [[ -z "${BASE_ENV}" ]]; then
-  BASE_ENV="base"
+if [[ -z "${ENV_NAME}" ]]; then
+  ENV_NAME="gstlal-dev"
 fi
-echo "Sub env: $SUB_ENV"
-echo "Base env: $BASE_ENV"
+echo "Env name: $ENV_NAME"
 
 # Discover repository paths
 echo "Discovering repository root paths"
@@ -93,10 +91,10 @@ export GSTLAL_FIR_WHITEN=0
 echo "Detecting OS"
 if [[ "$OSTYPE" == "linux-gnu"* ]]; then
   SYS_OS="linux"
-  ENV_LOCK_ROOT=$CONDA_PATH/envs/qualified-linux-64
+  ENV_LOCK_ROOT=$CONDA_PATH/envs/lock/$ENV_NAME-linux-64.lock
 elif [[ "$OSTYPE" == "darwin"* ]]; then
   SYS_OS="macos"
-  ENV_LOCK_ROOT=$CONDA_PATH/envs/qualified-osx-64
+  ENV_LOCK_ROOT=$CONDA_PATH/envs/lock/$ENV_NAME-osx-64.lock
 elif [[ "$OSTYPE" == "cygwin" ]]; then
   SYS_OS="cygwin"
   echo "Cygwin currently unsupported"
@@ -111,16 +109,7 @@ else
 fi
 echo "OS detected: $SYS_OS"
 
-# Check if non-vanilla env specified by name argument flag -n
-if [[ -z $SUB_ENV ]]; then
-  echo "No sub env passed"
-  ENV_LOCK_PATH="$ENV_LOCK_ROOT.lock"
-  ENV_NAME="gstlal-dev-env"
-else
-  echo "Sub env: $SUB_ENV"
-  ENV_LOCK_PATH="$ENV_LOCK_ROOT-$SUB_ENV.lock"
-  ENV_NAME="gstlal-dev-env-$SUB_ENV"
-fi
+ENV_LOCK_PATH="$ENV_LOCK_ROOT"
 
 # MacOS specific installs
 if [[ ($SYS_OS == "macos") ]]; then