Conda build

.conda-env.Dockerfile

+FROM conda/miniconda3-centos7 AS build
+ARG BUILD_DATE
+ARG CI_COMMIT_SHA
+
+# http://label-schema.org/rc1/
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.name="bayeswave-conda-env"
+LABEL org.label-schema.description="BayesWave: distinguish gravitational wave signals from noise and instrumental glitches"
+LABEL org.label-schema.url="https://docs.ligo.org/lscsoft/bayeswave/"
+LABEL org.label-schema.vcs-url="https://git.ligo.org/lscsoft/bayeswave"
+LABEL org.label-schema.build-date="${BUILD_DATE}"
+LABEL org.label-schema.vcs-ref="${CI_COMMIT_SHA}"
+
+## Build bayeswave conda environment
+RUN conda update -yq -nbase conda
+COPY environment.yaml .
+RUN conda env create -f environment.yaml
+SHELL ["conda", "run", "-n", "bayeswave", "/bin/bash", "-c"]
+
+## Use conda-pack to create a standalone enviornment
+## in /venv:
+ RUN conda install -c conda-forge conda-pack
+ RUN conda-pack -n bayeswave -o /tmp/env.tar && \
+   mkdir /venv && cd /venv && tar xf /tmp/env.tar && \
+   rm /tmp/env.tar
+  
+## We've put venv in same path it'll be in final image,
+## so now fix up paths:
+RUN /venv/bin/conda-unpack
+  
+## Next stage
+FROM centos:7
+
+## Essentials
+RUN yum install -y git which && \
+      yum clean all && \
+      rm -rf /var/cache/yum
+    
+## Copy /venv from the previous stage:
+COPY --from=build /venv /venv
+
+ENV PATH="/venv/bin:$PATH"
+SHELL ["/bin/bash", "-c"]
+

.dependencies-el7.Dockerfile

-FROM containers.ligo.org/docker/base:el7
-
-LABEL name="BayesWave Build Dependencies - EL7" \
-      maintainer="James Alexander Clark <james.clark@ligo.org>" \
-      support="Base image for buillding BayesWave " \
-      date="20190726"
-
-# Yum dependencies
-RUN yum upgrade -y && \
-      yum install -y cmake3 \
-      gcc \
-      gcc-c++ \
-      git \
-      help2man \
-      lalapps \
-      lal-devel \
-      lalframe-devel \
-      lalinference-devel \
-      lalsimulation-devel \
-      python-devel \
-      python-ligo-lw \
-      python-pip \
-      rpm-build && \
-      yum clean all && \
-      rm -rf /var/cache/yum
-
-RUN pip install --no-cache-dir htchirp

.gitlab-ci.yml

+# CI/CD workflow for bayeswave
+# 1. Build a container with conda env dependences.  Scheduled builds produce
+# the "latest" image; tags produce a tagged version of the dependencies at the
+# time of that tag.
+# 2. Build the bayeswave package from source in the conda env image and docs pages
+# 3. Test major executables
+# 4. Build & push the bayeswave runtime container from BayesWave source on top of the
+# conda env dependencies.
+# 5. Push documentation for tagged commits only
+
 variables:
   BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
   COMMIT: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-  NIGHTLY: $CI_REGISTRY_IMAGE:nightly
   TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
-  DEPENDS_IMAGE: $CI_REGISTRY_IMAGE/dependencies
+  CONDA_ENV_IMAGE: conda-env
   BUILD_DIR: test-install
   BUILD_TARGET: $CI_PROJECT_DIR/$BUILD_DIR
   TEST_OUTPUT: test-output
 
 stages:
+  - docker
   - build
   - test
-  - docker
   - deploy
 
-build-env:
+.docker_template: &docker_deploy
+  image: docker
+  before_script:
+    - echo "Logging in"
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+  script:
+    - if [ -z $CI_COMMIT_TAG ]; then IMAGE_TAG="latest"; else IMAGE_TAG="$CI_COMMIT_TAG"; fi
+    - if [ -z $IMAGE_NAME ]; then IMAGE="$CI_REGISTRY_IMAGE:$IMAGE_TAG"; else IMAGE="$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" ; fi
+    - echo "Building image - $IMAGE"
+    - docker build --no-cache
+        --build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA}
+        --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+        --build-arg BUILD_IMAGE=$CI_REGISTRY_IMAGE/conda-env:$IMAGE_TAG
+        -t $IMAGE --file $DOCKERFILE .
+    - docker push $IMAGE
+
+# Build the conda environment dependencies
+conda-env:manual:
+  stage: docker
+  <<: *docker_deploy
+  variables:
+    IMAGE_NAME: $CONDA_ENV_IMAGE
+    DOCKERFILE: .conda-env.Dockerfile
+  when: manual
+
+conda-env:schedule:
+  stage: docker
+  <<: *docker_deploy
+  variables:
+    IMAGE_NAME: $CONDA_ENV_IMAGE
+    DOCKERFILE: .conda-env.Dockerfile
+  only:
+    refs:
+      - schedules
+
+conda-env:tag:
+  stage: docker
+  <<: *docker_deploy
+  variables:
+    IMAGE_NAME: $CONDA_ENV_IMAGE
+    DOCKERFILE: .conda-env.Dockerfile
+  only:
+    refs:
+      - tags
+
+# -------------------------------------------------------
+# BUILD
+
+# Build bayeswave from source
+bayeswave:
   stage: build
+  image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
   script:
-    - mkdir -p $BUILD_TARGET
-    - sed "s|INSTALL_DIR|$BUILD_DIR|g" $CI_PROJECT_DIR/etc/bayeswave-user-env.sh > $BUILD_TARGET/bayeswave-user-env.sh
+    - mkdir -p build
+    - cmake . -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=true -DCMAKE_INSTALL_PREFIX=$BUILD_DIR
+    - cmake --build . -- VERBOSE=1
+    - cmake --build . --target install
   artifacts:
-    expire_in: 6h
+    expire_in: 1h
     paths:
       - $BUILD_DIR
 
-build-bayeswave:
+BayesWaveUtils:
   stage: build
-  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
-  script:
-    - pushd $CI_PROJECT_DIR
-    - mkdir -p build
-    - cmake3 . -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=true -DCMAKE_INSTALL_PREFIX=$BUILD_DIR
-    - cmake3 --build . -- VERBOSE=1
-    - cmake3 --build . --target install
+  image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
+  script: 
+    - pushd BayesWaveUtils
+    - python setup.py install --prefix $BUILD_TARGET
     - popd
   artifacts:
-    expire_in: 6h
+    expire_in: 1h
     paths:
       - $BUILD_DIR
 
-build-bayeswave-rpm:
+# Build environment script
+env-script:
   stage: build
-  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
   script:
-    - pushd $CI_PROJECT_DIR
-    - mkdir -p $CI_PROJECT_DIR/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,RPMS}
-    - echo '%_topdir %(echo $CI_PROJECT_DIR)/rpmbuild' > ~/.rpmmacros
-    - cmake3 . -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=true 
-    - cmake3 --build . --target package_source
-    - mv bayeswave-*.tar.xz $CI_PROJECT_DIR/rpmbuild/SOURCES
-    - rpmbuild  -ba bayeswave.spec
-    - popd
+    - mkdir -p $BUILD_TARGET
+    - sed "s|INSTALL_DIR|$BUILD_DIR|g" $CI_PROJECT_DIR/etc/bayeswave-user-env.sh > $BUILD_TARGET/bayeswave-user-env.sh
   artifacts:
-    expire_in: 6h
+    expire_in: 1h
     paths:
-      - $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm
-
-build-BayesWaveUtils:
+      - $BUILD_DIR
+docs:
   stage: build
-  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
-  script: 
-    - pushd BayesWaveUtils
-    - python setup.py install --prefix $BUILD_TARGET
-    - popd
+  image: python:3.7-slim-stretch
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  before_script:
+    # install pandoc >= 2.0.0
+    - apt-get -yqq update
+    - apt-get -yqq install curl
+    - curl --location --output pandoc.deb https://github.com/jgm/pandoc/releases/download/2.7.2/pandoc-2.7.2-1-amd64.deb
+    - dpkg --install pandoc.deb ||  apt-get -y -f install; dpkg --install pandoc.deb; 
+    # install python dependencies
+    - python3 -m pip install -r doc/requirements.txt
+  script:
+    - bash -ex doc/build.sh
   artifacts:
     expire_in: 6h
     paths:
-      - $BUILD_DIR
+      - doc/_build
+  cache:
+    paths:
+      - .cache/pip
+  only:
+      - tags
+
+# -------------------------------------------------------
+# TEST
 
-# FIXME: including the full commandline here is illustrative.  We have a test
-# script in the repository that does the same thing.
-test:BayesWave:
+# Test bayeswave functionality
+BayesWave:
   stage: test
-  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
+  image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
   script:
     - source $BUILD_DIR/bayeswave-user-env.sh
     - cat $BUILD_DIR/bayeswave-user-env.sh
@@ -84,120 +150,86 @@ test:BayesWave:
       --PSDlength 1024 --NCmin 2 --NCmax 2 --dataseed 1234 \
       --Niter 500 --outputDir $TEST_OUTPUT"
   dependencies:
-    - build-env
-    - build-bayeswave
+    - env-script
+    - bayeswave
 
-test:BayesWavePost:
+# Test bayeswavePost functionality
+BayesWavePost:
   stage: test
-  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
+  image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
   script:
     - source $BUILD_DIR/bayeswave-user-env.sh
+    - cat $BUILD_DIR/bayeswave-user-env.sh
+    - ls -R $BUILD_DIR 
     - BayesWavePost --help
   dependencies:
-    - build-env
-    - build-bayeswave
+    - env-script
+    - bayeswave
+
+bayeswave_pipe:
+  stage: test
+  image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
+  script:
+    - ls $BUILD_DIR
+    - source $BUILD_DIR/bayeswave-user-env.sh
+    - bayeswave_pipe --help
+  dependencies:
+    - env-script
+    - BayesWaveUtils
+
+# FIXME: c'mon now.  Someone should add an argparser to these codes so there is SOME kind of usage info
 
-      # test:bayeswave_pipe:
+      # megaplot:
       #   stage: test
-      #   image: containers.ligo.org/lscsoft/bayeswave:latest
+      #   image: $CONDA_ENV_IMAGE
       #   script:
       #     - ls $BUILD_DIR
       #     - source $BUILD_DIR/bayeswave-user-env.sh
-      #     - bayeswave_pipe --help
+      #     - megaplot.py --help
       #   dependencies:
-      #     - build-env
-      #     - build-BayesWaveUtils
+      #     - env-script
+      #     - BayesWaveUtils
+      #
+      # megasky:
+      #   stage: test
+      #   image: $CONDA_ENV_IMAGE
+      #   script:
+      #     - ls $BUILD_DIR
+      #     - source $BUILD_DIR/bayeswave-user-env.sh
+      #     - megasky.py --help
+      #   dependencies:
+      #     - env-script
+      #     - BayesWaveUtils
 
-docker:nightly:
-  # Update the dependency image and bayeswave every night
-  stage: docker
-  image: docker:latest
-  only:
-    - schedules
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
-  script:
-    # Build and push dependency image
-    - docker build --pull -t $DEPENDS_IMAGE:el7 --file .dependencies-el7.Dockerfile .
-    - docker push $DEPENDS_IMAGE:el7
-    - mkdir rpms && mv $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm rpms
-    # Build, test and push bayeswave image
-    - docker build --pull -t $CI_REGISTRY_IMAGE:nightly --file Dockerfile .
-    - docker run $CI_REGISTRY_IMAGE:nightly /test-bayeswave.sh
-    - docker push $CI_REGISTRY_IMAGE:nightly
-  dependencies:
-    - build-bayeswave-rpm
+# -------------------------------------------------------
+# DEPLOY
 
-docker:latest:
-  stage: docker
+# Latest image - install from master
+bayeswave-runtime:latest:
+  stage: deploy
   image: docker:latest
-  only:
-    - master@lscsoft/bayeswave
-    - master@james-clark/bayeswave
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
-  script:
-    - mkdir rpms && mv $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm rpms
-    - docker build --pull -t $CI_REGISTRY_IMAGE:latest --file Dockerfile .
-    # Build, test and push bayeswave image
-    - docker run $CI_REGISTRY_IMAGE:latest /test-bayeswave.sh
-    - docker push $CI_REGISTRY_IMAGE:latest
-  dependencies:
-    - build-bayeswave-rpm
+  <<: *docker_deploy
+  variables:
+    DOCKERFILE: Dockerfile
   except:
-    - schedules
+      - tags
 
-docker:tags:
-  # FIXME: Until we version the dependency image, the tags containers will
-  # always use the nightly build of the dependency image, which always has the
-  # latest release of the dependencies.
-  stage: docker
+bayeswave-runtime:tag:
+  stage: deploy
   image: docker:latest
-  only:
-    - tags@lscsoft/bayeswave
-    - tags@james-clark/bayeswave
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
-  script:
-    - mkdir rpms && mv $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm rpms
-    - docker build --pull -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG --file Dockerfile .
-    # Build, test and push bayeswave image
-    - docker run $CI_REGISTRY_IMAGE:latest /test-bayeswave.sh
-    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
-  dependencies:
-    - build-bayeswave-rpm
-  except:
-    - schedules
-
-docs:
-  stage: build
-  image: python:3.7-slim-stretch
+  <<: *docker_deploy
   variables:
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-  before_script:
-    # install pandoc >= 2.0.0
-    - apt-get -yqq update
-    - apt-get -yqq install curl
-    - curl --location --output pandoc.deb https://github.com/jgm/pandoc/releases/download/2.7.2/pandoc-2.7.2-1-amd64.deb
-    - dpkg --install pandoc.deb ||  apt-get -y -f install; dpkg --install pandoc.deb; 
-    # install python dependencies
-    - python3 -m pip install -r doc/requirements.txt
-  script:
-    - bash -ex doc/build.sh
-  artifacts:
-    expire_in: 6h
-    paths:
-      - doc/_build
-  cache:
-    paths:
-      - .cache/pip
+    DOCKERFILE: Dockerfile
+  only:
+      - tags
 
 pages:
   stage: deploy
   dependencies:
     - docs
   only:
-    - tags@lscsoft/bayeswave
+    - tags
+      #- tags@lscsoft/bayeswave
   script:
     - mv doc/_build/html public
   artifacts:

Dockerfile

-FROM containers.ligo.org/lscsoft/bayeswave/dependencies:el7
+ARG BUILD_IMAGE
+FROM $BUILD_IMAGE
+ARG BUILD_DATE
+ARG CI_COMMIT_SHA
 
-LABEL name="BayesWave - EL7" \
-      maintainer="James Alexander Clark <james.clark@ligo.org>" \
-      support="Reference Platform" \
-      date="20190726"
+# http://label-schema.org/rc1/
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.name="bayeswave-runtime"
+LABEL org.label-schema.description="BayesWave: distinguish gravitational wave signals from noise and instrumental glitches"
+LABEL org.label-schema.url="https://docs.ligo.org/lscsoft/bayeswave/"
+LABEL org.label-schema.vcs-url="https://git.ligo.org/lscsoft/bayeswave"
+LABEL org.label-schema.build-date="${BUILD_DATE}"
+LABEL org.label-schema.vcs-ref="${CI_COMMIT_SHA}"
 
-# Directories we may want to bind
-RUN mkdir -p /cvmfs /hdfs /ceph /hadoop /etc/condor /test
+RUN git clone https://git.ligo.org/james-clark/bayeswave.git && \
+      cd bayeswave && \
+      git checkout ${CI_COMMIT_SHA} && \
+      ./build.sh && \
+      pip install ./BayesWaveUtils && \
+      cd .. && rm -rf bayeswave
 
-# Build and install from RPM built in CI
-COPY rpms /rpms
-RUN  yum upgrade -y && \
-      yum -y localinstall /rpms/*.rpm && \
-      rm -rf /rpms && yum clean all
+# No longer need git
+RUN yum remove -y git
 
-# Python Utils
-COPY test/test-bayeswave.sh /test-bayeswave.sh
-COPY BayesWaveUtils /tmp/BayesWaveUtils
-RUN cd /tmp/BayesWaveUtils && \
-      python /tmp/BayesWaveUtils/setup.py install 
-RUN rm -rf /tmp/*
+## Directories we may want to bind
+RUN mkdir -p /cvmfs /hdfs /hadoop /etc/condor /test
 
-# Remove build dependencies
-RUN yum remove -y cmake3 \
-      gcc \
-      gcc-c++ \
-      help2man \
-      lal-devel \
-      lalframe-devel \
-      lalinference-devel \
-      lalsimulation-devel \
-      python-pip \
-      rpm-build && \
-      yum clean all && \
-      rm -rf /var/cache/yum
-
-WORKDIR /
-ENTRYPOINT ["/bin/bash"]
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]

build-conda-env.sh

+#!/bin/bash -e
+# -*- coding: utf-8 -*-
+# Copyright (C) 2018-2019 James Clark <james.clark@ligo.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+set -e
+
+
+# Build conda environment
+
+if ! which conda 2> /dev/null; then
+  echo "no conda detected, maybe try:"
+  echo "source /cvmfs/oasis.opensciencegrid.org/ligo/sw/conda/etc/profile.d/conda.sh"
+  exit 1
+fi
+
+conda env create -f environment.yaml

dev.Dockerfile

-FROM ligo/builder:el7
-
-LABEL name="BayesWave LALInference_o2 EL7" \
-      maintainer="James Alexander CLark <james.clark@ligo.org>" \
-      date="20190328" \
-      support="Experimental Platform"
-
-# Setup directories for binding
-RUN mkdir -p /cvmfs /hdfs /hadoop /etc/condor
-
-# Dependencies
-RUN yum upgrade -y && \
-    yum clean all && \
-    rm -rf /var/cache/yum
-
-RUN yum update -y && \
-      yum install -y lscsoft-lalsuite-dev cmake3 && \
-      yum clean all && \
-      rm -rf /var/cache/yum
-
-# Clone and build lalsuite from source
-RUN git clone https://git.ligo.org/lscsoft/lalsuite.git
-RUN cd lalsuite && \
-      ./00boot  && \
-      ./configure --prefix /opt/lscsoft/lalsuite --enable-swig-python --disable-lalstochastic --disable-laldetchar && \
-      make -j && \
-      make install && \
-      cd / && \
-      rm -rf lalsuite
-      #git checkout -b lalinference_o2 origin/lalinference_o2 && \
-
-# Copy files from context into container and build BayesWave
-COPY .git /.git
-COPY install.sh /install.sh
-COPY build.sh /build.sh
-COPY src /src
-COPY CMakeLists.txt /CMakeLists.txt
-COPY bayeswave.spec.in /bayeswave.spec.in
-COPY etc/bayeswave-user-env.sh /etc/bayeswave-user-env.sh
-COPY BayesWaveUtils /BayesWaveUtils
-RUN ["/bin/bash", "-c", "source /opt/lscsoft/lalsuite/etc/lalsuiterc && \
-      sh install.sh /opt/lscsoft/bayeswave"]
-RUN rm -rf /.git /install.sh /build.sh /src /bayeswave.spec.in /BayesWaveUtils /etc/bayeswave-user-env.sh
-
-# Set up an entry point so we can source lalsuite and bayeswave user env scripts at startup
-COPY docker-entrypoint.sh /usr/local/bin/
-RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat
-ENTRYPOINT ["docker-entrypoint.sh"]
-
-# Keep a note of explicit build commmand for potential future use:
-# RUN ["/bin/bash", "-c", "source /opt/lscsoft/lalsuite/etc/lalsuiterc && \
-#       mkdir build && \
-#       pushd build && \
-#       cmake3 .. \
-#         -DCMAKE_INSTALL_PREFIX=/opt/lscsoft/bayeswave \
-#         -DCMAKE_BUILD_TYPE=DEBUG \
-#         -DCMAKE_EXPORT_COMPILE_COMMANDS=true && \
-#       cmake3 --build . -- VERBOSE=1 && \
-#       cmake3 --build . --target install"]

docker-entrypoint.sh

+#!/bin/bash
+source /venv/bin/activate
+exec "$@"

environment.yaml

+channels:
+- conda-forge
+dependencies:
+- c-compiler
+- cmake
+- make
+- pkg-config
+- fftw * nompi*
+- gcc_linux-64
+- gsl
+- liblal
+- liblalframe
+- liblalinference
+- liblalsimulation
+- fftw
+- gsl
+- liblal
+- liblalframe
+- liblalinference
+- liblalsimulation
+- pip
+- python
+- astropy
+- healpy
+- lalapps
+- ligo-gracedb
+- ligo-segments
+- ligo.skymap
+- lscsoft-glue
+- matplotlib-base
+- numpy
+- python
+- python-lal
+- python-ligo-lw
+- scipy
+- setuptools
+name: bayeswave

test/test-bayeswave.sh

 #!/bin/bash
 
-
 # single ifo example with trivial MCMC iterations
 BayesWave \
     --ifo H1 --H1-flow 32 \