Skip to content
Snippets Groups Projects
Commit 74125611 authored by James Alexander Clark PhD's avatar James Alexander Clark PhD
Browse files

Merge branch 'conda-build' into 'master'

Conda build

See merge request lscsoft/bayeswave!202
parents fca14e4b 204be28f
No related branches found
No related tags found
1 merge request!11Master
FROM conda/miniconda3-centos7 AS build
ARG BUILD_DATE
ARG CI_COMMIT_SHA
# http://label-schema.org/rc1/
LABEL org.label-schema.schema-version="1.0"
LABEL org.label-schema.name="bayeswave-conda-env"
LABEL org.label-schema.description="BayesWave: distinguish gravitational wave signals from noise and instrumental glitches"
LABEL org.label-schema.url="https://docs.ligo.org/lscsoft/bayeswave/"
LABEL org.label-schema.vcs-url="https://git.ligo.org/lscsoft/bayeswave"
LABEL org.label-schema.build-date="${BUILD_DATE}"
LABEL org.label-schema.vcs-ref="${CI_COMMIT_SHA}"
## Build bayeswave conda environment
RUN conda update -yq -nbase conda
COPY environment.yaml .
RUN conda env create -f environment.yaml
SHELL ["conda", "run", "-n", "bayeswave", "/bin/bash", "-c"]
## Use conda-pack to create a standalone enviornment
## in /venv:
RUN conda install -c conda-forge conda-pack
RUN conda-pack -n bayeswave -o /tmp/env.tar && \
mkdir /venv && cd /venv && tar xf /tmp/env.tar && \
rm /tmp/env.tar
## We've put venv in same path it'll be in final image,
## so now fix up paths:
RUN /venv/bin/conda-unpack
## Next stage
FROM centos:7
## Essentials
RUN yum install -y git which && \
yum clean all && \
rm -rf /var/cache/yum
## Copy /venv from the previous stage:
COPY --from=build /venv /venv
ENV PATH="/venv/bin:$PATH"
SHELL ["/bin/bash", "-c"]
FROM containers.ligo.org/docker/base:el7
LABEL name="BayesWave Build Dependencies - EL7" \
maintainer="James Alexander Clark <james.clark@ligo.org>" \
support="Base image for buillding BayesWave " \
date="20190726"
# Yum dependencies
RUN yum upgrade -y && \
yum install -y cmake3 \
gcc \
gcc-c++ \
git \
help2man \
lalapps \
lal-devel \
lalframe-devel \
lalinference-devel \
lalsimulation-devel \
python-devel \
python-ligo-lw \
python-pip \
rpm-build && \
yum clean all && \
rm -rf /var/cache/yum
RUN pip install --no-cache-dir htchirp
# CI/CD workflow for bayeswave
# 1. Build a container with conda env dependences. Scheduled builds produce
# the "latest" image; tags produce a tagged version of the dependencies at the
# time of that tag.
# 2. Build the bayeswave package from source in the conda env image and docs pages
# 3. Test major executables
# 4. Build & push the bayeswave runtime container from BayesWave source on top of the
# conda env dependencies.
# 5. Push documentation for tagged commits only
variables:
BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
COMMIT: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
NIGHTLY: $CI_REGISTRY_IMAGE:nightly
TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
DEPENDS_IMAGE: $CI_REGISTRY_IMAGE/dependencies
CONDA_ENV_IMAGE: conda-env
BUILD_DIR: test-install
BUILD_TARGET: $CI_PROJECT_DIR/$BUILD_DIR
TEST_OUTPUT: test-output
stages:
- docker
- build
- test
- docker
- deploy
build-env:
.docker_template: &docker_deploy
image: docker
before_script:
- echo "Logging in"
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
script:
- if [ -z $CI_COMMIT_TAG ]; then IMAGE_TAG="latest"; else IMAGE_TAG="$CI_COMMIT_TAG"; fi
- if [ -z $IMAGE_NAME ]; then IMAGE="$CI_REGISTRY_IMAGE:$IMAGE_TAG"; else IMAGE="$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" ; fi
- echo "Building image - $IMAGE"
- docker build --no-cache
--build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA}
--build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
--build-arg BUILD_IMAGE=$CI_REGISTRY_IMAGE/conda-env:$IMAGE_TAG
-t $IMAGE --file $DOCKERFILE .
- docker push $IMAGE
# Build the conda environment dependencies
conda-env:manual:
stage: docker
<<: *docker_deploy
variables:
IMAGE_NAME: $CONDA_ENV_IMAGE
DOCKERFILE: .conda-env.Dockerfile
when: manual
conda-env:schedule:
stage: docker
<<: *docker_deploy
variables:
IMAGE_NAME: $CONDA_ENV_IMAGE
DOCKERFILE: .conda-env.Dockerfile
only:
refs:
- schedules
conda-env:tag:
stage: docker
<<: *docker_deploy
variables:
IMAGE_NAME: $CONDA_ENV_IMAGE
DOCKERFILE: .conda-env.Dockerfile
only:
refs:
- tags
# -------------------------------------------------------
# BUILD
# Build bayeswave from source
bayeswave:
stage: build
image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
script:
- mkdir -p $BUILD_TARGET
- sed "s|INSTALL_DIR|$BUILD_DIR|g" $CI_PROJECT_DIR/etc/bayeswave-user-env.sh > $BUILD_TARGET/bayeswave-user-env.sh
- mkdir -p build
- cmake . -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=true -DCMAKE_INSTALL_PREFIX=$BUILD_DIR
- cmake --build . -- VERBOSE=1
- cmake --build . --target install
artifacts:
expire_in: 6h
expire_in: 1h
paths:
- $BUILD_DIR
build-bayeswave:
BayesWaveUtils:
stage: build
image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
script:
- pushd $CI_PROJECT_DIR
- mkdir -p build
- cmake3 . -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=true -DCMAKE_INSTALL_PREFIX=$BUILD_DIR
- cmake3 --build . -- VERBOSE=1
- cmake3 --build . --target install
image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
script:
- pushd BayesWaveUtils
- python setup.py install --prefix $BUILD_TARGET
- popd
artifacts:
expire_in: 6h
expire_in: 1h
paths:
- $BUILD_DIR
build-bayeswave-rpm:
# Build environment script
env-script:
stage: build
image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
script:
- pushd $CI_PROJECT_DIR
- mkdir -p $CI_PROJECT_DIR/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,RPMS}
- echo '%_topdir %(echo $CI_PROJECT_DIR)/rpmbuild' > ~/.rpmmacros
- cmake3 . -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=true
- cmake3 --build . --target package_source
- mv bayeswave-*.tar.xz $CI_PROJECT_DIR/rpmbuild/SOURCES
- rpmbuild -ba bayeswave.spec
- popd
- mkdir -p $BUILD_TARGET
- sed "s|INSTALL_DIR|$BUILD_DIR|g" $CI_PROJECT_DIR/etc/bayeswave-user-env.sh > $BUILD_TARGET/bayeswave-user-env.sh
artifacts:
expire_in: 6h
expire_in: 1h
paths:
- $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm
build-BayesWaveUtils:
- $BUILD_DIR
docs:
stage: build
image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
script:
- pushd BayesWaveUtils
- python setup.py install --prefix $BUILD_TARGET
- popd
image: python:3.7-slim-stretch
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
before_script:
# install pandoc >= 2.0.0
- apt-get -yqq update
- apt-get -yqq install curl
- curl --location --output pandoc.deb https://github.com/jgm/pandoc/releases/download/2.7.2/pandoc-2.7.2-1-amd64.deb
- dpkg --install pandoc.deb || apt-get -y -f install; dpkg --install pandoc.deb;
# install python dependencies
- python3 -m pip install -r doc/requirements.txt
script:
- bash -ex doc/build.sh
artifacts:
expire_in: 6h
paths:
- $BUILD_DIR
- doc/_build
cache:
paths:
- .cache/pip
only:
- tags
# -------------------------------------------------------
# TEST
# FIXME: including the full commandline here is illustrative. We have a test
# script in the repository that does the same thing.
test:BayesWave:
# Test bayeswave functionality
BayesWave:
stage: test
image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
script:
- source $BUILD_DIR/bayeswave-user-env.sh
- cat $BUILD_DIR/bayeswave-user-env.sh
......@@ -84,120 +150,86 @@ test:BayesWave:
--PSDlength 1024 --NCmin 2 --NCmax 2 --dataseed 1234 \
--Niter 500 --outputDir $TEST_OUTPUT"
dependencies:
- build-env
- build-bayeswave
- env-script
- bayeswave
test:BayesWavePost:
# Test bayeswavePost functionality
BayesWavePost:
stage: test
image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
script:
- source $BUILD_DIR/bayeswave-user-env.sh
- cat $BUILD_DIR/bayeswave-user-env.sh
- ls -R $BUILD_DIR
- BayesWavePost --help
dependencies:
- build-env
- build-bayeswave
- env-script
- bayeswave
bayeswave_pipe:
stage: test
image: $CI_REGISTRY_IMAGE/$CONDA_ENV_IMAGE:latest
script:
- ls $BUILD_DIR
- source $BUILD_DIR/bayeswave-user-env.sh
- bayeswave_pipe --help
dependencies:
- env-script
- BayesWaveUtils
# FIXME: c'mon now. Someone should add an argparser to these codes so there is SOME kind of usage info
# test:bayeswave_pipe:
# megaplot:
# stage: test
# image: containers.ligo.org/lscsoft/bayeswave:latest
# image: $CONDA_ENV_IMAGE
# script:
# - ls $BUILD_DIR
# - source $BUILD_DIR/bayeswave-user-env.sh
# - bayeswave_pipe --help
# - megaplot.py --help
# dependencies:
# - build-env
# - build-BayesWaveUtils
# - env-script
# - BayesWaveUtils
#
# megasky:
# stage: test
# image: $CONDA_ENV_IMAGE
# script:
# - ls $BUILD_DIR
# - source $BUILD_DIR/bayeswave-user-env.sh
# - megasky.py --help
# dependencies:
# - env-script
# - BayesWaveUtils
docker:nightly:
# Update the dependency image and bayeswave every night
stage: docker
image: docker:latest
only:
- schedules
before_script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
script:
# Build and push dependency image
- docker build --pull -t $DEPENDS_IMAGE:el7 --file .dependencies-el7.Dockerfile .
- docker push $DEPENDS_IMAGE:el7
- mkdir rpms && mv $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm rpms
# Build, test and push bayeswave image
- docker build --pull -t $CI_REGISTRY_IMAGE:nightly --file Dockerfile .
- docker run $CI_REGISTRY_IMAGE:nightly /test-bayeswave.sh
- docker push $CI_REGISTRY_IMAGE:nightly
dependencies:
- build-bayeswave-rpm
# -------------------------------------------------------
# DEPLOY
docker:latest:
stage: docker
# Latest image - install from master
bayeswave-runtime:latest:
stage: deploy
image: docker:latest
only:
- master@lscsoft/bayeswave
- master@james-clark/bayeswave
before_script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
script:
- mkdir rpms && mv $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm rpms
- docker build --pull -t $CI_REGISTRY_IMAGE:latest --file Dockerfile .
# Build, test and push bayeswave image
- docker run $CI_REGISTRY_IMAGE:latest /test-bayeswave.sh
- docker push $CI_REGISTRY_IMAGE:latest
dependencies:
- build-bayeswave-rpm
<<: *docker_deploy
variables:
DOCKERFILE: Dockerfile
except:
- schedules
- tags
docker:tags:
# FIXME: Until we version the dependency image, the tags containers will
# always use the nightly build of the dependency image, which always has the
# latest release of the dependencies.
stage: docker
bayeswave-runtime:tag:
stage: deploy
image: docker:latest
only:
- tags@lscsoft/bayeswave
- tags@james-clark/bayeswave
before_script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
script:
- mkdir rpms && mv $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm rpms
- docker build --pull -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG --file Dockerfile .
# Build, test and push bayeswave image
- docker run $CI_REGISTRY_IMAGE:latest /test-bayeswave.sh
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
dependencies:
- build-bayeswave-rpm
except:
- schedules
docs:
stage: build
image: python:3.7-slim-stretch
<<: *docker_deploy
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
before_script:
# install pandoc >= 2.0.0
- apt-get -yqq update
- apt-get -yqq install curl
- curl --location --output pandoc.deb https://github.com/jgm/pandoc/releases/download/2.7.2/pandoc-2.7.2-1-amd64.deb
- dpkg --install pandoc.deb || apt-get -y -f install; dpkg --install pandoc.deb;
# install python dependencies
- python3 -m pip install -r doc/requirements.txt
script:
- bash -ex doc/build.sh
artifacts:
expire_in: 6h
paths:
- doc/_build
cache:
paths:
- .cache/pip
DOCKERFILE: Dockerfile
only:
- tags
pages:
stage: deploy
dependencies:
- docs
only:
- tags@lscsoft/bayeswave
- tags
#- tags@lscsoft/bayeswave
script:
- mv doc/_build/html public
artifacts:
......
FROM containers.ligo.org/lscsoft/bayeswave/dependencies:el7
ARG BUILD_IMAGE
FROM $BUILD_IMAGE
ARG BUILD_DATE
ARG CI_COMMIT_SHA
LABEL name="BayesWave - EL7" \
maintainer="James Alexander Clark <james.clark@ligo.org>" \
support="Reference Platform" \
date="20190726"
# http://label-schema.org/rc1/
LABEL org.label-schema.schema-version="1.0"
LABEL org.label-schema.name="bayeswave-runtime"
LABEL org.label-schema.description="BayesWave: distinguish gravitational wave signals from noise and instrumental glitches"
LABEL org.label-schema.url="https://docs.ligo.org/lscsoft/bayeswave/"
LABEL org.label-schema.vcs-url="https://git.ligo.org/lscsoft/bayeswave"
LABEL org.label-schema.build-date="${BUILD_DATE}"
LABEL org.label-schema.vcs-ref="${CI_COMMIT_SHA}"
# Directories we may want to bind
RUN mkdir -p /cvmfs /hdfs /ceph /hadoop /etc/condor /test
RUN git clone https://git.ligo.org/james-clark/bayeswave.git && \
cd bayeswave && \
git checkout ${CI_COMMIT_SHA} && \
./build.sh && \
pip install ./BayesWaveUtils && \
cd .. && rm -rf bayeswave
# Build and install from RPM built in CI
COPY rpms /rpms
RUN yum upgrade -y && \
yum -y localinstall /rpms/*.rpm && \
rm -rf /rpms && yum clean all
# No longer need git
RUN yum remove -y git
# Python Utils
COPY test/test-bayeswave.sh /test-bayeswave.sh
COPY BayesWaveUtils /tmp/BayesWaveUtils
RUN cd /tmp/BayesWaveUtils && \
python /tmp/BayesWaveUtils/setup.py install
RUN rm -rf /tmp/*
## Directories we may want to bind
RUN mkdir -p /cvmfs /hdfs /hadoop /etc/condor /test
# Remove build dependencies
RUN yum remove -y cmake3 \
gcc \
gcc-c++ \
help2man \
lal-devel \
lalframe-devel \
lalinference-devel \
lalsimulation-devel \
python-pip \
rpm-build && \
yum clean all && \
rm -rf /var/cache/yum
WORKDIR /
ENTRYPOINT ["/bin/bash"]
COPY docker-entrypoint.sh /docker-entrypoint.sh
ENTRYPOINT ["/docker-entrypoint.sh"]
#!/bin/bash -e
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2019 James Clark <james.clark@ligo.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
set -e
# Build conda environment
if ! which conda 2> /dev/null; then
echo "no conda detected, maybe try:"
echo "source /cvmfs/oasis.opensciencegrid.org/ligo/sw/conda/etc/profile.d/conda.sh"
exit 1
fi
conda env create -f environment.yaml
FROM ligo/builder:el7
LABEL name="BayesWave LALInference_o2 EL7" \
maintainer="James Alexander CLark <james.clark@ligo.org>" \
date="20190328" \
support="Experimental Platform"
# Setup directories for binding
RUN mkdir -p /cvmfs /hdfs /hadoop /etc/condor
# Dependencies
RUN yum upgrade -y && \
yum clean all && \
rm -rf /var/cache/yum
RUN yum update -y && \
yum install -y lscsoft-lalsuite-dev cmake3 && \
yum clean all && \
rm -rf /var/cache/yum
# Clone and build lalsuite from source
RUN git clone https://git.ligo.org/lscsoft/lalsuite.git
RUN cd lalsuite && \
./00boot && \
./configure --prefix /opt/lscsoft/lalsuite --enable-swig-python --disable-lalstochastic --disable-laldetchar && \
make -j && \
make install && \
cd / && \
rm -rf lalsuite
#git checkout -b lalinference_o2 origin/lalinference_o2 && \
# Copy files from context into container and build BayesWave
COPY .git /.git
COPY install.sh /install.sh
COPY build.sh /build.sh
COPY src /src
COPY CMakeLists.txt /CMakeLists.txt
COPY bayeswave.spec.in /bayeswave.spec.in
COPY etc/bayeswave-user-env.sh /etc/bayeswave-user-env.sh
COPY BayesWaveUtils /BayesWaveUtils
RUN ["/bin/bash", "-c", "source /opt/lscsoft/lalsuite/etc/lalsuiterc && \
sh install.sh /opt/lscsoft/bayeswave"]
RUN rm -rf /.git /install.sh /build.sh /src /bayeswave.spec.in /BayesWaveUtils /etc/bayeswave-user-env.sh
# Set up an entry point so we can source lalsuite and bayeswave user env scripts at startup
COPY docker-entrypoint.sh /usr/local/bin/
RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat
ENTRYPOINT ["docker-entrypoint.sh"]
# Keep a note of explicit build commmand for potential future use:
# RUN ["/bin/bash", "-c", "source /opt/lscsoft/lalsuite/etc/lalsuiterc && \
# mkdir build && \
# pushd build && \
# cmake3 .. \
# -DCMAKE_INSTALL_PREFIX=/opt/lscsoft/bayeswave \
# -DCMAKE_BUILD_TYPE=DEBUG \
# -DCMAKE_EXPORT_COMPILE_COMMANDS=true && \
# cmake3 --build . -- VERBOSE=1 && \
# cmake3 --build . --target install"]
#!/bin/bash
source /venv/bin/activate
exec "$@"
channels:
- conda-forge
dependencies:
- c-compiler
- cmake
- make
- pkg-config
- fftw * nompi*
- gcc_linux-64
- gsl
- liblal
- liblalframe
- liblalinference
- liblalsimulation
- fftw
- gsl
- liblal
- liblalframe
- liblalinference
- liblalsimulation
- pip
- python
- astropy
- healpy
- lalapps
- ligo-gracedb
- ligo-segments
- ligo.skymap
- lscsoft-glue
- matplotlib-base
- numpy
- python
- python-lal
- python-ligo-lw
- scipy
- setuptools
name: bayeswave
#!/bin/bash
# single ifo example with trivial MCMC iterations
BayesWave \
--ifo H1 --H1-flow 32 \
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment