Move containers to gitlab CI

.gitlab-ci.yml

@@ -15,6 +15,26 @@ stages:
   - docs
   - deploy
 
+# ------------------- Initial stage -------------------------------------------
+
+# Check author list is up to date
+authors:
+  stage: initial
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37
+  script:
+    - python test/check_author_list.py
+
+# Test containers scripts are up to date
+containers:
+  stage: initial
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37
+  script:
+    - cd containers
+    - python write_dockerfiles.py #HACK
+    # Fail if differences exist. If this fails, you may need to run
+    # write_dockerfiles.py and commit the changes.
+    - git diff --exit-code
+
 .test-python: &test-python
   stage: initial
   image: python
@@ -38,16 +58,68 @@ stages:
           ${script} --help;
       done
 
-# test basic setup on python3
+# Test basic setup on python 3.7
 basic-3.7:
   <<: *test-python
   image: python:3.7
 
-# test example on python 3.7
+# Test basic setup on python 3.8
+basic-3.8:
+  <<: *test-python
+  image: python:3.8
+
+# Test basic setup on python 3.9
+basic-3.9:
+  <<: *test-python
+  image: python:3.9
+
+
+precommits-py3.7:
+  stage: initial
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37
+  script:
+    - source activate python37
+    - mkdir -p .pip37
+    - pip install --upgrade pip
+    - pip --cache-dir=.pip37 install --upgrade bilby
+    - pip --cache-dir=.pip37 install .
+    - pip --cache-dir=.pip37 install pre-commit
+    # Run precommits (flake8, spellcheck, isort, no merge conflicts, etc)
+    - pre-commit run --all-files --verbose --show-diff-on-failure
+
+precommits-py3.8:
+  stage: initial
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python38
+  script:
+    - source activate python38
+    - mkdir -p .pip38
+    - pip install --upgrade pip
+    - pip --cache-dir=.pip38 install --upgrade bilby
+    - pip --cache-dir=.pip38 install .
+    - pip --cache-dir=.pip38 install pre-commit
+    # Run precommits (flake8, spellcheck, isort, no merge conflicts, etc)
+    - pre-commit run --all-files --verbose --show-diff-on-failure
+
+precommits-py3.9:
+  stage: initial
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39
+  script:
+    - source activate python39
+    - mkdir -p .pip38
+    - pip install --upgrade pip
+    - pip --cache-dir=.pip39 install --upgrade bilby
+    - pip --cache-dir=.pip39 install .
+    - pip --cache-dir=.pip39 install pre-commit
+    # Run precommits (flake8, spellcheck, isort, no merge conflicts, etc)
+    - pre-commit run --all-files --verbose --show-diff-on-failure
+
+# ------------------- Test stage -------------------------------------------
+
+# test example on python 3.7 and build coverage
 python-3.7:
   stage: test
   needs: ["basic-3.7", "precommits-py3.7"]
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python37
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37
   script:
     - python -m pip install .
 
@@ -64,43 +136,30 @@ python-3.7:
       - coverage_badge.svg
       - htmlcov/
 
-docs:
-  stage: docs
-  needs: ["basic-3.7"]
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python37
-  script:
-    # Make the documentation
-    - apt-get -yqq install pandoc
-    - python -m pip install .
-    - cd docs
-    - pip install ipykernel ipython jupyter
-    - cp ../examples/tutorials/*.ipynb ./
-    - rm basic_ptmcmc_tutorial.ipynb
-    - rm compare_samplers.ipynb
-    - rm visualising_the_results.ipynb
-    - jupyter nbconvert --to notebook --execute *.ipynb --inplace
-    - make clean
-    - make html
-
-  artifacts:
-    paths:
-      - docs/_build/html/
-
 # test example on python 3.8
 python-3.8:
   stage: test
-  needs: ["basic-3.7", "precommits-py3.7"]
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python38
+  needs: ["basic-3.8", "precommits-py3.8"]
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python38
   script:
     - python -m pip install .
+    - pytest
 
+# test example on python 3.9
+python-3.9:
+  stage: test
+  needs: ["basic-3.9", "precommits-py3.9"]
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39
+  script:
+    - python -m pip install .
     - pytest
 
+
 # test samplers on python 3.7
 python-3.7-samplers:
   stage: test
   needs: ["basic-3.7", "precommits-py3.7"]
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python37
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37
   script:
     - python -m pip install .
 
@@ -109,59 +168,75 @@ python-3.7-samplers:
 # test samplers on python 3.8
 python-3.8-samplers:
   stage: test
-  needs: ["basic-3.7", "precommits-py3.7"]
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python38
+  needs: ["basic-3.8", "precommits-py3.8"]
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python38
   script:
     - python -m pip install .
 
     - pytest test/integration/sampler_run_test.py --durations 10
 
-# Test containers are up to date
-containers:
-  stage: initial
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python37
+# test samplers on python 3.9
+python-3.9-samplers:
+  stage: test
+  needs: ["basic-3.9", "precommits-py3.9"]
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39
   script:
-    - cd containers
-    - python write_dockerfiles.py
-    # Fail if differences exist. If this fails, you may need to run
-    # write_dockerfiles.py and commit the changes.
-    - git diff --exit-code
+    - python -m pip install .
+
+    - pytest test/integration/sampler_run_test.py --durations 10
 
-# Tests run at a fixed schedule rather than on push
-scheduled-python-3.7:
+integration-tests-python-3.7:
   stage: test
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python37
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37
+  needs: ["basic-3.7", "precommits-py3.7"]
   only:
     - schedules
   script:
     - python -m pip install .
-
     # Run tests which are only done on schedule
     - pytest test/integration/example_test.py
 
-plotting:
+plotting-python-3.7:
   stage: test
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python37
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37
+  needs: ["basic-3.7", "precommits-py3.7"]
   only:
     - schedules
   script:
     - python -m pip install .
     - python -m pip install ligo.skymap
-
     - pytest test/gw/plot_test.py
 
-authors:
-  stage: initial
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python37
+
+# ------------------- Docs stage -------------------------------------------
+
+docs:
+  stage: docs
+  needs: ["python-3.7"]
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37
   script:
-    - python test/check_author_list.py
+    # Make the documentation
+    - apt-get -yqq install pandoc
+    - python -m pip install .
+    - cd docs
+    - pip install ipykernel ipython jupyter
+    - cp ../examples/tutorials/*.ipynb ./
+    - rm basic_ptmcmc_tutorial.ipynb
+    - rm compare_samplers.ipynb
+    - rm visualising_the_results.ipynb
+    - jupyter nbconvert --to notebook --execute *.ipynb --inplace
+    - make clean
+    - make html
+
+  artifacts:
+    paths:
+      - docs/_build/html/
+
+# ------------------- Deploy stage -------------------------------------------
 
-pages:
+deploy-docs:
   stage: deploy
   needs: ["docs", "python-3.7"]
-  dependencies:
-    - docs
-    - python-3.7
   script:
     - mkdir public/
     - mv htmlcov/ public/
@@ -174,9 +249,49 @@ pages:
   only:
     - master
 
-deploy_release:
+# Build the containers
+build-python37-container:
+  stage: deploy
+  image: docker:19.03.12
+  needs: ["containers"]
+  only:
+    - schedules
+  script:
+    - cd containers
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build --tag v3-bilby-python37 - < v3-dockerfile-test-suite-python37
+    - docker image tag v3-bilby-python37 containers.ligo.org/lscsoft/bilby/v2-bilby-python37:latest
+    - docker image push containers.ligo.org/lscsoft/bilby/v2-bilby-python37:latest
+
+build-python38-container:
   stage: deploy
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python37
+  image: docker:19.03.12
+  needs: ["containers"]
+  only:
+    - schedules
+  script:
+    - cd containers
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build --tag v3-bilby-python38 - < v3-dockerfile-test-suite-python38
+    - docker image tag v3-bilby-python38 containers.ligo.org/lscsoft/bilby/v2-bilby-python38:latest
+    - docker image push containers.ligo.org/lscsoft/bilby/v2-bilby-python38:latest
+
+build-python39-container:
+  stage: deploy
+  image: docker:19.03.12
+  needs: ["containers"]
+  only:
+    - schedules
+  script:
+    - cd containers
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build --tag v3-bilby-python39 - < v3-dockerfile-test-suite-python39
+    - docker image tag v3-bilby-python39 containers.ligo.org/lscsoft/bilby/v2-bilby-python39:latest
+    - docker image push containers.ligo.org/lscsoft/bilby/v2-bilby-python39:latest
+
+pypi-release:
+  stage: deploy
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37
   variables:
     TWINE_USERNAME: $PYPI_USERNAME
     TWINE_PASSWORD: $PYPI_PASSWORD
@@ -187,18 +302,3 @@ deploy_release:
     - twine upload dist/*
   only:
   - tags
-
-
-precommits-py3.7:
-  stage: initial
-  image: quay.io/bilbydev/v2-dockerfile-test-suite-python37
-  script:
-    - source activate python37
-    - mkdir -p .pip37
-    - pip install --upgrade pip
-    - pip --cache-dir=.pip37 install --upgrade bilby
-    - pip --cache-dir=.pip37 install .
-    - pip --cache-dir=.pip37 install pre-commit
-
-    # Run precommits (flake8, spellcheck, isort, no merge conflicts, etc)
-    - pre-commit run --all-files --verbose --show-diff-on-failure

containers/Dockerfile-test-suite-python37

-LABEL name="bilby Base Enterprise Linux 7" \
-maintainer="Gregory Ashton <gregory.ashton@ligo.org>" \
-date="20190104"
-
-ENV PATH /opt/conda/bin:$PATH
-
-# Install backend
-RUN apt-get update --fix-missing \
-&& apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libgl1-mesa-glx \
-dh-autoreconf build-essential libarchive-dev wget curl git libhdf5-serial-dev
-
-# Install python3.7
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \
-/bin/bash ~/miniconda.sh -b -p /opt/conda && \
-rm ~/miniconda.sh && \
-/opt/conda/bin/conda clean -tipsy && \
-ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-echo "conda activate base" >> ~/.bashrc
-
-# Update pip and setuptools
-RUN pip install --upgrade pip \
-&& LC_ALL=C pip install --upgrade setuptools
-
-# Install conda-installable programs
-RUN conda install -y numpy scipy matplotlib pandas
-
-# Install conda-forge-installable programs
-RUN conda install -c conda-forge deepdish arviz
-
-# Install requirements
-RUN pip install future \
-pycondor>=0.5 \
-configargparse \
-flake8 \
-mock \
-pipenv \
-coverage \
-pytest-cov \
-coverage-badge
-
-# Install documentation requirements
-RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs
-
-# Install additional bilby requirements
-RUN pip install corner lalsuite astropy gwpy theano healpy tables
-
-# Install samplers
-RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 pymultinest kombine ultranest dnest4
-
-# Install pymultinest requirements
-RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \
-libatlas3-base libatlas-base-dev cmake build-essential gfortran
-
-# Install pymultinest
-RUN git clone https://github.com/farhanferoz/MultiNest.git \
-&& (cd MultiNest/MultiNest_v3.11_CMake/multinest && mkdir build && cd build && cmake .. && make)
-
-ENV LD_LIBRARY_PATH $HOME/MultiNest/MultiNest_v3.11_CMake/multinest/lib:
-
-# Install Polychord
-RUN git clone https://github.com/PolyChord/PolyChordLite.git \
-&& (cd PolyChordLite && python setup.py --no-mpi install)
-
-# Install PTMCMCSampler
-RUN git clone https://github.com/jellis18/PTMCMCSampler.git \
-&& (cd PTMCMCSampler && python setup.py install)
-
-# Add the ROQ data to the image
-RUN mkdir roq_basis \
-    && cd roq_basis \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \

containers/Singularity.0.3.3

-Bootstrap: docker
-From: bilbydev/bilby-test-suite-python37
-
-%help
-A singularity container for running bilby scripts. To use, simply execute the
-container, providing the bilby_script and any additional arguments. E.g.,
-
-./name_of_this_container.simg run_script.py
-
-%post
-export PATH="/opt/conda/bin:$PATH"
-pip install bilby==0.3.3
-
-%runscript
-exec /opt/conda/bin/python "$@"

containers/Singularity.0.3.5

-Bootstrap: docker
-From: bilbydev/bilby-test-suite-python37
-
-%help
-A singularity container for running bilby scripts. To use, simply execute the
-container, providing the bilby_script and any additional arguments. E.g.,
-
-./name_of_this_container.simg run_script.py
-
-%post
-export PATH="/opt/conda/bin:$PATH"
-pip install bilby==0.3.5
-
-%runscript
-exec /opt/conda/bin/python "$@"

containers/Singularity.0.3.6

-Bootstrap: docker
-From: bilbydev/bilby-test-suite-python37
-
-%help
-A singularity container for running bilby scripts. To use, simply execute the
-container, providing the bilby_script and any additional arguments. E.g.,
-
-./name_of_this_container.simg run_script.py
-
-%post
-export PATH="/opt/conda/bin:$PATH"
-pip install pandas==0.23  # Temporary fix for issue with saving h5 files
-pip install bilby==0.3.6
-
-%runscript
-exec /opt/conda/bin/python "$@"

containers/Singularity.0.4.0

-Bootstrap: docker
-From: bilbydev/bilby-test-suite-python37
-
-%help
-A singularity container for running bilby scripts. To use, simply execute the
-container, providing the bilby_script and any additional arguments. E.g.,
-
-./name_of_this_container.simg run_script.py
-
-%post
-export PATH="/opt/conda/bin:$PATH"
-pip install pandas==0.23  # Temporary fix for issue with saving h5 files
-pip install bilby==0.4.0
-
-%runscript
-exec /opt/conda/bin/python "$@"

containers/Singularity.0.4.1

-Bootstrap: docker
-From: bilbydev/bilby-test-suite-python37
-
-%help
-A singularity container for running bilby scripts. To use, simply execute the
-container, providing the bilby_script and any additional arguments. E.g.,
-
-./name_of_this_container.simg run_script.py
-
-%post
-export PATH="/opt/conda/bin:$PATH"
-pip install bilby==0.4.1
-
-%runscript
-exec /opt/conda/bin/python "$@"

containers/dockerfile-template

 FROM containers.ligo.org/docker/base:conda
-LABEL name="bilby Base miniconda3" \
+LABEL name="bilby CI testing" \
 maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
 
 RUN conda update -n base -c defaults conda
@@ -14,7 +14,7 @@ RUN conda info
 RUN python --version
 
 # Install pymultinest requirements
-RUN apt-get update
+RUN apt-get update --allow-releaseinfo-change
 RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \
 libatlas3-base libatlas-base-dev cmake build-essential gfortran
 

containers/v2-dockerfile-test-suite-python37 → containers/v3-dockerfile-test-suite-python37

 # This dockerfile is written automatically and should not be modified by hand.
 
 FROM containers.ligo.org/docker/base:conda
-LABEL name="bilby Base miniconda3" \
+LABEL name="bilby CI testing" \
 maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
 
 RUN conda update -n base -c defaults conda
@@ -16,7 +16,7 @@ RUN conda info
 RUN python --version
 
 # Install pymultinest requirements
-RUN apt-get update
+RUN apt-get update --allow-releaseinfo-change
 RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \
 libatlas3-base libatlas-base-dev cmake build-essential gfortran
 

containers/v2-dockerfile-test-suite-python38 → containers/v3-dockerfile-test-suite-python38

 # This dockerfile is written automatically and should not be modified by hand.
 
 FROM containers.ligo.org/docker/base:conda
-LABEL name="bilby Base miniconda3" \
+LABEL name="bilby CI testing" \
 maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
 
 RUN conda update -n base -c defaults conda
@@ -16,7 +16,7 @@ RUN conda info
 RUN python --version
 
 # Install pymultinest requirements
-RUN apt-get update
+RUN apt-get update --allow-releaseinfo-change
 RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \
 libatlas3-base libatlas-base-dev cmake build-essential gfortran
 

containers/v2-dockerfile-test-suite-python36 → containers/v3-dockerfile-test-suite-python39

 # This dockerfile is written automatically and should not be modified by hand.
 
 FROM containers.ligo.org/docker/base:conda
-LABEL name="bilby Base miniconda3" \
+LABEL name="bilby CI testing" \
 maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
 
 RUN conda update -n base -c defaults conda
 
-ENV conda_env python36
+ENV conda_env python39
 
-RUN conda create -n ${conda_env} python=3.6
+RUN conda create -n ${conda_env} python=3.9
 RUN echo "source activate ${conda_env}" > ~/.bashrc
 ENV PATH /opt/conda/envs/${conda_env}/bin:$PATH
 RUN /bin/bash -c "source activate ${conda_env}"
@@ -16,7 +16,7 @@ RUN conda info
 RUN python --version
 
 # Install pymultinest requirements
-RUN apt-get update
+RUN apt-get update --allow-releaseinfo-change
 RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \
 libatlas3-base libatlas-base-dev cmake build-essential gfortran
 

containers/write_dockerfiles.py

@@ -3,12 +3,12 @@ from datetime import date
 with open("dockerfile-template", "r") as ff:
     template = ff.read()
 
-python_versions = [(3, 6), (3, 7), (3, 8)]
+python_versions = [(3, 7), (3, 8), (3, 9)]
 today = date.today().strftime("%Y%m%d")
 
 for python_major_version, python_minor_version in python_versions:
     with open(
-        "v2-dockerfile-test-suite-python"
+        "v3-dockerfile-test-suite-python"
         f"{python_major_version}{python_minor_version}",
         "w"
     ) as ff:

requirements.txt

@@ -10,4 +10,4 @@ dill
 tqdm
 h5py
 tables
-astropy<4.3
+astropy