diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d79832c2a00055ef3053956f5f860d0978c22ee1..9460a2cfe4a304e03f097fe1a95a7d78597ba751 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,26 @@ stages: - docs - deploy +# ------------------- Initial stage ------------------------------------------- + +# Check author list is up to date +authors: + stage: initial + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37 + script: + - python test/check_author_list.py + +# Test containers scripts are up to date +containers: + stage: initial + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37 + script: + - cd containers + - python write_dockerfiles.py #HACK + # Fail if differences exist. If this fails, you may need to run + # write_dockerfiles.py and commit the changes. + - git diff --exit-code + .test-python: &test-python stage: initial image: python @@ -38,16 +58,68 @@ stages: ${script} --help; done -# test basic setup on python3 +# Test basic setup on python 3.7 basic-3.7: <<: *test-python image: python:3.7 -# test example on python 3.7 +# Test basic setup on python 3.8 +basic-3.8: + <<: *test-python + image: python:3.8 + +# Test basic setup on python 3.9 +basic-3.9: + <<: *test-python + image: python:3.9 + + +precommits-py3.7: + stage: initial + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37 + script: + - source activate python37 + - mkdir -p .pip37 + - pip install --upgrade pip + - pip --cache-dir=.pip37 install --upgrade bilby + - pip --cache-dir=.pip37 install . + - pip --cache-dir=.pip37 install pre-commit + # Run precommits (flake8, spellcheck, isort, no merge conflicts, etc) + - pre-commit run --all-files --verbose --show-diff-on-failure + +precommits-py3.8: + stage: initial + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python38 + script: + - source activate python38 + - mkdir -p .pip38 + - pip install --upgrade pip + - pip --cache-dir=.pip38 install --upgrade bilby + - pip --cache-dir=.pip38 install . + - pip --cache-dir=.pip38 install pre-commit + # Run precommits (flake8, spellcheck, isort, no merge conflicts, etc) + - pre-commit run --all-files --verbose --show-diff-on-failure + +precommits-py3.9: + stage: initial + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39 + script: + - source activate python39 + - mkdir -p .pip38 + - pip install --upgrade pip + - pip --cache-dir=.pip39 install --upgrade bilby + - pip --cache-dir=.pip39 install . + - pip --cache-dir=.pip39 install pre-commit + # Run precommits (flake8, spellcheck, isort, no merge conflicts, etc) + - pre-commit run --all-files --verbose --show-diff-on-failure + +# ------------------- Test stage ------------------------------------------- + +# test example on python 3.7 and build coverage python-3.7: stage: test needs: ["basic-3.7", "precommits-py3.7"] - image: quay.io/bilbydev/v2-dockerfile-test-suite-python37 + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37 script: - python -m pip install . @@ -64,43 +136,30 @@ python-3.7: - coverage_badge.svg - htmlcov/ -docs: - stage: docs - needs: ["basic-3.7"] - image: quay.io/bilbydev/v2-dockerfile-test-suite-python37 - script: - # Make the documentation - - apt-get -yqq install pandoc - - python -m pip install . - - cd docs - - pip install ipykernel ipython jupyter - - cp ../examples/tutorials/*.ipynb ./ - - rm basic_ptmcmc_tutorial.ipynb - - rm compare_samplers.ipynb - - rm visualising_the_results.ipynb - - jupyter nbconvert --to notebook --execute *.ipynb --inplace - - make clean - - make html - - artifacts: - paths: - - docs/_build/html/ - # test example on python 3.8 python-3.8: stage: test - needs: ["basic-3.7", "precommits-py3.7"] - image: quay.io/bilbydev/v2-dockerfile-test-suite-python38 + needs: ["basic-3.8", "precommits-py3.8"] + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python38 script: - python -m pip install . + - pytest +# test example on python 3.9 +python-3.9: + stage: test + needs: ["basic-3.9", "precommits-py3.9"] + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39 + script: + - python -m pip install . - pytest + # test samplers on python 3.7 python-3.7-samplers: stage: test needs: ["basic-3.7", "precommits-py3.7"] - image: quay.io/bilbydev/v2-dockerfile-test-suite-python37 + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37 script: - python -m pip install . @@ -109,59 +168,75 @@ python-3.7-samplers: # test samplers on python 3.8 python-3.8-samplers: stage: test - needs: ["basic-3.7", "precommits-py3.7"] - image: quay.io/bilbydev/v2-dockerfile-test-suite-python38 + needs: ["basic-3.8", "precommits-py3.8"] + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python38 script: - python -m pip install . - pytest test/integration/sampler_run_test.py --durations 10 -# Test containers are up to date -containers: - stage: initial - image: quay.io/bilbydev/v2-dockerfile-test-suite-python37 +# test samplers on python 3.9 +python-3.9-samplers: + stage: test + needs: ["basic-3.9", "precommits-py3.9"] + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39 script: - - cd containers - - python write_dockerfiles.py - # Fail if differences exist. If this fails, you may need to run - # write_dockerfiles.py and commit the changes. - - git diff --exit-code + - python -m pip install . + + - pytest test/integration/sampler_run_test.py --durations 10 -# Tests run at a fixed schedule rather than on push -scheduled-python-3.7: +integration-tests-python-3.7: stage: test - image: quay.io/bilbydev/v2-dockerfile-test-suite-python37 + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37 + needs: ["basic-3.7", "precommits-py3.7"] only: - schedules script: - python -m pip install . - # Run tests which are only done on schedule - pytest test/integration/example_test.py -plotting: +plotting-python-3.7: stage: test - image: quay.io/bilbydev/v2-dockerfile-test-suite-python37 + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37 + needs: ["basic-3.7", "precommits-py3.7"] only: - schedules script: - python -m pip install . - python -m pip install ligo.skymap - - pytest test/gw/plot_test.py -authors: - stage: initial - image: quay.io/bilbydev/v2-dockerfile-test-suite-python37 + +# ------------------- Docs stage ------------------------------------------- + +docs: + stage: docs + needs: ["python-3.7"] + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37 script: - - python test/check_author_list.py + # Make the documentation + - apt-get -yqq install pandoc + - python -m pip install . + - cd docs + - pip install ipykernel ipython jupyter + - cp ../examples/tutorials/*.ipynb ./ + - rm basic_ptmcmc_tutorial.ipynb + - rm compare_samplers.ipynb + - rm visualising_the_results.ipynb + - jupyter nbconvert --to notebook --execute *.ipynb --inplace + - make clean + - make html + + artifacts: + paths: + - docs/_build/html/ + +# ------------------- Deploy stage ------------------------------------------- -pages: +deploy-docs: stage: deploy needs: ["docs", "python-3.7"] - dependencies: - - docs - - python-3.7 script: - mkdir public/ - mv htmlcov/ public/ @@ -174,9 +249,49 @@ pages: only: - master -deploy_release: +# Build the containers +build-python37-container: + stage: deploy + image: docker:19.03.12 + needs: ["containers"] + only: + - schedules + script: + - cd containers + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build --tag v3-bilby-python37 - < v3-dockerfile-test-suite-python37 + - docker image tag v3-bilby-python37 containers.ligo.org/lscsoft/bilby/v2-bilby-python37:latest + - docker image push containers.ligo.org/lscsoft/bilby/v2-bilby-python37:latest + +build-python38-container: stage: deploy - image: quay.io/bilbydev/v2-dockerfile-test-suite-python37 + image: docker:19.03.12 + needs: ["containers"] + only: + - schedules + script: + - cd containers + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build --tag v3-bilby-python38 - < v3-dockerfile-test-suite-python38 + - docker image tag v3-bilby-python38 containers.ligo.org/lscsoft/bilby/v2-bilby-python38:latest + - docker image push containers.ligo.org/lscsoft/bilby/v2-bilby-python38:latest + +build-python39-container: + stage: deploy + image: docker:19.03.12 + needs: ["containers"] + only: + - schedules + script: + - cd containers + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build --tag v3-bilby-python39 - < v3-dockerfile-test-suite-python39 + - docker image tag v3-bilby-python39 containers.ligo.org/lscsoft/bilby/v2-bilby-python39:latest + - docker image push containers.ligo.org/lscsoft/bilby/v2-bilby-python39:latest + +pypi-release: + stage: deploy + image: containers.ligo.org/lscsoft/bilby/v2-bilby-python37 variables: TWINE_USERNAME: $PYPI_USERNAME TWINE_PASSWORD: $PYPI_PASSWORD @@ -187,18 +302,3 @@ deploy_release: - twine upload dist/* only: - tags - - -precommits-py3.7: - stage: initial - image: quay.io/bilbydev/v2-dockerfile-test-suite-python37 - script: - - source activate python37 - - mkdir -p .pip37 - - pip install --upgrade pip - - pip --cache-dir=.pip37 install --upgrade bilby - - pip --cache-dir=.pip37 install . - - pip --cache-dir=.pip37 install pre-commit - - # Run precommits (flake8, spellcheck, isort, no merge conflicts, etc) - - pre-commit run --all-files --verbose --show-diff-on-failure diff --git a/containers/Dockerfile-test-suite-python37 b/containers/Dockerfile-test-suite-python37 deleted file mode 100644 index b1d3762e9cac4771b82fc6f7c88e2f121e26b654..0000000000000000000000000000000000000000 --- a/containers/Dockerfile-test-suite-python37 +++ /dev/null @@ -1,76 +0,0 @@ -LABEL name="bilby Base Enterprise Linux 7" \ -maintainer="Gregory Ashton <gregory.ashton@ligo.org>" \ -date="20190104" - -ENV PATH /opt/conda/bin:$PATH - -# Install backend -RUN apt-get update --fix-missing \ -&& apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libgl1-mesa-glx \ -dh-autoreconf build-essential libarchive-dev wget curl git libhdf5-serial-dev - -# Install python3.7 -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ -/bin/bash ~/miniconda.sh -b -p /opt/conda && \ -rm ~/miniconda.sh && \ -/opt/conda/bin/conda clean -tipsy && \ -ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ -echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ -echo "conda activate base" >> ~/.bashrc - -# Update pip and setuptools -RUN pip install --upgrade pip \ -&& LC_ALL=C pip install --upgrade setuptools - -# Install conda-installable programs -RUN conda install -y numpy scipy matplotlib pandas - -# Install conda-forge-installable programs -RUN conda install -c conda-forge deepdish arviz - -# Install requirements -RUN pip install future \ -pycondor>=0.5 \ -configargparse \ -flake8 \ -mock \ -pipenv \ -coverage \ -pytest-cov \ -coverage-badge - -# Install documentation requirements -RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs - -# Install additional bilby requirements -RUN pip install corner lalsuite astropy gwpy theano healpy tables - -# Install samplers -RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 pymultinest kombine ultranest dnest4 - -# Install pymultinest requirements -RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \ -libatlas3-base libatlas-base-dev cmake build-essential gfortran - -# Install pymultinest -RUN git clone https://github.com/farhanferoz/MultiNest.git \ -&& (cd MultiNest/MultiNest_v3.11_CMake/multinest && mkdir build && cd build && cmake .. && make) - -ENV LD_LIBRARY_PATH $HOME/MultiNest/MultiNest_v3.11_CMake/multinest/lib: - -# Install Polychord -RUN git clone https://github.com/PolyChord/PolyChordLite.git \ -&& (cd PolyChordLite && python setup.py --no-mpi install) - -# Install PTMCMCSampler -RUN git clone https://github.com/jellis18/PTMCMCSampler.git \ -&& (cd PTMCMCSampler && python setup.py install) - -# Add the ROQ data to the image -RUN mkdir roq_basis \ - && cd roq_basis \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \ diff --git a/containers/Singularity.0.3.3 b/containers/Singularity.0.3.3 deleted file mode 100644 index ed8a2c5fbcb199912a29c91879446b1cf9f0ca8c..0000000000000000000000000000000000000000 --- a/containers/Singularity.0.3.3 +++ /dev/null @@ -1,15 +0,0 @@ -Bootstrap: docker -From: bilbydev/bilby-test-suite-python37 - -%help -A singularity container for running bilby scripts. To use, simply execute the -container, providing the bilby_script and any additional arguments. E.g., - -./name_of_this_container.simg run_script.py - -%post -export PATH="/opt/conda/bin:$PATH" -pip install bilby==0.3.3 - -%runscript -exec /opt/conda/bin/python "$@" diff --git a/containers/Singularity.0.3.5 b/containers/Singularity.0.3.5 deleted file mode 100644 index ae53b871425bbb1337ad4b241ae32007b6ab151a..0000000000000000000000000000000000000000 --- a/containers/Singularity.0.3.5 +++ /dev/null @@ -1,15 +0,0 @@ -Bootstrap: docker -From: bilbydev/bilby-test-suite-python37 - -%help -A singularity container for running bilby scripts. To use, simply execute the -container, providing the bilby_script and any additional arguments. E.g., - -./name_of_this_container.simg run_script.py - -%post -export PATH="/opt/conda/bin:$PATH" -pip install bilby==0.3.5 - -%runscript -exec /opt/conda/bin/python "$@" diff --git a/containers/Singularity.0.3.6 b/containers/Singularity.0.3.6 deleted file mode 100644 index 379b9140542bee8147bf3b02448d60166a486fea..0000000000000000000000000000000000000000 --- a/containers/Singularity.0.3.6 +++ /dev/null @@ -1,16 +0,0 @@ -Bootstrap: docker -From: bilbydev/bilby-test-suite-python37 - -%help -A singularity container for running bilby scripts. To use, simply execute the -container, providing the bilby_script and any additional arguments. E.g., - -./name_of_this_container.simg run_script.py - -%post -export PATH="/opt/conda/bin:$PATH" -pip install pandas==0.23 # Temporary fix for issue with saving h5 files -pip install bilby==0.3.6 - -%runscript -exec /opt/conda/bin/python "$@" diff --git a/containers/Singularity.0.4.0 b/containers/Singularity.0.4.0 deleted file mode 100644 index f14895501f339b2ff2c76f4a800234297dab6d7e..0000000000000000000000000000000000000000 --- a/containers/Singularity.0.4.0 +++ /dev/null @@ -1,16 +0,0 @@ -Bootstrap: docker -From: bilbydev/bilby-test-suite-python37 - -%help -A singularity container for running bilby scripts. To use, simply execute the -container, providing the bilby_script and any additional arguments. E.g., - -./name_of_this_container.simg run_script.py - -%post -export PATH="/opt/conda/bin:$PATH" -pip install pandas==0.23 # Temporary fix for issue with saving h5 files -pip install bilby==0.4.0 - -%runscript -exec /opt/conda/bin/python "$@" diff --git a/containers/Singularity.0.4.1 b/containers/Singularity.0.4.1 deleted file mode 100644 index 5e966dea9532b43cac057125d9c75b98fb8ad53b..0000000000000000000000000000000000000000 --- a/containers/Singularity.0.4.1 +++ /dev/null @@ -1,15 +0,0 @@ -Bootstrap: docker -From: bilbydev/bilby-test-suite-python37 - -%help -A singularity container for running bilby scripts. To use, simply execute the -container, providing the bilby_script and any additional arguments. E.g., - -./name_of_this_container.simg run_script.py - -%post -export PATH="/opt/conda/bin:$PATH" -pip install bilby==0.4.1 - -%runscript -exec /opt/conda/bin/python "$@" diff --git a/containers/dockerfile-template b/containers/dockerfile-template index a985fda0292cdb2c0bc8626aae0db762c1330969..82354c22f95190242a5e34cd4da67ab6767583f4 100644 --- a/containers/dockerfile-template +++ b/containers/dockerfile-template @@ -1,5 +1,5 @@ FROM containers.ligo.org/docker/base:conda -LABEL name="bilby Base miniconda3" \ +LABEL name="bilby CI testing" \ maintainer="Gregory Ashton <gregory.ashton@ligo.org>" RUN conda update -n base -c defaults conda @@ -14,7 +14,7 @@ RUN conda info RUN python --version # Install pymultinest requirements -RUN apt-get update +RUN apt-get update --allow-releaseinfo-change RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \ libatlas3-base libatlas-base-dev cmake build-essential gfortran diff --git a/containers/v2-dockerfile-test-suite-python37 b/containers/v3-dockerfile-test-suite-python37 similarity index 96% rename from containers/v2-dockerfile-test-suite-python37 rename to containers/v3-dockerfile-test-suite-python37 index 3fabd2724e5f037466f5a0e72899527d623bfc8d..b633f76f8f237421793518add92348a7c53804a5 100644 --- a/containers/v2-dockerfile-test-suite-python37 +++ b/containers/v3-dockerfile-test-suite-python37 @@ -1,7 +1,7 @@ # This dockerfile is written automatically and should not be modified by hand. FROM containers.ligo.org/docker/base:conda -LABEL name="bilby Base miniconda3" \ +LABEL name="bilby CI testing" \ maintainer="Gregory Ashton <gregory.ashton@ligo.org>" RUN conda update -n base -c defaults conda @@ -16,7 +16,7 @@ RUN conda info RUN python --version # Install pymultinest requirements -RUN apt-get update +RUN apt-get update --allow-releaseinfo-change RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \ libatlas3-base libatlas-base-dev cmake build-essential gfortran diff --git a/containers/v2-dockerfile-test-suite-python38 b/containers/v3-dockerfile-test-suite-python38 similarity index 96% rename from containers/v2-dockerfile-test-suite-python38 rename to containers/v3-dockerfile-test-suite-python38 index d045772dbc175714aa29aaf290d12bc182e5fd83..98d03802ba11a15480d1ccd2d9065a054f18fe9c 100644 --- a/containers/v2-dockerfile-test-suite-python38 +++ b/containers/v3-dockerfile-test-suite-python38 @@ -1,7 +1,7 @@ # This dockerfile is written automatically and should not be modified by hand. FROM containers.ligo.org/docker/base:conda -LABEL name="bilby Base miniconda3" \ +LABEL name="bilby CI testing" \ maintainer="Gregory Ashton <gregory.ashton@ligo.org>" RUN conda update -n base -c defaults conda @@ -16,7 +16,7 @@ RUN conda info RUN python --version # Install pymultinest requirements -RUN apt-get update +RUN apt-get update --allow-releaseinfo-change RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \ libatlas3-base libatlas-base-dev cmake build-essential gfortran diff --git a/containers/v2-dockerfile-test-suite-python36 b/containers/v3-dockerfile-test-suite-python39 similarity index 93% rename from containers/v2-dockerfile-test-suite-python36 rename to containers/v3-dockerfile-test-suite-python39 index 5e775154d945e0e0a48fe54d5bf34773418c3ddd..6b64f65e0d139a924d3f951d0c9cb9249f0f9811 100644 --- a/containers/v2-dockerfile-test-suite-python36 +++ b/containers/v3-dockerfile-test-suite-python39 @@ -1,14 +1,14 @@ # This dockerfile is written automatically and should not be modified by hand. FROM containers.ligo.org/docker/base:conda -LABEL name="bilby Base miniconda3" \ +LABEL name="bilby CI testing" \ maintainer="Gregory Ashton <gregory.ashton@ligo.org>" RUN conda update -n base -c defaults conda -ENV conda_env python36 +ENV conda_env python39 -RUN conda create -n ${conda_env} python=3.6 +RUN conda create -n ${conda_env} python=3.9 RUN echo "source activate ${conda_env}" > ~/.bashrc ENV PATH /opt/conda/envs/${conda_env}/bin:$PATH RUN /bin/bash -c "source activate ${conda_env}" @@ -16,7 +16,7 @@ RUN conda info RUN python --version # Install pymultinest requirements -RUN apt-get update +RUN apt-get update --allow-releaseinfo-change RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \ libatlas3-base libatlas-base-dev cmake build-essential gfortran diff --git a/containers/write_dockerfiles.py b/containers/write_dockerfiles.py index 9fea157bbbc4933bdbe46738086fc030fc1f0c82..b7c154be5de892cddb715dfb941c9e8d7392b71c 100644 --- a/containers/write_dockerfiles.py +++ b/containers/write_dockerfiles.py @@ -3,12 +3,12 @@ from datetime import date with open("dockerfile-template", "r") as ff: template = ff.read() -python_versions = [(3, 6), (3, 7), (3, 8)] +python_versions = [(3, 7), (3, 8), (3, 9)] today = date.today().strftime("%Y%m%d") for python_major_version, python_minor_version in python_versions: with open( - "v2-dockerfile-test-suite-python" + "v3-dockerfile-test-suite-python" f"{python_major_version}{python_minor_version}", "w" ) as ff: diff --git a/requirements.txt b/requirements.txt index 6d8d4b42448331a8dced95bdbcc0150a99b0e162..423a97d093169b3a04f0cd6855a047fde8e7ad6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ dill tqdm h5py tables -astropy<4.3 +astropy