From 5993e1a3f8a8c044de6d703a3cdc5cf5b79f47d1 Mon Sep 17 00:00:00 2001
From: Gregory Ashton <gregory.ashton@ligo.org>
Date: Thu, 13 Oct 2022 16:02:24 +0000
Subject: [PATCH] Add python 3.10 image and testing
---
.gitlab-ci.yml | 51 +++++++---------
bilby/core/sampler/pymc.py | 1 -
containers/dockerfile-template | 10 +--
containers/v3-dockerfile-test-suite-python310 | 61 +++++++++++++++++++
containers/v3-dockerfile-test-suite-python38 | 10 +--
containers/v3-dockerfile-test-suite-python39 | 10 +--
containers/write_dockerfiles.py | 27 ++++++--
docs/installation.txt | 8 +--
setup.py | 1 +
test/core/sampler/pymc_test.py | 2 -
10 files changed, 114 insertions(+), 67 deletions(-)
create mode 100644 containers/v3-dockerfile-test-suite-python310
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 795b3b226..d32a7266f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -97,13 +97,12 @@ precommits-py3.9:
CACHE_DIR: ".pip39"
PYVERSION: "python39"
-# FIXME: when image builds for 3.10 change this back.
-#precommits-py3.10:
-# <<: *precommits
-# image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
-# variables:
-# CACHE_DIR: ".pip310"
-# PYVERSION: "python310"
+precommits-py3.10:
+ <<: *precommits
+ image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
+ variables:
+ CACHE_DIR: ".pip310"
+ PYVERSION: "python310"
install:
stage: initial
@@ -146,19 +145,16 @@ python-3.9:
- htmlcov/
expire_in: 30 days
-# add back when 3.10 image is available
-#python-3.10:
-# <<: *unit-test
-# needs: ["basic-3.10", "precommits-py3.10"]
-# image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
+python-3.10:
+ <<: *unit-test
+ needs: ["basic-3.10", "precommits-py3.10"]
+ image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
.test-sampler: &test-sampler
stage: test
script:
- python -m pip install .
- - python -m pip install schwimmbad
- python -m pip list installed
-
- pytest test/integration/sampler_run_test.py --durations 10 -v
python-3.8-samplers:
@@ -171,11 +167,10 @@ python-3.9-samplers:
needs: ["basic-3.9", "precommits-py3.9"]
image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39
-# add back when 3.10 image is available
-#python-3.10-samplers:
-# <<: *test-sampler
-# needs: ["basic-3.10", "precommits-py3.10"]
-# image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
+python-3.10-samplers:
+ <<: *test-sampler
+ needs: ["basic-3.10", "precommits-py3.10"]
+ image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
integration-tests-python-3.9:
stage: test
@@ -209,11 +204,10 @@ plotting-python-3.9:
image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39
needs: ["basic-3.9", "precommits-py3.9"]
-# add back when 3.10 image is available
-#plotting-python-3.10:
-# <<: *plotting
-# image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
-# needs: ["basic-3.10", "precommits-py3.10"]
+plotting-python-3.10:
+ <<: *plotting
+ image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
+ needs: ["basic-3.10", "precommits-py3.10"]
# ------------------- Docs stage -------------------------------------------
@@ -276,11 +270,10 @@ build-python39-container:
variables:
PYVERSION: "python39"
-# add back when 3.10 image is available
-#build-python310-container:
-# <<: *build-container
-# variables:
-# PYVERSION: "python310"
+build-python310-container:
+ <<: *build-container
+ variables:
+ PYVERSION: "python310"
pypi-release:
stage: deploy
diff --git a/bilby/core/sampler/pymc.py b/bilby/core/sampler/pymc.py
index 95a57dd4e..9472d3ebb 100644
--- a/bilby/core/sampler/pymc.py
+++ b/bilby/core/sampler/pymc.py
@@ -61,7 +61,6 @@ class Pymc(MCMCSampler):
n_init=200000,
initvals=None,
trace=None,
- chain_idx=0,
chains=2,
cores=1,
tune=500,
diff --git a/containers/dockerfile-template b/containers/dockerfile-template
index d652e5787..37df43f93 100644
--- a/containers/dockerfile-template
+++ b/containers/dockerfile-template
@@ -30,15 +30,7 @@ RUN conda install -n ${{conda_env}} -c conda-forge scikit-image celerite george
# Install dependencies and samplers
RUN pip install corner healpy cython tables
-RUN conda install -n ${{conda_env}} -c conda-forge dynesty emcee nestle ptemcee
-RUN conda install -n ${{conda_env}} -c conda-forge pymultinest ultranest
-RUN conda install -n ${{conda_env}} -c conda-forge cpnest kombine dnest4 zeus-mcmc
-RUN conda install -n ${{conda_env}} -c conda-forge ptmcmcsampler
-RUN conda install -n ${{conda_env}} -c conda-forge pytorch
-RUN conda install -n ${{conda_env}} -c conda-forge theano-pymc
-RUN conda install -n ${{conda_env}} -c conda-forge pymc3
-RUN conda install -n ${{conda_env}} -c conda-forge pymc pymc-base
-RUN pip install nessai
+RUN conda install -n ${{conda_env}} {conda_samplers} -c conda-forge -c pytorch
# Install Polychord
RUN apt-get update --allow-releaseinfo-change
diff --git a/containers/v3-dockerfile-test-suite-python310 b/containers/v3-dockerfile-test-suite-python310
new file mode 100644
index 000000000..0c5afe520
--- /dev/null
+++ b/containers/v3-dockerfile-test-suite-python310
@@ -0,0 +1,61 @@
+# This dockerfile is written automatically and should not be modified by hand.
+
+FROM containers.ligo.org/docker/base:conda
+LABEL name="bilby CI testing" \
+maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
+
+RUN conda update -n base -c defaults conda
+
+ENV conda_env python310
+
+RUN conda create -n ${conda_env} python=3.10
+RUN echo "source activate ${conda_env}" > ~/.bashrc
+ENV PATH /opt/conda/envs/${conda_env}/bin:$PATH
+RUN /bin/bash -c "source activate ${conda_env}"
+RUN conda info
+RUN python --version
+
+# Install conda-installable programs
+RUN conda install -n ${conda_env} -y matplotlib numpy scipy pandas astropy flake8
+RUN conda install -n ${conda_env} -c anaconda coverage configargparse future dill
+RUN conda install -n ${conda_env} -c conda-forge black pytest-cov deepdish arviz
+
+# Install pip-requirements
+RUN pip install --upgrade pip
+RUN pip install --upgrade setuptools coverage-badge parameterized
+
+# Install documentation requirements
+RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs autodoc
+
+# Install testing requirements
+RUN conda install -n ${conda_env} -c conda-forge scikit-image celerite george
+
+# Install dependencies and samplers
+RUN pip install corner healpy cython tables
+RUN conda install -n ${conda_env} dynesty emcee nestle ptemcee pymultinest ultranest cpnest kombine dnest4 zeus-mcmc pytorch pymc nessai -c conda-forge -c pytorch
+
+# Install Polychord
+RUN apt-get update --allow-releaseinfo-change
+RUN apt-get install -y build-essential
+RUN apt-get install -y libblas3 libblas-dev
+RUN apt-get install -y liblapack3 liblapack-dev
+RUN apt-get install -y libatlas3-base libatlas-base-dev
+RUN apt-get install -y gfortran
+
+RUN git clone https://github.com/PolyChord/PolyChordLite.git \
+&& (cd PolyChordLite && python setup.py --no-mpi install)
+
+# Install GW packages
+RUN conda install -n ${conda_env} -c conda-forge python-lalsimulation bilby.cython
+RUN pip install ligo-gracedb gwpy ligo.skymap
+
+# Add the ROQ data to the image
+RUN mkdir roq_basis \
+ && cd roq_basis \
+ && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \
+ && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \
+ && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \
+ && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \
+ && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \
+ && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_addcal.hdf5 \
+ && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_multiband_addcal.hdf5
diff --git a/containers/v3-dockerfile-test-suite-python38 b/containers/v3-dockerfile-test-suite-python38
index 55ccfe42e..f8301025c 100644
--- a/containers/v3-dockerfile-test-suite-python38
+++ b/containers/v3-dockerfile-test-suite-python38
@@ -32,15 +32,7 @@ RUN conda install -n ${conda_env} -c conda-forge scikit-image celerite george
# Install dependencies and samplers
RUN pip install corner healpy cython tables
-RUN conda install -n ${conda_env} -c conda-forge dynesty emcee nestle ptemcee
-RUN conda install -n ${conda_env} -c conda-forge pymultinest ultranest
-RUN conda install -n ${conda_env} -c conda-forge cpnest kombine dnest4 zeus-mcmc
-RUN conda install -n ${conda_env} -c conda-forge ptmcmcsampler
-RUN conda install -n ${conda_env} -c conda-forge pytorch
-RUN conda install -n ${conda_env} -c conda-forge theano-pymc
-RUN conda install -n ${conda_env} -c conda-forge pymc3
-RUN conda install -n ${conda_env} -c conda-forge pymc pymc-base
-RUN pip install nessai
+RUN conda install -n ${conda_env} dynesty emcee nestle ptemcee pymultinest ultranest cpnest kombine dnest4 zeus-mcmc pytorch pymc nessai -c conda-forge -c pytorch
# Install Polychord
RUN apt-get update --allow-releaseinfo-change
diff --git a/containers/v3-dockerfile-test-suite-python39 b/containers/v3-dockerfile-test-suite-python39
index 3281481c3..d436cdfc6 100644
--- a/containers/v3-dockerfile-test-suite-python39
+++ b/containers/v3-dockerfile-test-suite-python39
@@ -32,15 +32,7 @@ RUN conda install -n ${conda_env} -c conda-forge scikit-image celerite george
# Install dependencies and samplers
RUN pip install corner healpy cython tables
-RUN conda install -n ${conda_env} -c conda-forge dynesty emcee nestle ptemcee
-RUN conda install -n ${conda_env} -c conda-forge pymultinest ultranest
-RUN conda install -n ${conda_env} -c conda-forge cpnest kombine dnest4 zeus-mcmc
-RUN conda install -n ${conda_env} -c conda-forge ptmcmcsampler
-RUN conda install -n ${conda_env} -c conda-forge pytorch
-RUN conda install -n ${conda_env} -c conda-forge theano-pymc
-RUN conda install -n ${conda_env} -c conda-forge pymc3
-RUN conda install -n ${conda_env} -c conda-forge pymc pymc-base
-RUN pip install nessai
+RUN conda install -n ${conda_env} dynesty emcee nestle ptemcee pymultinest ultranest cpnest kombine dnest4 zeus-mcmc pytorch pymc nessai -c conda-forge -c pytorch
# Install Polychord
RUN apt-get update --allow-releaseinfo-change
diff --git a/containers/write_dockerfiles.py b/containers/write_dockerfiles.py
index d4394c0e4..3dc242ac4 100644
--- a/containers/write_dockerfiles.py
+++ b/containers/write_dockerfiles.py
@@ -3,13 +3,31 @@ from datetime import date
with open("dockerfile-template", "r") as ff:
template = ff.read()
-python_versions = [(3, 8), (3, 9)]
+python_versions = [(3, 8), (3, 9), (3, 10)]
today = date.today().strftime("%Y%m%d")
+conda_sampler_dict = dict(
+ python38=[
+ "dynesty", "emcee", "nestle", "ptemcee", "pymultinest", "ultranest",
+ "cpnest", "kombine", "dnest4", "zeus-mcmc",
+ "pytorch", "pymc", "nessai",
+ ],
+ python39=[
+ "dynesty", "emcee", "nestle", "ptemcee", "pymultinest", "ultranest",
+ "cpnest", "kombine", "dnest4", "zeus-mcmc",
+ "pytorch", "pymc", "nessai",
+ ],
+ python310=[
+ "dynesty", "emcee", "nestle", "ptemcee", "pymultinest", "ultranest",
+ "cpnest", "kombine", "dnest4", "zeus-mcmc",
+ "pytorch", "pymc", "nessai",
+ ]
+)
+
for python_major_version, python_minor_version in python_versions:
+ key = f"python{python_major_version}{python_minor_version}"
with open(
- "v3-dockerfile-test-suite-python"
- f"{python_major_version}{python_minor_version}",
+ f"v3-dockerfile-test-suite-{key}",
"w"
) as ff:
ff.write(
@@ -19,5 +37,6 @@ for python_major_version, python_minor_version in python_versions:
ff.write(template.format(
date=today,
python_major_version=python_major_version,
- python_minor_version=python_minor_version
+ python_minor_version=python_minor_version,
+ conda_samplers=" ".join(conda_sampler_dict[key])
))
diff --git a/docs/installation.txt b/docs/installation.txt
index bcda5f211..1533c559b 100644
--- a/docs/installation.txt
+++ b/docs/installation.txt
@@ -10,7 +10,7 @@ Installation
$ conda install -c conda-forge bilby
- Supported python versions: 3.6+.
+ Supported python versions: 3.8-3.10.
.. tab:: Pip
@@ -18,7 +18,7 @@ Installation
$ pip install bilby
- Supported python versions: 3.6+.
+ Supported python versions: 3.8-3.10.
This will install all requirements for running :code:`bilby` for general
@@ -47,7 +47,7 @@ wave inference, please additionally run the following commands.
Install bilby from source
-------------------------
-:code:`bilby` is developed and tested with Python 3.6+. In the
+:code:`bilby` is developed and tested with Python 3.8-3.10. In the
following, we assume you have a working python installation, `python pip
<https://packaging.python.org/tutorials/installing-packages/#use-pip-for-installing)>`_,
and `git <https://git-scm.com/>`_. See :ref:`installing-python` for our
@@ -116,7 +116,7 @@ file, you can do this from the link above, or run the command
$ wget https://repo.anaconda.com/archive/Anaconda3-5.2.0-Linux-x86_64.sh
-this will download an installer for python 3.6, for other versions check
+this will download a linux installer for python, for other versions check
the `anaconda page <https://www.anaconda.com/download/#linux>`_.
Then, `run the command
<https://conda.io/docs/user-guide/install/linux.html>`_
diff --git a/setup.py b/setup.py
index 2d55f5d54..e8b575d29 100644
--- a/setup.py
+++ b/setup.py
@@ -87,6 +87,7 @@ setup(
classifiers=[
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
],
diff --git a/test/core/sampler/pymc_test.py b/test/core/sampler/pymc_test.py
index c904e1fd8..3ef4fac80 100644
--- a/test/core/sampler/pymc_test.py
+++ b/test/core/sampler/pymc_test.py
@@ -33,7 +33,6 @@ class TestPyMC(unittest.TestCase):
n_init=200000,
initvals=None,
trace=None,
- chain_idx=0,
chains=2,
cores=1,
tune=500,
@@ -57,7 +56,6 @@ class TestPyMC(unittest.TestCase):
n_init=200000,
initvals=None,
trace=None,
- chain_idx=0,
chains=2,
cores=1,
tune=500,
--
GitLab