Merge branch 'add-python-310-image' into 'master'

Add python 3.10 image and testing

See merge request !1115

.gitlab-ci.yml

@@ -97,13 +97,12 @@ precommits-py3.9:
     CACHE_DIR: ".pip39"
     PYVERSION: "python39"
 
-# FIXME: when image builds for 3.10 change this back.
-#precommits-py3.10:
-#  <<: *precommits
-#  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
-#  variables:
-#    CACHE_DIR: ".pip310"
-#    PYVERSION: "python310"
+precommits-py3.10:
+  <<: *precommits
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
+  variables:
+    CACHE_DIR: ".pip310"
+    PYVERSION: "python310"
 
 install:
   stage: initial
@@ -146,19 +145,16 @@ python-3.9:
       - htmlcov/
     expire_in: 30 days
 
-# add back when 3.10 image is available
-#python-3.10:
-#  <<: *unit-test
-#  needs: ["basic-3.10", "precommits-py3.10"]
-#  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
+python-3.10:
+  <<: *unit-test
+  needs: ["basic-3.10", "precommits-py3.10"]
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
 
 .test-sampler: &test-sampler
   stage: test
   script:
     - python -m pip install .
-    - python -m pip install schwimmbad
     - python -m pip list installed
-
     - pytest test/integration/sampler_run_test.py --durations 10 -v
 
 python-3.8-samplers:
@@ -171,11 +167,10 @@ python-3.9-samplers:
   needs: ["basic-3.9", "precommits-py3.9"]
   image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39
 
-# add back when 3.10 image is available
-#python-3.10-samplers:
-#  <<: *test-sampler
-#  needs: ["basic-3.10", "precommits-py3.10"]
-#  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
+python-3.10-samplers:
+  <<: *test-sampler
+  needs: ["basic-3.10", "precommits-py3.10"]
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
 
 integration-tests-python-3.9:
   stage: test
@@ -209,11 +204,10 @@ plotting-python-3.9:
   image: containers.ligo.org/lscsoft/bilby/v2-bilby-python39
   needs: ["basic-3.9", "precommits-py3.9"]
 
-# add back when 3.10 image is available
-#plotting-python-3.10:
-#  <<: *plotting
-#  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
-#  needs: ["basic-3.10", "precommits-py3.10"]
+plotting-python-3.10:
+  <<: *plotting
+  image: containers.ligo.org/lscsoft/bilby/v2-bilby-python310
+  needs: ["basic-3.10", "precommits-py3.10"]
 
 # ------------------- Docs stage -------------------------------------------
 
@@ -276,11 +270,10 @@ build-python39-container:
   variables:
     PYVERSION: "python39"
 
-# add back when 3.10 image is available
-#build-python310-container:
-#  <<: *build-container
-#  variables:
-#    PYVERSION: "python310"
+build-python310-container:
+  <<: *build-container
+  variables:
+    PYVERSION: "python310"
 
 pypi-release:
   stage: deploy

bilby/core/sampler/pymc.py

@@ -61,7 +61,6 @@ class Pymc(MCMCSampler):
         n_init=200000,
         initvals=None,
         trace=None,
-        chain_idx=0,
         chains=2,
         cores=1,
         tune=500,

containers/dockerfile-template

@@ -30,15 +30,7 @@ RUN conda install -n ${{conda_env}} -c conda-forge scikit-image celerite george
 
 # Install dependencies and samplers
 RUN pip install corner healpy cython tables
-RUN conda install -n ${{conda_env}} -c conda-forge dynesty emcee nestle ptemcee
-RUN conda install -n ${{conda_env}} -c conda-forge pymultinest ultranest
-RUN conda install -n ${{conda_env}} -c conda-forge cpnest kombine dnest4 zeus-mcmc
-RUN conda install -n ${{conda_env}} -c conda-forge ptmcmcsampler
-RUN conda install -n ${{conda_env}} -c conda-forge pytorch
-RUN conda install -n ${{conda_env}} -c conda-forge theano-pymc
-RUN conda install -n ${{conda_env}} -c conda-forge pymc3
-RUN conda install -n ${{conda_env}} -c conda-forge pymc pymc-base
-RUN pip install nessai
+RUN conda install -n ${{conda_env}} {conda_samplers} -c conda-forge -c pytorch
 
 # Install Polychord
 RUN apt-get update --allow-releaseinfo-change

containers/v3-dockerfile-test-suite-python310

+# This dockerfile is written automatically and should not be modified by hand.
+
+FROM containers.ligo.org/docker/base:conda
+LABEL name="bilby CI testing" \
+maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
+
+RUN conda update -n base -c defaults conda
+
+ENV conda_env python310
+
+RUN conda create -n ${conda_env} python=3.10
+RUN echo "source activate ${conda_env}" > ~/.bashrc
+ENV PATH /opt/conda/envs/${conda_env}/bin:$PATH
+RUN /bin/bash -c "source activate ${conda_env}"
+RUN conda info
+RUN python --version
+
+# Install conda-installable programs
+RUN conda install -n ${conda_env} -y matplotlib numpy scipy pandas astropy flake8
+RUN conda install -n ${conda_env} -c anaconda coverage configargparse future dill
+RUN conda install -n ${conda_env} -c conda-forge black pytest-cov deepdish arviz
+
+# Install pip-requirements
+RUN pip install --upgrade pip
+RUN pip install --upgrade setuptools coverage-badge parameterized
+
+# Install documentation requirements
+RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs autodoc
+
+# Install testing requirements
+RUN conda install -n ${conda_env} -c conda-forge scikit-image celerite george
+
+# Install dependencies and samplers
+RUN pip install corner healpy cython tables
+RUN conda install -n ${conda_env} dynesty emcee nestle ptemcee pymultinest ultranest cpnest kombine dnest4 zeus-mcmc pytorch pymc nessai -c conda-forge -c pytorch
+
+# Install Polychord
+RUN apt-get update --allow-releaseinfo-change
+RUN apt-get install -y build-essential
+RUN apt-get install -y libblas3 libblas-dev
+RUN apt-get install -y liblapack3 liblapack-dev
+RUN apt-get install -y libatlas3-base libatlas-base-dev
+RUN apt-get install -y gfortran
+
+RUN git clone https://github.com/PolyChord/PolyChordLite.git \
+&& (cd PolyChordLite && python setup.py --no-mpi install)
+
+# Install GW packages
+RUN conda install -n ${conda_env} -c conda-forge python-lalsimulation bilby.cython
+RUN pip install ligo-gracedb gwpy ligo.skymap
+
+# Add the ROQ data to the image
+RUN mkdir roq_basis \
+    && cd roq_basis \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \
+    && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_addcal.hdf5 \
+    && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_multiband_addcal.hdf5

containers/v3-dockerfile-test-suite-python38

@@ -32,15 +32,7 @@ RUN conda install -n ${conda_env} -c conda-forge scikit-image celerite george
 
 # Install dependencies and samplers
 RUN pip install corner healpy cython tables
-RUN conda install -n ${conda_env} -c conda-forge dynesty emcee nestle ptemcee
-RUN conda install -n ${conda_env} -c conda-forge pymultinest ultranest
-RUN conda install -n ${conda_env} -c conda-forge cpnest kombine dnest4 zeus-mcmc
-RUN conda install -n ${conda_env} -c conda-forge ptmcmcsampler
-RUN conda install -n ${conda_env} -c conda-forge pytorch
-RUN conda install -n ${conda_env} -c conda-forge theano-pymc
-RUN conda install -n ${conda_env} -c conda-forge pymc3
-RUN conda install -n ${conda_env} -c conda-forge pymc pymc-base
-RUN pip install nessai
+RUN conda install -n ${conda_env} dynesty emcee nestle ptemcee pymultinest ultranest cpnest kombine dnest4 zeus-mcmc pytorch pymc nessai -c conda-forge -c pytorch
 
 # Install Polychord
 RUN apt-get update --allow-releaseinfo-change

containers/v3-dockerfile-test-suite-python39

@@ -32,15 +32,7 @@ RUN conda install -n ${conda_env} -c conda-forge scikit-image celerite george
 
 # Install dependencies and samplers
 RUN pip install corner healpy cython tables
-RUN conda install -n ${conda_env} -c conda-forge dynesty emcee nestle ptemcee
-RUN conda install -n ${conda_env} -c conda-forge pymultinest ultranest
-RUN conda install -n ${conda_env} -c conda-forge cpnest kombine dnest4 zeus-mcmc
-RUN conda install -n ${conda_env} -c conda-forge ptmcmcsampler
-RUN conda install -n ${conda_env} -c conda-forge pytorch
-RUN conda install -n ${conda_env} -c conda-forge theano-pymc
-RUN conda install -n ${conda_env} -c conda-forge pymc3
-RUN conda install -n ${conda_env} -c conda-forge pymc pymc-base
-RUN pip install nessai
+RUN conda install -n ${conda_env} dynesty emcee nestle ptemcee pymultinest ultranest cpnest kombine dnest4 zeus-mcmc pytorch pymc nessai -c conda-forge -c pytorch
 
 # Install Polychord
 RUN apt-get update --allow-releaseinfo-change

containers/write_dockerfiles.py

@@ -3,13 +3,31 @@ from datetime import date
 with open("dockerfile-template", "r") as ff:
     template = ff.read()
 
-python_versions = [(3, 8), (3, 9)]
+python_versions = [(3, 8), (3, 9), (3, 10)]
 today = date.today().strftime("%Y%m%d")
 
+conda_sampler_dict = dict(
+    python38=[
+        "dynesty", "emcee", "nestle", "ptemcee", "pymultinest", "ultranest",
+        "cpnest", "kombine", "dnest4", "zeus-mcmc",
+        "pytorch", "pymc", "nessai",
+    ],
+    python39=[
+        "dynesty", "emcee", "nestle", "ptemcee", "pymultinest", "ultranest",
+        "cpnest", "kombine", "dnest4", "zeus-mcmc",
+        "pytorch", "pymc", "nessai",
+    ],
+    python310=[
+        "dynesty", "emcee", "nestle", "ptemcee", "pymultinest", "ultranest",
+        "cpnest", "kombine", "dnest4", "zeus-mcmc",
+        "pytorch", "pymc", "nessai",
+    ]
+)
+
 for python_major_version, python_minor_version in python_versions:
+    key = f"python{python_major_version}{python_minor_version}"
     with open(
-        "v3-dockerfile-test-suite-python"
-        f"{python_major_version}{python_minor_version}",
+        f"v3-dockerfile-test-suite-{key}",
         "w"
     ) as ff:
         ff.write(
@@ -19,5 +37,6 @@ for python_major_version, python_minor_version in python_versions:
         ff.write(template.format(
             date=today,
             python_major_version=python_major_version,
-            python_minor_version=python_minor_version
+            python_minor_version=python_minor_version,
+            conda_samplers=" ".join(conda_sampler_dict[key])
         ))

docs/installation.txt

@@ -10,7 +10,7 @@ Installation
 
           $ conda install -c conda-forge bilby
 
-      Supported python versions: 3.6+.
+      Supported python versions: 3.8-3.10.
 
    .. tab:: Pip
 
@@ -18,7 +18,7 @@ Installation
 
           $ pip install bilby
 
-      Supported python versions: 3.6+.
+      Supported python versions: 3.8-3.10.
 
 
 This will install all requirements for running :code:`bilby` for general
@@ -47,7 +47,7 @@ wave inference, please additionally run the following commands.
 Install bilby from source
 -------------------------
 
-:code:`bilby` is developed and tested with Python 3.6+. In the
+:code:`bilby` is developed and tested with Python 3.8-3.10. In the
 following, we assume you have a working python installation, `python pip
 <https://packaging.python.org/tutorials/installing-packages/#use-pip-for-installing)>`_,
 and `git <https://git-scm.com/>`_. See :ref:`installing-python` for our
@@ -116,7 +116,7 @@ file, you can do this from the link above, or run the command
 
    $ wget https://repo.anaconda.com/archive/Anaconda3-5.2.0-Linux-x86_64.sh
 
-this will download an installer for python 3.6, for other versions check
+this will download a linux installer for python, for other versions check
 the `anaconda page <https://www.anaconda.com/download/#linux>`_.
 Then, `run the command
 <https://conda.io/docs/user-guide/install/linux.html>`_

setup.py

@@ -87,6 +87,7 @@ setup(
     classifiers=[
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
     ],

test/core/sampler/pymc_test.py

@@ -33,7 +33,6 @@ class TestPyMC(unittest.TestCase):
             n_init=200000,
             initvals=None,
             trace=None,
-            chain_idx=0,
             chains=2,
             cores=1,
             tune=500,
@@ -57,7 +56,6 @@ class TestPyMC(unittest.TestCase):
             n_init=200000,
             initvals=None,
             trace=None,
-            chain_idx=0,
             chains=2,
             cores=1,
             tune=500,