From d0eda9049bbf1058bf30883d3be2466dab097722 Mon Sep 17 00:00:00 2001
From: Colm Talbot <colm.talbot@ligo.org>
Date: Fri, 4 Sep 2020 07:26:18 -0500
Subject: [PATCH] add auto generated dockerfiles

---
 .gitlab-ci.yml                               | 11 +++
 containers/Dockerfile-bilby                  |  7 --
 containers/Dockerfile-test-suite-python27    | 71 ------------------
 containers/Dockerfile-test-suite-python37    | 78 --------------------
 containers/dockerfile-template               | 55 ++++++++++++++
 containers/v2-dockerfile-test-suite-python35 | 14 ++--
 containers/v2-dockerfile-test-suite-python36 | 14 ++--
 containers/v2-dockerfile-test-suite-python37 | 14 ++--
 containers/v2-dockerfile-test-suite-python38 | 14 ++--
 containers/write_dockerfiles.py              | 23 ++++++
 10 files changed, 109 insertions(+), 192 deletions(-)
 delete mode 100644 containers/Dockerfile-bilby
 delete mode 100644 containers/Dockerfile-test-suite-python27
 delete mode 100644 containers/Dockerfile-test-suite-python37
 create mode 100644 containers/dockerfile-template
 create mode 100644 containers/write_dockerfiles.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6085e0e7c..4f89b66f8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -103,6 +103,17 @@ python-3.6-samplers:
 
     - pytest test/core/sampler/sampler_run_test.py
 
+# Test containers are up to date
+containers:
+  stage: test
+  image: bilbydev/v2-dockerfile-test-suite-python37
+  script:
+    - cd containers
+    - python write_dockerfiles.py
+    # Fail if differences exist. If this fails, you may need to run
+    # write_dockerfiles.py and commit the changes.
+    - git diff --exit-code
+
 # Tests run at a fixed schedule rather than on push
 scheduled-python-3.7:
   stage: test
diff --git a/containers/Dockerfile-bilby b/containers/Dockerfile-bilby
deleted file mode 100644
index 45a53475d..000000000
--- a/containers/Dockerfile-bilby
+++ /dev/null
@@ -1,7 +0,0 @@
-FROM bilbydev/bilby-test-suite-python37
-
-LABEL name="bilby" \
-maintainer="Gregory Ashton <gregory.ashton@ligo.org>" \
-date="20190130"
-
-RUN pip install bilby
diff --git a/containers/Dockerfile-test-suite-python27 b/containers/Dockerfile-test-suite-python27
deleted file mode 100644
index c4b7b1dfe..000000000
--- a/containers/Dockerfile-test-suite-python27
+++ /dev/null
@@ -1,71 +0,0 @@
-FROM ubuntu:18.04
-
-LABEL name="bilby Base Enterprise Linux 7" \
-maintainer="Gregory Ashton <gregory.ashton@ligo.org>" \
-date="20190104"
-
-ENV PATH /opt/conda/bin:$PATH
-
-# Install backend
-RUN apt-get update --fix-missing \
-&& apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libgl1-mesa-glx \
-dh-autoreconf build-essential libarchive-dev wget curl git libhdf5-serial-dev
-
-# Install python2.7
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \
-/bin/bash ~/miniconda.sh -b -p /opt/conda && \
-rm ~/miniconda.sh && \
-ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-echo "conda activate base" >> ~/.bashrc
-
-# Install conda-installable programs
-RUN conda install -y numpy scipy matplotlib pandas==0.23
-
-RUN conda install -c conda-forge deepdish arviz
-
-# Install requirements
-RUN pip install --upgrade pip \
-&& pip install --upgrade setuptools \
-&& pip install future \
-pycondor>=0.5 \
-configargparse \
-flake8 \
-mock \
-pipenv \
-coverage \
-pytest-cov \
-coverage-badge
-
-# Install additional bilby requirements
-RUN pip install corner lalsuite astropy gwpy theano tables
-
-# Install samplers
-RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 pymultinest kombine ultranest dnest4
-
-# Install pymultinest requirements
-RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \
-libatlas3-base libatlas-base-dev cmake build-essential gfortran
-
-# Install pymultinest
-RUN git clone https://github.com/farhanferoz/MultiNest.git \
-&& (cd MultiNest/MultiNest_v3.11_CMake/multinest && mkdir build && cd build && cmake .. && make)
-
-ENV LD_LIBRARY_PATH $HOME/MultiNest/MultiNest_v3.11_CMake/multinest/lib:
-
-# Install Polychord
-RUN git clone https://github.com/PolyChord/PolyChordLite.git \
-&& (cd PolyChordLite && python setup.py --no-mpi install)
-
-# Install PTMCMCSampler
-RUN git clone https://github.com/jellis18/PTMCMCSampler.git \
-&& (cd PTMCMCSampler && python setup.py install)
-
-# Add the ROQ data to the image
-RUN mkdir roq_basis \
-    && cd roq_basis \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \
diff --git a/containers/Dockerfile-test-suite-python37 b/containers/Dockerfile-test-suite-python37
deleted file mode 100644
index cf7727897..000000000
--- a/containers/Dockerfile-test-suite-python37
+++ /dev/null
@@ -1,78 +0,0 @@
-FROM ubuntu:18.04
-
-LABEL name="bilby Base Enterprise Linux 7" \
-maintainer="Gregory Ashton <gregory.ashton@ligo.org>" \
-date="20190104"
-
-ENV PATH /opt/conda/bin:$PATH
-
-# Install backend
-RUN apt-get update --fix-missing \
-&& apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libgl1-mesa-glx \
-dh-autoreconf build-essential libarchive-dev wget curl git libhdf5-serial-dev
-
-# Install python3.7
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \
-/bin/bash ~/miniconda.sh -b -p /opt/conda && \
-rm ~/miniconda.sh && \
-/opt/conda/bin/conda clean -tipsy && \
-ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-echo "conda activate base" >> ~/.bashrc
-
-# Update pip and setuptools
-RUN pip install --upgrade pip \
-&& LC_ALL=C pip install --upgrade setuptools
-
-# Install conda-installable programs
-RUN conda install -y numpy scipy matplotlib pandas
-
-# Install conda-forge-installable programs
-RUN conda install -c conda-forge deepdish arviz
-
-# Install requirements
-RUN pip install future \
-pycondor>=0.5 \
-configargparse \
-flake8 \
-mock \
-pipenv \
-coverage \
-pytest-cov \
-coverage-badge
-
-# Install documentation requirements
-RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs
-
-# Install additional bilby requirements
-RUN pip install corner lalsuite astropy gwpy theano healpy tables
-
-# Install samplers
-RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 pymultinest kombine ultranest dnest4
-
-# Install pymultinest requirements
-RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \
-libatlas3-base libatlas-base-dev cmake build-essential gfortran
-
-# Install pymultinest
-RUN git clone https://github.com/farhanferoz/MultiNest.git \
-&& (cd MultiNest/MultiNest_v3.11_CMake/multinest && mkdir build && cd build && cmake .. && make)
-
-ENV LD_LIBRARY_PATH $HOME/MultiNest/MultiNest_v3.11_CMake/multinest/lib:
-
-# Install Polychord
-RUN git clone https://github.com/PolyChord/PolyChordLite.git \
-&& (cd PolyChordLite && python setup.py --no-mpi install)
-
-# Install PTMCMCSampler
-RUN git clone https://github.com/jellis18/PTMCMCSampler.git \
-&& (cd PTMCMCSampler && python setup.py install)
-
-# Add the ROQ data to the image
-RUN mkdir roq_basis \
-    && cd roq_basis \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \
-    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \
diff --git a/containers/dockerfile-template b/containers/dockerfile-template
new file mode 100644
index 000000000..46d4f5470
--- /dev/null
+++ b/containers/dockerfile-template
@@ -0,0 +1,55 @@
+FROM continuumio/miniconda3
+LABEL name="bilby Base miniconda3" \
+maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
+
+RUN conda update -n base -c defaults conda
+
+ENV conda_env python{python_major_version}{python_minor_version}
+
+RUN conda create -n ${{conda_env}} python={python_major_version}.{python_minor_version}
+RUN echo "source activate ${{conda_env}}" > ~/.bashrc
+ENV PATH /opt/conda/envs/${{conda_env}}/bin:$PATH
+RUN /bin/bash -c "source activate ${{conda_env}}"
+RUN conda info
+RUN python --version
+
+# Install pymultinest requirements
+RUN apt-get update
+RUN apt-get install -y libblas3 libblas-dev liblapack3 liblapack-dev \
+libatlas3-base libatlas-base-dev cmake build-essential gfortran
+
+# Install conda-installable programs
+RUN conda install -n ${{conda_env}} -y matplotlib numpy scipy pandas astropy flake8 mock
+RUN conda install -n ${{conda_env}} -c anaconda coverage configargparse future
+
+# Install conda-forge-installable programs
+RUN conda install -n ${{conda_env}} -c conda-forge black ligo-gracedb gwpy lalsuite ligo.skymap pytest-cov deepdish arviz
+
+# Install pip-requirements
+RUN pip install --upgrade pip
+RUN pip install --upgrade setuptools coverage-badge
+
+# Install documentation requirements
+RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs autodoc
+
+# Install dependencies and samplers
+RUN pip install corner lalsuite theano healpy cython tables
+RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 kombine dnest4
+RUN conda install -n ${{conda_env}} -c conda-forge pymultinest ultranest
+
+# Install Polychord
+RUN git clone https://github.com/PolyChord/PolyChordLite.git \
+&& (cd PolyChordLite && python setup.py --no-mpi install)
+
+# Install PTMCMCSampler
+RUN git clone https://github.com/jellis18/PTMCMCSampler.git \
+&& (cd PTMCMCSampler && python setup.py install)
+
+# Add the ROQ data to the image
+RUN mkdir roq_basis \
+    && cd roq_basis \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \
+    && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat
diff --git a/containers/v2-dockerfile-test-suite-python35 b/containers/v2-dockerfile-test-suite-python35
index 1fc568b37..38d4c8b2f 100644
--- a/containers/v2-dockerfile-test-suite-python35
+++ b/containers/v2-dockerfile-test-suite-python35
@@ -1,7 +1,8 @@
+# This dockerfile is written automatically and should not be modified by hand.
+
 FROM continuumio/miniconda3
 LABEL name="bilby Base miniconda3" \
-maintainer="Gregory Ashton <gregory.ashton@ligo.org>" \
-date="20191114"
+maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
 
 RUN conda update -n base -c defaults conda
 
@@ -35,13 +36,8 @@ RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs autodoc
 
 # Install dependencies and samplers
 RUN pip install corner lalsuite theano healpy cython tables
-RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 pymultinest kombine ultranest dnest4
-
-# Install multinest
-RUN git clone https://github.com/farhanferoz/MultiNest.git \
-&& (cd MultiNest/MultiNest_v3.12_CMake/multinest && mkdir build && cd build && cmake .. && make)
-
-ENV LD_LIBRARY_PATH $HOME/MultiNest/MultiNest_v3.12_CMake/multinest/lib:
+RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 kombine dnest4
+RUN conda install -n ${conda_env} -c conda-forge pymultinest ultranest
 
 # Install Polychord
 RUN git clone https://github.com/PolyChord/PolyChordLite.git \
diff --git a/containers/v2-dockerfile-test-suite-python36 b/containers/v2-dockerfile-test-suite-python36
index 79deb07b1..ae75fff03 100644
--- a/containers/v2-dockerfile-test-suite-python36
+++ b/containers/v2-dockerfile-test-suite-python36
@@ -1,7 +1,8 @@
+# This dockerfile is written automatically and should not be modified by hand.
+
 FROM continuumio/miniconda3
 LABEL name="bilby Base miniconda3" \
-maintainer="Gregory Ashton <gregory.ashton@ligo.org>" \
-date="20191114"
+maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
 
 RUN conda update -n base -c defaults conda
 
@@ -35,13 +36,8 @@ RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs autodoc
 
 # Install dependencies and samplers
 RUN pip install corner lalsuite theano healpy cython tables
-RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 pymultinest kombine ultranest dnest4
-
-# Install multinest
-RUN git clone https://github.com/farhanferoz/MultiNest.git \
-&& (cd MultiNest/MultiNest_v3.12_CMake/multinest && mkdir build && cd build && cmake .. && make)
-
-ENV LD_LIBRARY_PATH $HOME/MultiNest/MultiNest_v3.12_CMake/multinest/lib:
+RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 kombine dnest4
+RUN conda install -n ${conda_env} -c conda-forge pymultinest ultranest
 
 # Install Polychord
 RUN git clone https://github.com/PolyChord/PolyChordLite.git \
diff --git a/containers/v2-dockerfile-test-suite-python37 b/containers/v2-dockerfile-test-suite-python37
index 81f953294..9e27fcab4 100644
--- a/containers/v2-dockerfile-test-suite-python37
+++ b/containers/v2-dockerfile-test-suite-python37
@@ -1,7 +1,8 @@
+# This dockerfile is written automatically and should not be modified by hand.
+
 FROM continuumio/miniconda3
 LABEL name="bilby Base miniconda3" \
-maintainer="Gregory Ashton <gregory.ashton@ligo.org>" \
-date="20191114"
+maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
 
 RUN conda update -n base -c defaults conda
 
@@ -35,13 +36,8 @@ RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs autodoc
 
 # Install dependencies and samplers
 RUN pip install corner lalsuite theano healpy cython tables
-RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 pymultinest kombine ultranest dnest4
-
-# Install multinest
-RUN git clone https://github.com/farhanferoz/MultiNest.git \
-&& (cd MultiNest/MultiNest_v3.12_CMake/multinest && mkdir build && cd build && cmake .. && make)
-
-ENV LD_LIBRARY_PATH $HOME/MultiNest/MultiNest_v3.12_CMake/multinest/lib:
+RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 kombine dnest4
+RUN conda install -n ${conda_env} -c conda-forge pymultinest ultranest
 
 # Install Polychord
 RUN git clone https://github.com/PolyChord/PolyChordLite.git \
diff --git a/containers/v2-dockerfile-test-suite-python38 b/containers/v2-dockerfile-test-suite-python38
index 4b3f15430..690055494 100644
--- a/containers/v2-dockerfile-test-suite-python38
+++ b/containers/v2-dockerfile-test-suite-python38
@@ -1,7 +1,8 @@
+# This dockerfile is written automatically and should not be modified by hand.
+
 FROM continuumio/miniconda3
 LABEL name="bilby Base miniconda3" \
-maintainer="Gregory Ashton <gregory.ashton@ligo.org>" \
-date="20191114"
+maintainer="Gregory Ashton <gregory.ashton@ligo.org>"
 
 RUN conda update -n base -c defaults conda
 
@@ -35,13 +36,8 @@ RUN pip install sphinx numpydoc nbsphinx sphinx_rtd_theme sphinx-tabs autodoc
 
 # Install dependencies and samplers
 RUN pip install corner lalsuite theano healpy cython tables
-RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 pymultinest kombine ultranest dnest4
-
-# Install multinest
-RUN git clone https://github.com/farhanferoz/MultiNest.git \
-&& (cd MultiNest/MultiNest_v3.12_CMake/multinest && mkdir build && cd build && cmake .. && make)
-
-ENV LD_LIBRARY_PATH $HOME/MultiNest/MultiNest_v3.12_CMake/multinest/lib:
+RUN pip install cpnest dynesty emcee nestle ptemcee pymc3 kombine dnest4
+RUN conda install -n ${conda_env} -c conda-forge pymultinest ultranest
 
 # Install Polychord
 RUN git clone https://github.com/PolyChord/PolyChordLite.git \
diff --git a/containers/write_dockerfiles.py b/containers/write_dockerfiles.py
new file mode 100644
index 000000000..75f2e969b
--- /dev/null
+++ b/containers/write_dockerfiles.py
@@ -0,0 +1,23 @@
+from datetime import date
+
+with open("dockerfile-template", "r") as ff:
+    template = ff.read()
+
+python_versions = [(3, 5), (3, 6), (3, 7), (3, 8)]
+today = date.today().strftime("%Y%m%d")
+
+for python_major_version, python_minor_version in python_versions:
+    with open(
+        "v2-dockerfile-test-suite-python"
+        f"{python_major_version}{python_minor_version}",
+        "w"
+    ) as ff:
+        ff.write(
+            "# This dockerfile is written automatically and should not be "
+            "modified by hand.\n\n"
+        )
+        ff.write(template.format(
+            date=today,
+            python_major_version=python_major_version,
+            python_minor_version=python_minor_version
+        ))
-- 
GitLab