Merge branch 'lalsuite-image-version-bump' into 'master'

Overhaul continuous integration and docker images See merge request !128

Merge branch 'lalsuite-image-version-bump' into 'master'
21264a6b · James Alexander Clark PhD · 47b95372 · a678847e · 21264a6b · 21264a6b
Commit 21264a6b authored 5 years ago by James Alexander Clark PhD
--- a/.dependencies-el7.Dockerfile
+++ b/.dependencies-el7.Dockerfile
+FROM ligo/base:el7
+
+LABEL name="BayesWave Build Dependencies - EL7" \
+      maintainer="James Alexander Clark <james.clark@ligo.org>" \
+      support="Base image for buillding BayesWave " \
+      date="20190726"
+
+# Yum dependencies
+RUN yum upgrade -y && \
+      yum install -y cmake3 \
+      gcc \
+      gcc-c++ \
+      git \
+      help2man \
+      lalapps \
+      lal-devel \
+      lalframe-devel \
+      lalinference-devel \
+      lalsimulation-devel \
+      python-devel \
+      python-ligo-lw \
+      rpm-build && \
+      yum clean all && \
+      rm -rf /var/cache/yum
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,6 +3,7 @@ variables:
  COMMIT: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
  NIGHTLY: $CI_REGISTRY_IMAGE:nightly
  TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
+  DEPENDS_IMAGE: $CI_REGISTRY_IMAGE/dependencies
  BUILD_DIR: test-install
  BUILD_TARGET: $CI_PROJECT_DIR/$BUILD_DIR
  TEST_OUTPUT: test-output
@@ -19,12 +20,13 @@ build-env:
    - mkdir -p $BUILD_TARGET
    - sed "s|INSTALL_DIR|$BUILD_DIR|g" $CI_PROJECT_DIR/etc/bayeswave-user-env.sh > $BUILD_TARGET/bayeswave-user-env.sh
  artifacts:
+    expire_in: 6h
    paths:
      - $BUILD_DIR

 build-bayeswave:
  stage: build
-  image: ligo/software:el7
+  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
  script:
    - pushd $CI_PROJECT_DIR
    - mkdir -p build
@@ -33,17 +35,36 @@ build-bayeswave:
    - cmake3 --build . --target install
    - popd
  artifacts:
+    expire_in: 6h
    paths:
      - $BUILD_DIR

+build-bayeswave-rpm:
+  stage: build
+  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
+  script:
+    - pushd $CI_PROJECT_DIR
+    - mkdir -p $CI_PROJECT_DIR/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,RPMS}
+    - echo '%_topdir %(echo $CI_PROJECT_DIR)/rpmbuild' > ~/.rpmmacros
+    - cmake3 . -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=true 
+    - cmake3 --build . --target package_source
+    - mv bayeswave-*.tar.xz $CI_PROJECT_DIR/rpmbuild/SOURCES
+    - rpmbuild  -ba bayeswave.spec
+    - popd
+  artifacts:
+    expire_in: 6h
+    paths:
+      - $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm
+
 build-BayesWaveUtils:
  stage: build
-  image: ligo/software:el7
+  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
  script: 
    - pushd BayesWaveUtils
    - python setup.py install --prefix $BUILD_TARGET
    - popd
  artifacts:
+    expire_in: 6h
    paths:
      - $BUILD_DIR

@@ -51,7 +72,7 @@ build-BayesWaveUtils:
 # script in the repository that does the same thing.
 test:BayesWave:
  stage: test
-  image: ligo/software:el7
+  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
  script:
    - source $BUILD_DIR/bayeswave-user-env.sh
    - cat $BUILD_DIR/bayeswave-user-env.sh
@@ -68,7 +89,7 @@ test:BayesWave:

 test:BayesWavePost:
  stage: test
-  image: ligo/software:el7
+  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
  script:
    - source $BUILD_DIR/bayeswave-user-env.sh
    - BayesWavePost --help
@@ -78,7 +99,7 @@ test:BayesWavePost:

 test:bayeswave_pipe:
  stage: test
-  image: ligo/software:el7
+  image: containers.ligo.org/lscsoft/bayeswave/dependencies:el7
  script:
    - ls $BUILD_DIR
    - source $BUILD_DIR/bayeswave-user-env.sh
@@ -87,16 +108,25 @@ test:bayeswave_pipe:
    - build-env
    - build-BayesWaveUtils

-test:docker:
-  stage: test
+docker:nightly:
+  # Update the dependency image and bayeswave every night
+  stage: docker
  image: docker:latest
+  only:
+    - schedules
  before_script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
  script:
-    - docker build --pull -t $CI_REGISTRY_IMAGE:latest --file Dockerfile .
-    - docker run $CI_REGISTRY_IMAGE:latest /test-bayeswave.sh
-
-# FIXME: Add a proper test stage for the docker images
+    # Build and push dependency image
+    - docker build --pull -t $DEPENDS_IMAGE:el7 --file .dependencies-el7.Dockerfile .
+    - docker push $DEPENDS_IMAGE:el7
+    - mkdir rpms && mv $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm rpms
+    # Build, test and push bayeswave image
+    - docker build --pull -t $CI_REGISTRY_IMAGE:nightly --file Dockerfile .
+    - docker run $CI_REGISTRY_IMAGE:nightly /test-bayeswave.sh
+    - docker push $CI_REGISTRY_IMAGE:nightly
+  dependencies:
+    - build-bayeswave-rpm

 docker:latest:
  stage: docker
@@ -107,11 +137,18 @@ docker:latest:
  before_script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
  script:
+    - mkdir rpms && mv $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm rpms
    - docker build --pull -t $CI_REGISTRY_IMAGE:latest --file Dockerfile .
+    # Build, test and push bayeswave image
+    - docker run $CI_REGISTRY_IMAGE:latest /test-bayeswave.sh
    - docker push $CI_REGISTRY_IMAGE:latest
-
+  dependencies:
+    - build-bayeswave-rpm

 docker:tags:
+  # FIXME: Until we version the dependency image, the tags containers will
+  # always use the nightly build of the dependency image, which always has the
+  # latest release of the dependencies.
  stage: docker
  image: docker:latest
  only:
@@ -120,8 +157,13 @@ docker:tags:
  before_script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
  script:
+    - mkdir rpms && mv $CI_PROJECT_DIR/rpmbuild/RPMS/x86_64/bayeswave-*.el7.x86_64.rpm rpms
    - docker build --pull -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG --file Dockerfile .
+    # Build, test and push bayeswave image
+    - docker run $CI_REGISTRY_IMAGE:latest /test-bayeswave.sh
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
+  dependencies:
+    - build-bayeswave-rpm

 docs:
  stage: build
@@ -139,6 +181,7 @@ docs:
  script:
    - bash -ex doc/build.sh
  artifacts:
+    expire_in: 6h
    paths:
      - doc/_build
  cache:
@@ -154,5 +197,6 @@ pages:
  script:
    - mv doc/_build/html public
  artifacts:
+    expire_in: 6h
    paths:
      - public
--- a/Dockerfile
+++ b/Dockerfile
-FROM containers.ligo.org/lscsoft/lalsuite/lalsuite-v6.53:el7
-ARG version
-RUN echo "Building bayeswave"
-MAINTAINER James Alexander Clark <james.clark@ligo.org>
+FROM containers.ligo.org/lscsoft/bayeswave/dependencies:el7

-RUN yum upgrade -y && \
-      yum install -y gcc \
-      cmake3 \
-      gcc-c++ \
-      help2man \
-      rpm-build \
-      which \
-      git \
-      python-ligo-lw && \
-      yum clean all && \
-      rm -rf /var/cache/yum
-
-# RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
-#     python get-pip.py && \
-#     rm get-pip.py
-
-# Copy and build BayesWave
-WORKDIR /tmp
-RUN mkdir /tmp/bayeswave
-COPY bayeswave.spec.in /tmp/bayeswave
-COPY .git /tmp/bayeswave
-COPY CMakeLists.txt /tmp/bayeswave
-COPY src /tmp/bayeswave/src
-COPY etc/bayeswave-user-env.sh /tmp/bayeswave/etc/bayeswave-user-env.sh
-COPY README.md /tmp
-COPY BayesWaveUtils /tmp/bayeswave/BayesWaveUtils
-ADD test/test-bayeswave.sh /
-
-# Python Utils
-RUN cd /tmp/bayeswave/BayesWaveUtils && \
-      python /tmp/bayeswave/BayesWaveUtils/setup.py install 
-
-RUN mkdir -p ~/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
-RUN echo '%_topdir %(echo $HOME)/rpmbuild' > ~/.rpmmacros
-RUN mkdir dist && \
-      pushd dist && \
-      cmake3 /tmp/bayeswave && \
-      cmake3 --build . --target package_source && \
-      mv bayeswave-*.tar.xz /root/rpmbuild/SOURCES/ && \
-      popd
-RUN rpmbuild -ba /tmp/bayeswave/bayeswave.spec
-RUN rpm -ivh /root/rpmbuild/RPMS/x86_64/bayeswave-1.0.3-1.el7.x86_64.rpm
+LABEL name="BayesWave - EL7" \
+      maintainer="James Alexander Clark <james.clark@ligo.org>" \
+      support="Reference Platform" \
+      date="20190726"

-RUN rm -rf .git install.sh /src /BayesWaveUtils /etc/bayeswave-user-env.sh /build.sh /CMakeLists.txt /build
+# Directories we may want to bind
 RUN mkdir -p /cvmfs /hdfs /hadoop /etc/condor /test

-# Clean up
-RUN rm -rf /tmp/* ~/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
+# Build and install from RPM built in CI
+COPY rpms /rpms
+RUN  yum upgrade -y && \
+      yum -y localinstall /rpms/*.rpm && \
+      rm -rf /rpms && yum clean all

+# Python Utils
+COPY test/test-bayeswave.sh /test-bayeswave.sh
+COPY BayesWaveUtils /tmp/BayesWaveUtils
+RUN cd /tmp/BayesWaveUtils && \
+      python /tmp/BayesWaveUtils/setup.py install 
+RUN rm -rf /tmp/*
+
+# Remove build dependencies
+RUN yum remove -y cmake3 \
+      gcc \
+      gcc-c++ \
+      help2man \
+      lal-devel \
+      lalframe-devel \
+      lalinference-devel \
+      lalsimulation-devel \
+      rpm-build && \
+      yum clean all && \
+      rm -rf /var/cache/yum

 WORKDIR /
 ENTRYPOINT ["/bin/bash"]
-
-
-
-