Merge branch 'nightly-debian-packages' into 'master'

Nightly debian packages and other GitLab-CI/Docker improvements

Closes #61 and #62

See merge request !271

.Dockerfile-el7.gitlab-ci

+FROM ligo/base:el7
+
+LABEL name="LALSuite Nightly - EL7" \
+      maintainer="Adam Mercer <adam.mercer@ligo.org>" \
+      support="Reference Platform" \
+      date="20180506"
+
+# add RPMs to container
+COPY rpms /rpms
+
+# install RPMs & cleanup
+RUN yum makecache && \
+      yum -y localinstall /rpms/*.rpm && \
+      rm -rf /rpms && yum clean all

.Dockerfile-stretch.gitlab-ci

+FROM ligo/base:stretch
+
+LABEL name="LALSuite Nightly - Debian Stretch" \
+      maintainer="Adam Mercer <adam.mercer@ligo.org>" \
+      support="Reference Platform" \
+      date="20180506"
+
+# add debian packages to container
+COPY debs /debs
+
+# install debs & cleanup
+RUN apt-get update && \
+      dpkg -i /debs/*.deb || apt-get --assume-yes -f install && \
+      rm -rf /debs && rm -rf /var/lib/apts/lists/*

.gitlab-ci.yml

@@ -15,6 +15,7 @@ stages:
   - level2
   - level3
   - level4
+  - docker
   - nightly
   - wheels
   - deploy
@@ -26,13 +27,14 @@ before_script:
   - export CCACHE_DIR=${PWD}/ccache
   - export PKG_CONFIG_PATH=${LAL_DIR}/lib/pkgconfig
   - mkdir -p opt/lalsuite
-  - if [ ${CI_PIPELINE_SOURCE} = schedule ]; then EXTRA_CONFIG_FLAGS="--enable-nightly"; fi
+  - if [ ${CI_PIPELINE_SOURCE} = "schedule" ] || [ ${CI_PIPELINE_SOURCE} = "web" ]; then EXTRA_CONFIG_FLAGS="--enable-nightly"; fi
 
 cache:
   key: $CI_JOB_NAME
   paths:
     - ccache
 
+# FIXME: remove schedules after package builds have been running for a while
 .levelN:package: &levelN-package
   script:
     - cd ${CI_JOB_NAME#level?:*:}
@@ -56,7 +58,6 @@ cache:
   only:
     - pushes
     - schedules
-    - web
 
 .levelN:package:stretch: &levelN-package-stretch
   image: ligo/lalsuite-dev:stretch
@@ -220,9 +221,9 @@ level4:el7:lalapps:
     - level3:el7:laldetchar
     - level3:el7:lalinference
 
-# release a debian docker image upon every tag in lalsuite
+# FIXME: remove after package builds have been running for a while
 docker:tags:
-  stage: nightly
+  stage: docker
   before_script: []
   script:
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
@@ -233,9 +234,9 @@ docker:tags:
   only:
     - tags
 
-# release a debian docker image of each nightly build
+# FIXME: remove after package builds have been running for a while
 docker:nightly:
-  stage: nightly
+  stage: docker
   before_script: []
   script:
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
@@ -503,7 +504,7 @@ nightly:wheel:cp36-cp36m-macosx:
     - rpm -ivh rpmbuild/RPMS/x86_64/lal*.rpm || true
     - cd ${CI_JOB_NAME#level?:rpm:}
     - ./00boot
-    - ./configure --enable-swig --enable-nightly
+    - ./configure --enable-swig ${EXTRA_CONFIG_FLAGS}
     - make dist
     - rpmbuild -tb --define "_topdir $CI_PROJECT_DIR/rpmbuild" ${CI_JOB_NAME#level?:rpm:}-*.tar.xz
   artifacts:
@@ -512,6 +513,7 @@ nightly:wheel:cp36-cp36m-macosx:
       - rpmbuild/RPMS/x86_64/${CI_JOB_NAME#level?:rpm:}-*.rpm
   only:
     - schedules
+    - tags
     - web
 
 level0:rpm:lal:
@@ -607,6 +609,249 @@ level4:rpm:lalapps:
     - level2:rpm:lalstochastic
     - level3:rpm:lalinference
 
+# build a nightly container from the RPMs
+docker:el7:nightly:
+  stage: docker
+  before_script: []
+  script:
+    # add RPMs to directory to pass to docker
+    - mkdir rpms && mv rpmbuild/RPMS/x86_64/*.rpm rpms
+    - rm -rf rpmbuild*
+    # build container and push to registry
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - docker build --pull -t $CI_REGISTRY_IMAGE/el7:nightly --file .Dockerfile-el7.gitlab-ci .
+    - docker push $CI_REGISTRY_IMAGE/el7:nightly
+  dependencies:
+    - level0:rpm:lal
+    - level1:rpm:lalframe
+    - level1:rpm:lalmetaio
+    - level1:rpm:lalsimulation
+    - level1:rpm:lalxml
+    - level2:rpm:lalburst
+    - level2:rpm:lalinspiral
+    - level2:rpm:lalpulsar
+    - level2:rpm:lalstochastic
+    - level3:rpm:laldetchar
+    - level3:rpm:lalinference
+    - level4:rpm:lalapps
+  only:
+    - schedules
+
+# build a tagged container from the RPMs
+docker:el7:tags:
+  stage: docker
+  before_script: []
+  script:
+    # add RPMs to directory to pass to docker
+    - mkdir rpms && mv rpmbuild/RPMS/x86_64/*.rpm rpms
+    - rm -rf rpmbuild*
+    # build container and push to registry
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - docker build --pull -t $CI_REGISTRY_IMAGE/el7:$CI_COMMIT_TAG --file .Dockerfile-el7.gitlab-ci .
+    - docker push $CI_REGISTRY_IMAGE/el7:$CI_COMMIT_TAG
+  dependencies:
+    - level0:rpm:lal
+    - level1:rpm:lalframe
+    - level1:rpm:lalmetaio
+    - level1:rpm:lalsimulation
+    - level1:rpm:lalxml
+    - level2:rpm:lalburst
+    - level2:rpm:lalinspiral
+    - level2:rpm:lalpulsar
+    - level2:rpm:lalstochastic
+    - level3:rpm:laldetchar
+    - level3:rpm:lalinference
+    - level4:rpm:lalapps
+  only:
+    - tags
+
+.levelN:deb: &levelN-deb-package
+  image: ligo/lalsuite-dev:stretch
+  script:
+    # install debs from artifacts
+    - dpkg -i *.deb || true
+    # make dist tarball
+    - cd ${CI_JOB_NAME#level?:deb:}
+    - ./00boot
+    - ./configure --enable-swig ${EXTRA_CONFIG_FLAGS}
+    - make dist
+    # create orig tarball
+    - PACKAGE=$(echo ${CI_JOB_NAME} | sed 's/.*://')
+    - TARBALL="${PACKAGE}-*.tar.*"
+    - SUFFIX=$(echo $TARBALL | sed 's/.*\.\(tar\..*\)/\1/')
+    - VERSION=$(echo $TARBALL | sed 's/[^-]*-\(.*\)\.tar\..*/\1/' | tr '-' '~')
+    - cd ${CI_PROJECT_DIR}
+    - cp ${CI_JOB_NAME#level?:deb:}/${TARBALL} ${PACKAGE}_${VERSION}.orig.${SUFFIX}
+    # update changelog
+    - export DEBFULLNAME="GitLab"
+    - export DEBEMAIL="gitlab@git.ligo.org"
+    - tar xf ${CI_JOB_NAME#level?:deb:}/${TARBALL}
+    - cd ${PACKAGE}-*
+    - dch -v ${VERSION}-1 'Rebuilt automatically on git.ligo.org CI'
+    # build packages
+    - debuild -us -uc -r
+  artifacts:
+    expire_in: 3h
+    paths:
+      - ${CI_JOB_NAME#level?:deb:}*.changes
+      - ${CI_JOB_NAME#level?:deb:}*.deb
+      - python*-${CI_JOB_NAME#level?:deb:}*.deb
+      - ${CI_JOB_NAME#level?:deb:}*.dsc
+      - ${CI_JOB_NAME#level?:deb:}*.orig.*
+  only:
+    - schedules
+    - tags
+    - web
+
+level0:deb:lal:
+  <<: *levelN-deb-package
+  stage: level0
+
+level1:deb:lalframe:
+  <<: *levelN-deb-package
+  stage: level1
+  dependencies:
+    - level0:deb:lal
+
+level1:deb:lalmetaio:
+  <<: *levelN-deb-package
+  stage: level1
+  dependencies:
+    - level0:deb:lal
+
+level1:deb:lalsimulation:
+  <<: *levelN-deb-package
+  stage: level1
+  dependencies:
+    - level0:deb:lal
+
+level1:deb:lalxml:
+  <<: *levelN-deb-package
+  stage: level1
+  dependencies:
+    - level0:deb:lal
+
+level2:deb:lalburst:
+  <<: *levelN-deb-package
+  stage: level2
+  dependencies:
+    - level0:deb:lal
+    - level1:deb:lalmetaio
+    - level1:deb:lalsimulation
+
+level2:deb:lalinspiral:
+  <<: *levelN-deb-package
+  stage: level2
+  dependencies:
+    - level0:deb:lal
+    - level1:deb:lalframe
+    - level1:deb:lalmetaio
+    - level1:deb:lalsimulation
+
+level2:deb:lalpulsar:
+  <<: *levelN-deb-package
+  stage: level2
+  dependencies:
+    - level0:deb:lal
+
+level2:deb:lalstochastic:
+  <<: *levelN-deb-package
+  stage: level2
+  dependencies:
+   - level0:deb:lal
+   - level1:deb:lalmetaio
+
+level3:deb:laldetchar:
+  <<: *levelN-deb-package
+  stage: level3
+  dependencies:
+    - level0:deb:lal
+    - level1:deb:lalmetaio
+    - level1:deb:lalsimulation
+    - level2:deb:lalburst
+
+level3:deb:lalinference:
+  <<: *levelN-deb-package
+  stage: level3
+  dependencies:
+    - level0:deb:lal
+    - level1:deb:lalframe
+    - level1:deb:lalmetaio
+    - level1:deb:lalsimulation
+    - level2:deb:lalburst
+    - level2:deb:lalinspiral
+    - level2:deb:lalpulsar
+
+level4:deb:lalapps:
+  <<: *levelN-deb-package
+  stage: level4
+  dependencies:
+    - level0:deb:lal
+    - level1:deb:lalframe
+    - level1:deb:lalmetaio
+    - level1:deb:lalsimulation
+    - level2:deb:lalburst
+    - level2:deb:lalinspiral
+    - level2:deb:lalpulsar
+    - level2:deb:lalstochastic
+    - level3:deb:lalinference
+
+# build a nightly container from the debian packages
+docker:stretch:nightly:
+  stage: docker
+  before_script: []
+  script:
+    # add deb packages to directory to pass to docker
+    - mkdir debs && mv *.deb debs
+    - rm *.changes *.dsc *.orig.*
+    # build container and push to registry
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - docker build --pull -t $CI_REGISTRY_IMAGE/stretch:nightly --file .Dockerfile-stretch.gitlab-ci .
+    - docker push $CI_REGISTRY_IMAGE/stretch:nightly
+  dependencies:
+    - level0:deb:lal
+    - level1:deb:lalframe
+    - level1:deb:lalmetaio
+    - level1:deb:lalsimulation
+    - level1:deb:lalxml
+    - level2:deb:lalburst
+    - level2:deb:lalinspiral
+    - level2:deb:lalpulsar
+    - level2:deb:lalstochastic
+    - level3:deb:laldetchar
+    - level3:deb:lalinference
+    - level4:deb:lalapps
+  only:
+    - schedules
+
+# build a tagged container from the debian packages
+docker:stretch:tags:
+  stage: docker
+  before_script: []
+  script:
+    # add deb packages to directory to pass to docker
+    - mkdir debs && mv *.deb debs
+    - rm *.changes *.dsc *.orig.*
+    # build container and push to registry
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - docker build --pull -t $CI_REGISTRY_IMAGE/stretch:$CI_COMMIT_TAG --file .Dockerfile-stretch.gitlab-ci .
+    - docker push $CI_REGISTRY_IMAGE/stretch:$CI_COMMIT_TAG
+  dependencies:
+    - level0:deb:lal
+    - level1:deb:lalframe
+    - level1:deb:lalmetaio
+    - level1:deb:lalsimulation
+    - level1:deb:lalxml
+    - level2:deb:lalburst
+    - level2:deb:lalinspiral
+    - level2:deb:lalpulsar
+    - level2:deb:lalstochastic
+    - level3:deb:laldetchar
+    - level3:deb:lalinference
+    - level4:deb:lalapps
+  only:
+    - tags
+
 pages:
   image: ligo/lalsuite-dev:stretch
   stage: nightly

gnuscripts/debian/lalsuite.mk

@@ -22,7 +22,7 @@ export PYBUILD_CONFIGURE_ARGS = cd {build_dir} && {dir}/configure \
 	$(CONFIGUREARGS) PYTHON=$$(which {interpreter}) pythondir={install_dir}
 export PYBUILD_BUILD_ARGS = $(MAKE) $(MAKEARGS)
 export PYBUILD_INSTALL_ARGS = $(MAKE) $(MAKEARGS) DESTDIR={destdir} install
-export PYBUILD_TEST_ARGS = $(MAKE) $(MAKEARGS) check
+export PYBUILD_TEST_ARGS = $(MAKE) $(MAKEARGS) VERBOSE=1 check
 
 %:
 	dh $@ \

lal/debian/control.in

@@ -2,6 +2,7 @@ Source: lal
 Section: science
 Priority: optional
 Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: bc,
   debhelper (>= 9),
   dh-python,

lalapps/debian/control.in

@@ -2,12 +2,13 @@ Source: lalapps
 Section: science
 Priority: optional
 Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: bc,
   debhelper (>= 9),
   dh-python,
   help2man,
   less,
-  libcfitsio3-dev,
+  libcfitsio3-dev | libcfitsio-dev,
   libfftw3-dev,
   libframe-dev (>= 8.0),
   libgsl-dev | libgsl0-dev (>= 1.9),

lalburst/debian/control.in

 Source: lalburst
 Section: science
 Priority: optional
-Maintainer: Adam Mercer <adam.mercer@ligo.org>
+Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
   libgsl-dev | libgsl0-dev (>= 1.9),

laldetchar/debian/control.in

 Source: laldetchar
 Section: science
 Priority: optional
-Maintainer: Adam Mercer <adam.mercer@ligo.org>
+Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
   help2man,

lalframe/debian/control.in

 Source: lalframe
 Section: science
 Priority: optional
-Maintainer: Adam Mercer <adam.mercer@ligo.org>
+Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
   ldas-tools-framecpp-c-dev (>= @MIN_FRAMECPPC_VERSION@),

lalinference/debian/control.in

 Source: lalinference
 Section: science
 Priority: optional
-Maintainer: Adam Mercer <adam.mercer@ligo.org>
+Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
   help2man,
   libchealpix-dev (>= 3.30.0),
-  libgsl0-dev (>= 1.15),
+  libgsl-dev | libgsl0-dev (>= 1.15),
   libhdf5-dev,
   libmetaio-dev (>= 8.2),
   liboctave-dev,
@@ -72,7 +73,7 @@ Architecture: any
 Depends: ${misc:Depends},
   ${shlibs:Depends},
   libchealpix-dev (>= 3.30.0),
-  libgsl0-dev (>= 1.15),
+  libgsl-dev | libgsl0-dev (>= 1.15),
   libhdf5-dev,
   libmetaio-dev (>= 8.2),
   python-healpy (>= 1.9.1),

lalinspiral/debian/control.in

 Source: lalinspiral
 Section: science
 Priority: optional
-Maintainer: Adam Mercer <adam.mercer@ligo.org>
+Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
   libframe-dev (>= 8.0),

lalmetaio/debian/control.in

 Source: lalmetaio
 Section: science
 Priority: optional
-Maintainer: Adam Mercer <adam.mercer@ligo.org>
+Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
   libmetaio-dev (>= 8.2),

lalpulsar/debian/control.in

 Source: lalpulsar
 Section: science
 Priority: optional
-Maintainer: Adam Mercer <adam.mercer@ligo.org>
+Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
-  libcfitsio3-dev,
+  libcfitsio3-dev | libcfitsio-dev,
   libfftw3-dev,
   libgsl-dev | libgsl0-dev (>= 1.9),
   liboctave-dev,
@@ -37,7 +38,7 @@ Section: libdevel
 Architecture: any
 Depends: ${misc:Depends},
   ${shlibs:Depends},
-  libcfitsio3-dev,
+  libcfitsio3-dev | libcfitsio-dev,
   libfftw3-dev,
   libgsl-dev | libgsl0-dev (>= 1.9),
   lal-dev (>= @MIN_LAL_VERSION@~),

lalsimulation/debian/control.in

 Source: lalsimulation
 Section: science
 Priority: optional
-Maintainer: Adam Mercer <adam.mercer@ligo.org>
+Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
   help2man,

lalstochastic/debian/control.in

@@ -2,6 +2,7 @@ Source: lalstochastic
 Section: science
 Priority: optional
 Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
   libmetaio-dev (>= 8.2),

lalxml/debian/control.in

 Source: lalxml
 Section: science
 Priority: optional
-Maintainer: Adam Mercer <adam.mercer@ligo.org>
+Maintainer: Steffen Grunewald <steffen.grunewald@aei.mpg.de>
+Uploaders: Adam Mercer <adam.mercer@ligo.org>, GitLab <gitlab@git.ligo.org>
 Build-Depends: debhelper (>= 9),
   dh-python,
   libgsl-dev | libgsl0-dev (>= 1.9),