.gitattributes

+*.gwf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -28,4 +28,8 @@ configure
 */gnuscripts/missing
 */gnuscripts/py-compile
 */gnuscripts/test-driver
+*/tests/*.log
+*/tests/*.trs
 libtool
+.vscode
+.idea

.gitlab-ci.Dockerfile.e7-testing

+# This is a Dockerfile to build an EL7 MKL-optimized GstLAL 
+# runtime container. 
+
+FROM containers.ligo.org/alexander.pace/gstlal-dev/gstlal-dev:el7-testing
+
+# Labeling/packaging stuff:
+LABEL name="GstLAL Runtime Package, EL7 Testing" \
+      maintainer="Alexander Pace <alexander.pace@ligo.org>" \
+      date="2020-05-06" \
+      support="Reference Platform, EL7 (Testing)"
+
+USER root
+
+## Copy Optimized RPMs to container
+COPY rpms /rpms
+
+# Install Optimized RPMs, delete old RPMs
+RUN GLOBIGNORE="*calibration*" && \
+      yum makecache && \
+      yum -y localinstall /rpms/*.rpm && \
+      unset GLOBIGNORE
+
+# Update gstreamer registry cache:
+ENV GST_REGISTRY_1_0=/usr/share/gstreamer-1.0/.registry.bin
+RUN gst-inspect-1.0
+
+# Clean up and close-out
+
+#RUN rm -rf /rpms && \
+RUN yum clean all
+
+# Export MKL environment variables: 
+
+ENV MKL_INTERFACE_LAYER LP64
+ENV MKL_THREADING_LAYER SEQUENTIAL
+ENV GSTLAL_FIR_WHITEN 0
+ENV TMPDIR /tmp
+
+
+ENTRYPOINT bash
+

.gitlab-ci.Dockerfile.rl8

+ARG lalsuite_version_tag
+FROM containers.ligo.org/gstlal/gstlal-dev:$lalsuite_version_tag
+
+# Labeling/packaging stuff:
+LABEL name="GstLAL Runtime Package, RL8" \
+      maintainer="Alexander Pace <alexander.pace@ligo.org>" \
+      date="2022-05-03" \
+      support="Reference Platform, RL8"
+
+USER root
+
+## Copy Optimized RPMs to container
+COPY rpms /rpms
+
+# Install Optimized RPMs, delete old RPMs
+RUN sed -i "s|http://|https://|" /etc/yum.repos.d/*.repo
+RUN yum makecache && \
+    yum -y localinstall /rpms/*.rpm 
+
+# Update gstreamer registry cache:
+ENV GST_REGISTRY_1_0=/usr/share/gstreamer-1.0/.registry.bin
+RUN gst-inspect-1.0
+
+# Clean up and close-out
+
+RUN rm -rf /rpms && \
+    yum clean all
+
+# Export MKL environment variables: 
+
+ENV MKL_INTERFACE_LAYER LP64
+ENV MKL_THREADING_LAYER SEQUENTIAL
+ENV GSTLAL_FIR_WHITEN 0
+ENV TMPDIR /tmp
+
+ENTRYPOINT bash
+

.gitlab-ci.yml

 image: docker:latest
 
 variables:
+
+  # Set the gstlal-dev version tag, which corresponds to a tagged 
+  # version of lalsuite. A list of available tags is available in the
+  # gstlal-dev container registry:
+  #
+  # https://git.ligo.org/gstlal/gstlal-dev/container_registry/809
+  LALSUITE_VERSION_TAG: "lalsuite-master-x86_64"
+
+  # Docker pipeline variables:
   DOCKER_DRIVER: overlay
+  DOCKER_BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  DOCKER_LATEST: $CI_REGISTRY_IMAGE:latest
   BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
   COMMIT: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
   NIGHTLY: $CI_REGISTRY_IMAGE:nightly
   TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
-  GSTLAL_DIR: $CI_PROJECT_DIR/opt/gstlal
+
+  # don't need git history
+  GIT_DEPTH: 1
+
+  # GstLAL pipeline-specific runtime variables. 
+  TMPDIR: /tmp
+  GSTLAL_FIR_WHITEN: 0
 
 before_script:
-    # Set up directory structure and copy over built-dependencies from container:
-    - mkdir public
-    - mkdir -p ${GSTLAL_DIR}
-    
-    # Update apt info
-    - apt update
-    
-    # Define GstLAL build parameters:
-    - export PATH="${GSTLAL_DIR}/bin:/usr/lib/ccache:/opt/local/libexec/ccache:${PATH}"
-    - export LD_LIBRARY_PATH="${GSTLAL_DIR}/lib:${GSTLAL_DIR}/lib64:${LD_LIBRARY_PATH}"
-    - export LIBRARY_PATH="${GSTLAL_DIR}/lib:${GSTLAL_DIR}/lib64:${LIBRARY_PATH}"
-    - export PKG_CONFIG_PATH="${GSTLAL_DIR}/lib/pkgconfig:${GSTLAL_DIR}/lib64/pkgconfig:${PKG_CONFIG_PATH}"
-    - export GI_TYPELIB_PATH="${GSTLAL_DIR}/lib/girepository-1.0:${GSTLAL_DIR}/lib64/girepository-1.0:${GI_TYPELIB_PATH}"
-    - export GST_PLUGIN_PATH="${GSTLAL_DIR}/lib/gstreamer-1.0:${GSTLAL_DIR}/lib64/gstreamer-1.0:${GST_PLUGIN_PATH}"
-    - export PYTHONPATH="${GSTLAL_DIR}/lib64/python2.7/site-packages:${GSTLAL_DIR}/lib/python2.7/site-packages:${PYTHONPATH}"
-    - export GST_REGISTRY_1_0="${GSTLAL_DIR}/registry.bin"
-    - export CCACHE_DIR=${PWD}/ccache
-    
-    # Define MKL environment variables:
-    - export MKLROOT="/opt/intel/compilers_and_libraries_2018.3.222/linux/mkl"
-    - export LIBRARY_PATH="/opt/intel/compilers_and_libraries_2018.3.222/linux/tbb/lib/intel64_lin/gcc4.7:/opt/intel/compilers_and_libraries_2018.3.222/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.3.222/linux/mkl/lib/intel64_lin"
-    - export CPATH="/opt/intel/compilers_and_libraries_2018.3.222/linux/mkl/include"
-    - export PATH=PATH="/opt/intel/compilers_and_libraries_2018.3.222/linux/bin/intel64:${PATH}"
-    - export PKG_CONFIG_PATH="/opt/intel/compilers_and_libraries_2018.3.222/linux/mkl/bin/pkgconfig:${PKG_CONFIG_PATH}"
-    - export LD_LIBRARY_PATH="/opt/intel/compilers_and_libraries_2018.3.222/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.3.222/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.3.222/linux/tbb/lib/intel64_lin/gcc4.7:/opt/intel/compilers_and_libraries_2018.3.222/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.3.222/linux/mkl/lib/intel64_lin"
-    
-    # Define build and linking parameters:
-    - export CFLAGS="-O3 -fPIC -DMKL_ILP64 -m64 -I${MKLROOT}/include -I${GSTLAL_DIR}/include"
-    - export LDFLAGS="-lfftw3 -lfftw3f -lfftw3_threads -lfftw3f_threads -L${GSTLAL_DIR}/lib -L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_ilp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl"
-    - export LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libfftw3.so:/usr/lib/x86_64-linux-gnu/libfftw3f.so"
-    
-    - export TMPDIR=/tmp
-    
-    
+  - sed -i 's|http://|https://|' /etc/yum.repos.d/*.repo
+  - if [ -d ${CI_PROJECT_DIR}/rpmbuild/RPMS ]; then yum -y install ${CI_PROJECT_DIR}/rpmbuild/RPMS/*/*.rpm; fi
+
 cache:
   key: $CI_JOB_NAME
   paths:
     - ccache
 
 stages:
-    - level0
-    - level1
-    - level2
-    - test-gstlal
-    - test-inspiral
-    - test-burst
-    - test-offline
-    - nightly-pages
-    - verify
-
-level0:gstlal:
-  image: aepace/gstlal-dev:stretch-mkl
-  stage: level0
+  - deps
+  - level0
+  - level1
+  - level2
+  - docker
+  - docker-latest
+  - test-gstlal
+  - test-gstlal-full-build
+  - test-gstlal-ugly
+  - test-burst
+  - test-inspiral
+  - test-offline
+  - docs
+  - nightly-pages
+
+
+#
+# build rpms
+#
+
+.levelN:rpm: &levelN-rpm-package
+  interruptible: true
+  image: containers.ligo.org/gstlal/gstlal-dev:${LALSUITE_VERSION_TAG}
+  variables:
+    GIT_STRATEGY: fetch
+    RPM_BUILD_CPUS: 4
   script:
-    #- echo "$PWD" > ./pwd_output.out
-    - cd gstlal
+    - if [ -d rpmbuild ]; then yum -y install rpmbuild/RPMS/x86_64/*.rpm; fi
+    - cd ${CI_JOB_NAME#level?:rpm:}
     - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
+    - ./configure --enable-gtk-doc $EXTRA_CONFIG_FLAGS
     - make
-    - make install
+    - make dist
+    # Install dependencies
+    - yum-builddep -y ${CI_JOB_NAME#level?:rpm:}.spec
+    - rpmbuild -tb -D "_topdir ${CI_PROJECT_DIR}/rpmbuild" -D 'debug_package %{nil}' -D 'build_cflags ${CFLAGS}' -D 'build_ldflags ${LDFLAGS}' ${CI_JOB_NAME#level?:rpm:}-*.tar.gz
   artifacts:
-    expire_in: 3h
+    expire_in: 18h
     paths:
-      - ${GSTLAL_DIR}
-      - gstlal
-    when: always
+      - ${CI_PROJECT_DIR}/rpmbuild/RPMS/
   only:
-    - pushes
     - schedules
-    
-level1:gstlal-ugly:
-  image: aepace/gstlal-dev:stretch-mkl
-  stage: level1
-  dependencies:
-    - level0:gstlal
-  script:
-    - cd gstlal-ugly
-    - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
-    - make
-    - make install
-  artifacts:
-    expire_in: 3h
-    paths:
-      - ${GSTLAL_DIR}
-      - gstlal-ugly
-    when: always
-  only:
+    - tags
     - pushes
-    - schedules
+    - web
+
+level0:rpm:gstlal:
+  <<: *levelN-rpm-package
+  stage: level0
+  needs: [ ]
 
-level2:gstlal-calibration:
-  image: aepace/gstlal-dev:stretch-mkl
+level1:rpm:gstlal-ugly:
+  <<: *levelN-rpm-package
+  stage: level1
+  needs:
+    - level0:rpm:gstlal
+
+level2:rpm:gstlal-inspiral:
+  <<: *levelN-rpm-package
   stage: level2
-  dependencies:
-    - level1:gstlal-ugly
-  script:
-    - cd gstlal-calibration
-    - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
-    - make
-    - make install
-  artifacts:
-    expire_in: 3h
-    paths:
-      - ${GSTLAL_DIR}
-      - gstlal-calibration
-    when: always
-  only:
-    - pushes
-    - schedules
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+  variables:
+    EXTRA_CONFIG_FLAGS: "--disable-massmodel"
 
-level2:gstlal-inspiral:
-  image: aepace/gstlal-dev:stretch-mkl
+level2:rpm:gstlal-burst:
+  <<: *levelN-rpm-package
   stage: level2
-  dependencies:
-    - level1:gstlal-ugly
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+
+# Docker Images
+
+.docker:rpm: &docker-rpm
+  interruptible: true
+  stage: docker
+  before_script: [ ]
   script:
-    - cd gstlal-inspiral
-    - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
-    - make
-    - make install
-  artifacts:
-    expire_in: 3h
-    paths:
-      - ${GSTLAL_DIR}
-      - gstlal-inspiral
-    when: always
+    # add RPMs to directory to pass to docker
+    - mkdir rpms
+
+    # Copy rpms to new container.
+    - mv rpmbuild/RPMS/x86_64/*.rpm rpms
+
+    # Clear out the old rpmbuild directory
+    - rm -rf rpmbuild*
+
+    # Build the container:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - docker build --pull -t $DOCKER_BRANCH --build-arg lalsuite_version_tag="${LALSUITE_VERSION_TAG}" --file .gitlab-ci.Dockerfile.rl8 .
+    - docker push $DOCKER_BRANCH
   only:
-    - pushes
     - schedules
-    
-level2:gstlal-burst:
-  image: aepace/gstlal-dev:stretch-mkl
-  stage: level2
-  dependencies:
-    - level1:gstlal-ugly
-  script:
-    - cd gstlal-burst
-    - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
-    - make
-    - make install
-  artifacts:
-    expire_in: 3h
-    paths:
-      - ${GSTLAL_DIR}
-      - gstlal-burst
-    when: always
-  only:
     - pushes
-    - schedules
-  
-gstlal-verify:
-  image: aepace/gstlal-dev:stretch-mkl
-  stage: verify
-  script:
-    # -------------------------------------------------------------------
-    #  gstlal
-    # -------------------------------------------------------------------
-    #- echo "$PWD" > ./pwd_output.out
-    - cd gstlal
-    - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
-    - make
-    - make install
-    - cd ..
-    # -------------------------------------------------------------------
-    #  gstlal-ugly
-    # -------------------------------------------------------------------
-    - cd gstlal-ugly
-    - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
-    - make
-    - make install
-    - cd ..
-    # -------------------------------------------------------------------
-    #  gstlal-calibration
-    # -------------------------------------------------------------------
-    - cd gstlal-calibration
-    - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
-    - make
-    - make install
-    - cd ..
-    # -------------------------------------------------------------------
-    #  gstlal-inspiral
-    # -------------------------------------------------------------------
-    - cd gstlal-inspiral
-    - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
-    - make
-    - make install
-    - cd ..
-    # -------------------------------------------------------------------
-    #  gstlal-burst
-    # -------------------------------------------------------------------
-    - cd gstlal-burst
-    - ./00init.sh
-    - ./configure --prefix=${GSTLAL_DIR}
-    - make
-    - make install
-    - cd ..
+
+docker:rl8:
+  <<: *docker-rpm
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+    - level2:rpm:gstlal-inspiral
+    - level2:rpm:gstlal-burst
+  except:
+    - /gstlal-([a-z]+-|)[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+/
+
+docker:release:gstlal:rl8:
+  <<: *docker-rpm
+  needs:
+    - level0:rpm:gstlal
+  only:
+    - /gstlal-[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+/
+
+docker:release:gstlal-ugly:rl8:
+  <<: *docker-rpm
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+  only:
+    - /gstlal-ugly-[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+/
+
+docker:release:gstlal-inspiral:rl8:
+  <<: *docker-rpm
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+    - level2:rpm:gstlal-inspiral
   only:
-    - manual
+    - /gstlal-inspiral-[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+/
 
-test:gstlal:
-  image: aepace/gstlal-dev:stretch-mkl
+docker:release:gstlal-burst:rl8:
+  <<: *docker-rpm
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+    - level2:rpm:gstlal-burst
+  only:
+    - /gstlal-burst-[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+/
+
+
+latest_image:
+  interruptible: true
+  stage: docker-latest
+  before_script: [ ]
+  needs:
+    - docker:release:gstlal-inspiral:rl8
+  only:
+    - /gstlal-inspiral-[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+/
+  script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - docker tag $DOCKER_BRANCH $DOCKER_LATEST
+    - docker push $DOCKER_LATEST
+  retry:
+    max: 2
+
+# test stages (rl8)
+test:gstlal:rl8:
+  interruptible: true
+  image: containers.ligo.org/gstlal/gstlal-dev:${LALSUITE_VERSION_TAG}
   stage: test-gstlal
-  dependencies:
-    - level0:gstlal
+  needs:
+    - level0:rpm:gstlal
   script:
     - export GSTLAL_FIR_WHITEN=0
+    - gst-inspect-1.0
 
     # Get the necessary ROM data:
-    - git lfs install
-    - git clone https://git.ligo.org/lscsoft/lalsuite-extra.git ${GSTLAL_DIR}/lalsuite-extra
-    - export LAL_DATA_PATH=${GSTLAL_DIR}/lalsuite-extra/data/lalsimulation/
+    - git clone https://git.ligo.org/alexander.pace/gstlal-testing-data.git ${GSTLAL_DIR}/gstlal-testing-data
+    - export LAL_DATA_PATH=${GSTLAL_DIR}/gstlal-testing-data/
 
     # Run doctests
-    - apt-get install -y python-pytest
     - cd gstlal
-    - python -m pytest -v --doctest-modules --ignore gst/python --ignore port-tools --ignore tests --ignore python/misc.py --ignore python/pipeparts/__init__.py --ignore python/matplotlibhelper.py
+    - python3 -m pytest -c pytest.ini -m "not requires_full_build"
   only:
+    - schedules
     - pushes
+  allow_failure: true
+
+test:gstlal-full-build:rl8:
+  interruptible: true
+  image: containers.ligo.org/gstlal/gstlal-dev:${LALSUITE_VERSION_TAG}
+  stage: test-gstlal-full-build
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+    - level2:rpm:gstlal-burst
+    - level2:rpm:gstlal-inspiral
+  script:
+    - export GSTLAL_FIR_WHITEN=0
+    - gst-inspect-1.0
+
+    # Run doctests
+    - cd gstlal
+    - python3 -m pytest -c pytest.ini -m "requires_full_build"
+  only:
     - schedules
+    - pushes
+  allow_failure: true
 
-test:gstlal-inspiral:
-  image: aepace/gstlal-dev:stretch-mkl
+test:gstlal-inspiral:rl8:
+  interruptible: true
+  image: containers.ligo.org/gstlal/gstlal-dev:${LALSUITE_VERSION_TAG}
   stage: test-inspiral
-  dependencies:
-    - level2:gstlal-inspiral
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+    - level2:rpm:gstlal-inspiral
   script:
     - export GSTLAL_FIR_WHITEN=0
-    - apt-get install -y python-pytest
+    - gst-inspect-1.0
+
+    # Run doctests
     - cd gstlal-inspiral
-    - python -m pytest -v --doctest-modules --ignore gst/python --ignore tests --ignore python/lloidplots.py --ignore python/llweb.py --ignore python/plotsegments.py --ignore python/plotsensitivity.py --ignore python/snglinspiraltable.py --ignore python/spawaveform.py --ignore python/spiirparts.py --ignore python/webpage.py --ignore python/imr_utils.py --ignore python/stats/inspiral_extrinsics.py --ignore python/templates.py
+    - python3 -m pytest -c pytest.ini
   only:
+    - schedules
     - pushes
+  allow_failure: true
+
+test:gstlal-ugly:rl8:
+  interruptible: true
+  image: containers.ligo.org/gstlal/gstlal-dev:${LALSUITE_VERSION_TAG}
+  stage: test-gstlal-ugly
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+  script:
+    # Install RPMs and set up the test environment:
+    - export GSTLAL_FIR_WHITEN=0
+    - gst-inspect-1.0
+
+    # Run doctests
+    - cd gstlal-ugly
+    - python3 -m pytest -c pytest.ini
+  only:
     - schedules
+    - pushes
+  allow_failure: true
 
-test:gstlal-burst:
-  image: aepace/gstlal-dev:stretch-mkl
+test:gstlal-burst:rl8:
+  interruptible: true
+  image: containers.ligo.org/gstlal/gstlal-dev:${LALSUITE_VERSION_TAG}
   stage: test-burst
-  dependencies:
-    - level2:gstlal-burst
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+    - level2:rpm:gstlal-burst
+    - level2:rpm:gstlal-inspiral
   script:
-    - apt-get install -y python-pytest
+    - export GSTLAL_FIR_WHITEN=0
+    - gst-inspect-1.0
     - cd gstlal-burst
-    - python -m pytest -v --doctest-modules --ignore python/excesspower --ignore tests/trigger_test_01.py
+    - python3 -m pytest -c pytest.ini
   only:
-    - pushes
     - schedules
+    - pushes
+  allow_failure: true
 
-test:offline:
-  image: aepace/gstlal-dev:stretch-mkl
+test:offline:rl8:
+  interruptible: true
+  image: containers.ligo.org/gstlal/gstlal-dev:${LALSUITE_VERSION_TAG}
   stage: test-offline
-  dependencies:
-    - level1:gstlal-ugly
-    - level2:gstlal-inspiral
+  needs:
+    - level0:rpm:gstlal
+    - level1:rpm:gstlal-ugly
+    - level2:rpm:gstlal-inspiral
+    - level2:rpm:gstlal-burst
+    - test:gstlal:rl8
+    - test:gstlal-inspiral:rl8
+    - test:gstlal-burst:rl8
   script:
+   # Set up directory structure and copy over built-dependencies from container:
+    - mkdir public
+    # Install RPMs and set up the test environment:
+    - gst-inspect-1.0
+
     # Export variables for the offline tutorial
-    - export GSTLAL_FIR_WHITEN=0
-    - export GSTLAL_WEBVIS_DIR=../../gstlal-ugly/share/vis/
-    - export LAL_PATH=${GSTLAL_DIR}
+    - export LAL_PATH=/usr
     - export USER=gstlal_CI_test
-    
-    # Get documentation packages, and download lalsuite-extra data:
-    - apt-get install --assume-yes texlive-base texlive-binaries texlive-extra-utils texlive-font-utils texlive-fonts-extra texlive-fonts-extra-doc texlive-fonts-recommended dvipng
-    - apt-get install --assume-yes texlive-fonts-recommended-doc texlive-generic-extra texlive-generic-recommended texlive-latex-base texlive-latex-base-doc texlive-latex-extra
-    - apt-get install --assume-yes texlive-latex-extra-doc texlive-latex-recommended texlive-latex-recommended-doc texlive-pictures texlive-pictures-doc texlive-pstricks texlive-pstricks-doc texlive-science texlive-science-doc
+    - export PYTHONUNBUFFERED=1
+
+    - yum -y install bc
 
     # Get the necessary ROM data:
-    - git lfs install
-    - git clone https://git.ligo.org/lscsoft/lalsuite-extra.git ${GSTLAL_DIR}/lalsuite-extra
-    - export LAL_DATA_PATH=${GSTLAL_DIR}/lalsuite-extra/data/lalsimulation/
+    - git clone https://git.ligo.org/alexander.pace/gstlal-testing-data.git ${GSTLAL_DIR}/gstlal-testing-data
+    - export LAL_DATA_PATH=${GSTLAL_DIR}/gstlal-testing-data/
 
     - cd gstlal-inspiral/tests
-    - make -f Makefile.offline_tutorial_test
-    - cp -rf ./WEBDIR/gstlal_offline_tutorial ../../public/
+
+    # Run the makefile:
+    - make -f Makefile.offline_tutorial_test ENABLE_PLOTTING=0
+
+    # Back-up the results docs:
+    #- cp -rf ./WEBDIR/gstlal_offline_tutorial ../../public/
+
+  #artifacts:
+  #  expire_in: 24h
+  #  paths:
+  #    - gstlal-inspiral/tests/WEBDIR/gstlal_offline_tutorial
+  #    - public
+  #  when: always
+  only:
+    - schedules
+    - pushes
+  allow_failure: true
+
+
+# Documentation
+
+docs:
+  interruptible: true
+  image: $DOCKER_BRANCH
+  stage: docs
+  before_script: [ ]
+  script:
+    - |
+      yum install -y dvipng texlive-pdftex 
+      python3 -m pip install sphinx myst-parser sphinx-rtd-theme
+      mkdir -p docs/
+      cd doc; make html IS_CI=1
+      cd ..; cp -rf doc/_build/* docs/
+  needs:
+    - docker:rl8
   artifacts:
-    expire_in: 24h
     paths:
-      - gstlal-inspiral/tests
-      - public
-    when: always
+      - docs
   only:
     - pushes
     - schedules
-  
+
 pages:
-  image: aepace/gstlal-dev:stretch-mkl
+  interruptible: true
   stage: nightly-pages
+  before_script: [ ]
   script:
-    - echo "Building Documentation"
-    # Download and install sphinx, texlive, etc. I'm installing these here
-    # since they don't need to be part of the dev container for running, only
-    # for building documentation. This shaves ~600MB off the size of the gstlal-dev
-    # container:
-    - apt-get install --assume-yes python-sphinx texlive-base texlive-binaries texlive-extra-utils texlive-font-utils texlive-fonts-extra texlive-fonts-extra-doc texlive-fonts-recommended
-    - apt-get install --assume-yes texlive-fonts-recommended-doc texlive-generic-extra texlive-generic-recommended texlive-latex-base texlive-latex-base-doc texlive-latex-extra
-    - apt-get install --assume-yes texlive-latex-extra-doc texlive-latex-recommended texlive-latex-recommended-doc texlive-pictures texlive-pictures-doc texlive-pstricks texlive-pstricks-doc texlive-science texlive-science-doc
-    - export GSTLAL_FIR_WHITEN=0
-    - export TMPDIR=tmp/ 
-    - cd doc; make html
-    - cd ..; cp -rf doc/_build/* public/
-  dependencies:
-    - level2:gstlal-inspiral
-    - level2:gstlal-calibration
-    - level2:gstlal-burst
+    # copy doc artifacts
+    - mkdir -p public/
+    - cp -r docs/* public/
+  needs:
+    - docs
   artifacts:
     paths:
       - public

.gitlab/CODEOWNERS

+#################################################################
+#                GstLAL Codeowners Specification                #
+#################################################################
+
+
+#################################################################
+#                    Repository-Level Files                     #
+#################################################################
+
+# Workflow files
+.gitignore @kipp.cannon @chad-hanna @alexander.pace @patrick.godwin
+.gitlab-ci.yml @kipp.cannon @chad-hanna @alexander.pace @patrick.godwin
+.gitlab/CODEOWNERS @kipp.cannon @chad-hanna @alexander.pace @patrick.godwin
+
+# Catch-all rule for repo (will be overridden by a more specific rule below)
+* @kipp.cannon @chad-hanna @alexander.pace @patrick.godwin
+
+
+#################################################################
+#                     GstLAL Package Files                      #
+#################################################################
+
+# Package catch-all
+gstlal/ @kipp.cannon @chad-hanna @alexander.pace @patrick.godwin
+
+# Specific modules
+gstlal/python/datasource.py @kipp.cannon @chad-hanna @alexander.pace @patrick.godwin @james.kennington
+
+
+#################################################################
+#                   GstLAL-Ugly Package Files                   #
+#################################################################
+
+# Package catch-all
+gstlal-ugly/ @kipp.cannon @chad-hanna @alexander.pace @leo.tsukada @cody.messick @patrick.godwin
+
+# GST Plugins
+gstlal-ugly/gst/framecpp/ @kipp.cannon @ed-maros
+gstlal-ugly/gst/gds/ @kipp.cannon @ed-maros
+gstlal-ugly/gst/nds/ @kipp.cannon @ed-maros
+
+
+#################################################################
+#                   GstLAL-Burst Package Files                  #
+#################################################################
+
+# Package catch-all
+gstlal-burst/ @patrick.godwin
+
+# Cosmic String
+gstlal-burst/gst/lal/gstlal_string_triggergen.* @daichi.tsuna
+gstlal-burst/python/string/ @daichi.tsuna
+
+
+#################################################################
+#                 GstLAL-Inspiral Package Files                 #
+#################################################################
+
+# Package catch-all
+gstlal-inspiral/ @kipp.cannon @chad-hanna @alexander.pace @leo.tsukada @cody.messick @patrick.godwin

Dockerfile.stretch

-# Start with the lalsuite nightly container:
-FROM containers.ligo.org/lscsoft/lalsuite/nightly:stretch
-
-# Labeling/packaging stuff:
-LABEL name="GstLAL Development Package, Debian" \
-      maintainer="Alexander Pace <alexander.pace@ligo.org>" \
-      date="20180625" \
-      support="Reference Platform, Stretch"
-
-# Set up the base user and build environment:
-# Note that I've had to install the following packages to get things to
-# work. Some duplication with the command line, but the list is clearer:
-#   * build-essential (gcc, etc.)
-#   * wget
-#   * vim
-#   * git
-#   * flex (from gobject-introspection)
-#   * bison (from gobject-introspection)
-#   * libffi-dev (from gobject-introspection)
-
-USER root
-
-# Define MKL version variables:
-ENV VERSION 2018
-ENV UPDATE 3
-ENV BUILD 051
-
-RUN apt-get update && apt-get install -y build-essential wget vim git flex bison libffi-dev git-lfs
-
-RUN wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
-    apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
-    sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
-    apt-get update && \
-    apt-get install -y intel-mkl-64bit-$VERSION.$UPDATE-$BUILD
-
-# Install system dependencies:
-RUN apt-get --assume-yes install gtk-doc-tools libfftw3-3 libfftw3-dev liborc-0.4-0 python-gobject-dev python-numpy python-scipy lscsoft-gds python-ligo-segments python3-ligo-segments
-
-# Set some environment variables. Hopefully these propagate to the
-# gitlab-ci environment. Possibly re-define and then just copy the stuff over.
-
-ENV GSTLAL_DIR /usr
-ENV PATH ${GSTLAL_DIR}/bin:${PATH}
-ENV PKG_CONFIG_PATH ${GSTLAL_DIR}/lib/pkgconfig:${GSTLAL_DIR}/lib64/pkgconfig:${PKG_CONFIG_PATH}
-ENV GI_TYPELIB_PATH ${GSTLAL_DIR}/lib/girepository-1.0:${GI_TYPELIB_PATH}
-ENV GST_PLUGIN_PATH ${GSTLAL_DIR}/lib/gstreamer-1.0:${GST_PLUGIN_PATH}
-ENV PYTHONPATH ${GSTLAL_DIR}/lib64/python2.7/site-packages:${GSTLAL_DIR}/lib/python2.7/site-packages:${PYTHONPATH}
-
-ENV MKLROOT=/opt/intel/compilers_and_libraries_2018.3.222/linux/mkl
-ENV LIBRARY_PATH=/opt/intel/compilers_and_libraries_2018.3.222/linux/tbb/lib/intel64_lin/gcc4.7:/opt/intel/compilers_and_libraries_2018.3.222/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.3.222/linux/mkl/lib/intel64_lin
-ENV CPATH=/opt/intel/compilers_and_libraries_2018.3.222/linux/mkl/include
-ENV PATH=PATH=/opt/intel/compilers_and_libraries_2018.3.222/linux/bin/intel64:${PATH}
-ENV PKG_CONFIG_PATH /opt/intel/compilers_and_libraries_2018.3.222/linux/mkl/bin/pkgconfig:${PKG_CONFIG_PATH}
-ENV LD_LIBRARY_PATH /opt/intel/compilers_and_libraries_2018.3.222/linux/compiler/lib/intel64:/opt/intel/compilers_and_libraries_2018.3.222/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.3.222/linux/tbb/lib/intel64_lin/gcc4.7:/opt/intel/compilers_and_libraries_2018.3.222/linux/compiler/lib/intel64_lin:/opt/intel/compilers_and_libraries_2018.3.222/linux/mkl/lib/intel64_lin
-
-# Set build variables:
-ENV CC gcc
-ENV CXX g++
-ENV CFLAGS "-O3 -fPIC -DMKL_ILP64 -m64 -I${MKLROOT}/include -I${GSTLAL_DIR}/include"
-ENV LDFLAGS "-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_ilp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl"
-
-# These are the packages to download. Use the same nomenclature as the optimized Makefile.
-
-ENV GSTPYTHON gst-python-1.4.0
-ENV GSTPLUGINSGOOD gst-plugins-good-1.4.5
-ENV GSTPLUGINSBASE gst-plugins-base-1.4.5
-ENV GSTREAMER gstreamer-1.4.5
-ENV PYGOBJECT pygobject-3.28.0
-ENV GINTRO gobject-introspection-1.50.0
-
-# Make a tar directory and download packages:
-RUN mkdir /tars
-RUN mkdir /src
-
-RUN wget --no-check-certificate http://gstreamer.freedesktop.org/src/gst-python/${GSTPYTHON}.tar.xz --directory-prefix=/tars -nc
-RUN wget --no-check-certificate http://gstreamer.freedesktop.org/src/gst-plugins-good/${GSTPLUGINSGOOD}.tar.xz --directory-prefix=/tars -nc
-RUN wget --no-check-certificate http://gstreamer.freedesktop.org/src/gst-plugins-base/${GSTPLUGINSBASE}.tar.xz --directory-prefix=/tars -nc
-RUN wget --no-check-certificate http://gstreamer.freedesktop.org/src/gstreamer/${GSTREAMER}.tar.xz --directory-prefix=/tars -nc
-RUN wget --no-check-certificate http://ftp.gnome.org/pub/GNOME/sources/pygobject/3.28/${PYGOBJECT}.tar.xz --directory-prefix=/tars -nc
-RUN wget --no-check-certificate http://ftp.gnome.org/pub/gnome/sources/gobject-introspection/1.50/${GINTRO}.tar.xz --directory-prefix=/tars -nc
-
-# Get glue git repo:
-RUN cd /src && \
-    git clone https://git.ligo.org/lscsoft/glue.git
-#    cd glue && \
-#    python setup.py install --prefix=${GSTLAL_DIR}
-
-# Install gobject-introspection:
-RUN tar -xf /tars/${GINTRO}.tar.xz -C /src && \
-    cd /src/${GINTRO} && \
-    ./configure --prefix=${GSTLAL_DIR} --disable-static --with-python=/usr/bin/python && \
-    make && \
-    make install
-
-# Install pygobject:
-RUN tar -xf /tars/${PYGOBJECT}.tar.xz -C /src && \
-    cd /src/${PYGOBJECT} && \
-    ./configure --prefix=${GSTLAL_DIR} --with-python=python2 --enable-cairo=no && \
-    make && \
-    make install
-
-# Install gstreamer:
-RUN tar -xf /tars/${GSTREAMER}.tar.xz -C /src && \
-    cd /src/${GSTREAMER} && \
-    ./configure --prefix=${GSTLAL_DIR}  && \
-    make && \
-    make install
-
-# Install gst-plugins-base:
-RUN tar -xf /tars/${GSTPLUGINSBASE}.tar.xz -C /src && \
-    cd /src/${GSTPLUGINSBASE} && \
-    ./configure --prefix=${GSTLAL_DIR}  && \
-    make && \
-    make install
-
-# Install gst-plugins-good:
-RUN tar -xf /tars/${GSTPLUGINSGOOD}.tar.xz -C /src && \
-    cd /src/${GSTPLUGINSGOOD} && \
-    ./configure --prefix=${GSTLAL_DIR}  && \
-    make && \
-    make install
-
-# Install gst-python:
-RUN tar -xf /tars/${GSTPYTHON}.tar.xz -C /src && \
-    cd /src/${GSTPYTHON} && \
-    #mkdir -p ${GSTLAL_DIR}/lib64/python2.7/site-packages/gi/overrides && \
-    ./configure --prefix=${GSTLAL_DIR} --exec-prefix=${GSTLAL_DIR} --with-libpython-dir=/usr/lib/python2.7/config-x86_64-linux-gnu --with-pygi-overrides-dir=${GSTLAL_DIR}/lib/python2.7/dist-packages/gi/overrides && \
-    make && \
-    make install
-
-# Install glue:
-RUN cd /src/glue &&  \
-    #python setup.py install
-    python setup.py install --prefix=${GSTLAL_DIR}
-
-# Install some stuff needed for documentation. I'm putting it as part of this
-# container because it takes for-freaking-ever to install in CI:
-
-RUN apt-get --assume-yes install doxygen graphviz python-matplotlib python3-matplotlib
-
-# Dumb hack to get ligolw_segments from the python-pylal package. I can't just install
-# python-pylal because it's been depreciated and because there are conflicts with files
-# that are coming from lalapps.
-
-RUN mkdir pylaltmp && \
-    apt-get install --download-only -o=dir::cache=./pylaltmp python-pylal && \
-    ls ./pylaltmp/* && \
-    dpkg-deb -x pylaltmp/archives/python-pylal*.deb pylaltmp && \
-    sed -i 's/pylal/glue/g' pylaltmp/usr/bin/ligolw_segments && \
-    cp pylaltmp/usr/bin/ligolw_segments /usr/bin && \
-    rm -r pylaltmp
-
-
-# Set up a random uid for the avahi user before starting the daemon.
-
-#RUN randuid=$(od -An -N1 -i /dev/random) && \
-#    adduser --uid $randuid avahi
-RUN apt-get --assume-yes install python-avahi avahi-daemon
-
-# Clean up and close-out
-
-RUN rm -rf /tars /src
-
-# Build gstreamer registry cache:
-
-RUN gst-inspect-1.0
-#ENV GST_REGISTRY_UPDATE=no
-
-ENTRYPOINT service dbus start && service avahi-daemon start && bash
\ No newline at end of file

README.md

+![gstlal](https://git.ligo.org/lscsoft/gstlal/raw/master/doc/source/_static/gstlal.png "gstlal")
+
+GStreamer elements for gravitational-wave data analysis
+==================================================================
+
+[![pipeline status](https://git.ligo.org/lscsoft/gstlal/badges/master/pipeline.svg)](https://git.ligo.org/lscsoft/gstlal/commits/master)
+
+|                      | version  | source   |
+| :------------------: | :------: | :------: |
+| `gstlal`             | 1.9.0    | [gstlal-1.9.0.tar.gz](http://software.igwn.org/lscsoft/source/gstlal-1.9.0.tar.gz)  |
+| `gstlal-ugly`        | 1.9.1    | [gstlal-ugly-1.9.1.tar.gz](http://software.igwn.org/lscsoft/source/gstlal-ugly-1.9.1.tar.gz)  |
+| `gstlal-inspiral`    | 1.9.1    | [gstlal-inspiral-1.9.1.tar.gz](http://software.igwn.org/lscsoft/source/gstlal-inspiral-1.9.1.tar.gz)  |
+| `gstlal-burst`       | 0.3.1    | [gstlal-burst-0.3.1.tar.gz](http://software.igwn.org/lscsoft/source/gstlal-burst-0.3.1.tar.gz)  |
+
+Full documentation is provided [here](https://lscsoft.docs.ligo.org/gstlal/).
+
+**GstLAL** provides a suite of GStreamer elements that expose gravitational-wave data analysis tools from the LALSuite library for use in GStreamer signal-processing pipelines.
+
+Examples include an element to add simulated gravitational waves to an h(t) stream, and a source element to provide the contents of .gwf frame files to a GStreamer pipeline.
+Overview
+
+The **GstLAL** software package is used for the following activities:
+
+  * GstLAL: Provides core Gstreamer plugins for signal processing workflows with LIGO data and core python bindings for constructing such workflows.
+  * GstLAL Inspiral: Provides additional signal processing plugins that are specific for LIGO / Virgo searches for compact binaries as well as a substantial amount of python code for post-processing raw signal processing results into gravitational wave candidate lists. Several publications about the methodology and workflow exist, see publications
+  * GstLAL Ugly: An incubator project for gradual inclusion in the other packages.

doc/Makefile

+IS_CI ?= 0
+
 gstlal-modules :
 	# FIXME FIXME FIXME total hack to work around unique python packaging
-	#mv ../gstlal/python/__init__.py ../gstlal/python/__init__.py.bk
+	if [ $(IS_CI) = 0 ]; then \
+		mv ../gstlal/python/__init__.py ../gstlal/python/__init__.py.bk; \
+	fi
 	sphinx-apidoc -e -o source/gstlal/python-modules ../gstlal/python ../gstlal/python/misc.py ../gstlal/python/bottle.py ../gstlal/python/coherent_null.py ../gstlal/python/matplotlibhelper.py
-	#mv ../gstlal/python/__init__.py.bk ../gstlal/python/__init__.py
+	if [ $(IS_CI) = 0 ]; then \
+		mv ../gstlal/python/__init__.py.bk ../gstlal/python/__init__.py; \
+	fi
+	sed -i "1s/python/\`\`gstlal\`\`/" source/gstlal/python-modules/modules.rst
 
 gstlal-inspiral-modules :
 	sphinx-apidoc -e -o source/gstlal-inspiral/python-modules ../gstlal-inspiral/python ../gstlal-inspiral/python/cbc_template_iir.py ../gstlal-inspiral/python/spawaveform.py ../gstlal-inspiral/python/emcee.py ../gstlal-inspiral/python/spiirparts.py ../gstlal-inspiral/python/snglinspiraltable.py
-
-gstlal-calibration-modules :
-	sphinx-apidoc -e -o source/gstlal-calibration/python-modules ../gstlal-calibration/python
+	sed -i "1s/python/\`\`gstlal-inspiral\`\`/" source/gstlal-inspiral/python-modules/modules.rst
 
 gstlal-burst-modules :
 	sphinx-apidoc -e -o source/gstlal-burst/python-modules ../gstlal-burst/python ../gstlal-burst/python/excesspower ../gstlal-burst/python/fxtools/sngltriggertable.py
+	sed -i "1s/python/\`\`gstlal-burst\`\`/" source/gstlal-burst/python-modules/modules.rst
 
 gstlal-ugly-modules :
 	sphinx-apidoc -e -o source/gstlal-ugly/python-modules ../gstlal-ugly/python ../gstlal-ugly/python/gviz_api.py ../gstlal-ugly/python/coherent_inspiral_metric.py  ../gstlal-ugly/python/coherent_inspiral_metric_plots.py ../gstlal-ugly/python/coherent_inspiral_metric_detector_details.py ../gstlal-ugly/python/aggregator.py 
+	sed -i "1s/python/\`\`gstlal-ugly\`\`/" source/gstlal-ugly/python-modules/modules.rst
 
 gstlal-bin :
 	./sphinx-bindoc source/gstlal/bin ../gstlal/bin gstlal_ligo_data_find_check
+	sed -i "1s/bin/\`\`gstlal\`\`/" source/gstlal/bin/bin.rst
 
 gstlal-inspiral-bin :
-	./sphinx-bindoc source/gstlal-inspiral/bin ../gstlal-inspiral/bin gstlal_inspiral_flopulator gstlal_inspiral_lvalert_omegascan gstlal_ll_inspiral_calculate_range gstlal_ll_inspiral_daily_page_online gstlal_iir_inspiral gstlal_inspiral_iir_bank_pipe gstlal_ll_inspiral_save_state gstlal_inspiral_marginalize_likelihoods_online gstlal_inspiral_plot_extrinsic_params gstlal_ll_inspiral_get_urls gstlal_inspiral_plot_svd_bank gstlal_inspiral_lvalert_background_plotter 
-
-gstlal-calibration-bin :
-	./sphinx-bindoc source/gstlal-calibration/bin ../gstlal-calibration/bin
+	./sphinx-bindoc source/gstlal-inspiral/bin ../gstlal-inspiral/bin gstlal_inspiral_flopulator gstlal_inspiral_lvalert_omegascan gstlal_ll_inspiral_calculate_range gstlal_ll_inspiral_daily_page_online gstlal_iir_inspiral gstlal_inspiral_iir_bank_pipe gstlal_ll_inspiral_save_state gstlal_inspiral_marginalize_likelihoods_online gstlal_inspiral_plot_extrinsic_params gstlal_ll_inspiral_get_urls gstlal_inspiral_plot_svd_bank gstlal_inspiral_lvalert_background_plotter gstlal_inspiral_add_p_of_ifos_given_horizon gstlal_inspiral_add_dt_dphi_snr_ratio_pdfs 
+	sed -i "1s/bin/\`\`gstlal-inspiral\`\`/" source/gstlal-inspiral/bin/bin.rst
 
 gstlal-burst-bin :
 	./sphinx-bindoc source/gstlal-burst/bin ../gstlal-burst/bin gstlal_excesspower gstlal_excesspower_trigvis
+	sed -i "1s/bin/\`\`gstlal-burst\`\`/" source/gstlal-burst/bin/bin.rst
 
 gstlal-ugly-bin :
 	./sphinx-bindoc source/gstlal-ugly/bin ../gstlal-ugly/bin gstlal_h1h2_coh_frames gstlal_geo_player gstlal_ll_inspiral_daily_summary_page gstlal_cache_to_segments gstlal_inspiral_plot_efficiency gstlal_burst_injections_by_rate_plot gstlal_display_from_logs gstlal_inspiral_bankviz gstlal_ll_inspiral_regen_reg_from_condor_q gstlal_cbc_sensitivity_vs_flow_at_fixed_computational_cost gstlal_inspiral_check_livetimes gstlal_rds_gen gstlal_injections_by_local_rate_plot gstlal_ninja_smooth_reference_psd gstlal_shift_frames 
+	sed -i "1s/bin/\`\`gstlal-ugly\`\`/" source/gstlal-ugly/bin/bin.rst
 
-html : gstlal-modules gstlal-inspiral-modules gstlal-calibration-modules gstlal-ugly-modules gstlal-burst-modules gstlal-bin gstlal-inspiral-bin gstlal-calibration-bin gstlal-ugly-bin gstlal-burst-bin
+html : gstlal-modules gstlal-inspiral-modules gstlal-ugly-modules gstlal-burst-modules gstlal-bin gstlal-inspiral-bin gstlal-ugly-bin gstlal-burst-bin
 	sphinx-build -b html source _build
 
 clean :

doc/source/_static/css/my_theme.css

+/* Uses skorch docs theme from https://skorch.readthedocs.io */
+
+.wy-side-nav-search {
+    background-color: #ffffff;
+
+}
+
+.wy-side-nav-search > div.version {
+    color: black;
+}
+
+.wy-side-nav-search > a {
+    color: black;
+}
+
+/* Adds logo in header for mobile */
+
+.wy-nav-top {
+    background-color: #ffffff;
+    background-image: url('../logo.svg');
+    background-repeat: no-repeat;
+    background-position: bottom;
+    padding: 0;
+    margin: 0.4045em 0.809em;
+    color: #333;
+}
+
+.wy-nav-top > a {
+    display: none;
+}