.gitlab-ci.yml

-image: igwn/base:buster
-
 stages:
-- test
-- deploy
+  - dist
+  - test
+  - gen_cache
+  - update_cache
+  - docs
+  - deploy
+
+# build the docker image we will use in all the jobs, with all
+# dependencies pre-installed/configured
+.dependencies: &dependencies
+  stage: dist
+  variables:
+    IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_JOB_NAME:$CI_COMMIT_REF_NAME
+    GIT_STRATEGY: none
+  script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - |
+      cat <<EOF > Dockerfile
+      FROM igwn/base:buster
+      RUN apt-get update -qq
+      RUN apt-get -y install --no-install-recommends git python3 python3-yaml python3-scipy python3-matplotlib python3-ipython lalsimulation-python3 python3-pypdf2 python3-h5py
+      RUN git clone https://gitlab-ci-token:ci_token@git.ligo.org/gwinc/inspiral_range.git
+      EOF
+    - docker build -t $IMAGE_TAG .
+    - docker push $IMAGE_TAG
+
+# actually generate the docker image
+images/base:
+  <<: *dependencies
 
+# run the tests and generate the test report on failure
 test:
   stage: test
-  before_script:
-  - echo $CI_COMMIT_SHA | cut -b1-8 > gitID.txt
+  image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
   script:
-  - rm -rf ifo gwinc_test_report.pdf
-  - mkdir ifo
-  - apt-get update -qq
-  - apt-get install -y -qq git python3-yaml python3-scipy python3-matplotlib python3-ipython lalsimulation-python3 python3-pypdf2 python3-h5py
-  - git clone https://gitlab-ci-token:ci_token@git.ligo.org/gwinc/inspiral_range.git
-  - export PYTHONPATH=inspiral_range
-  - export MPLBACKEND=agg
-  - python3 -m gwinc.test -r gwinc_test_report.pdf
-  - for ifo in aLIGO Aplus Voyager CE1 CE2; do
-  -     python3 -m gwinc $ifo -s ifo/$ifo.png
-  -     python3 -m gwinc $ifo -s ifo/$ifo.h5
-  - done
-  - python3 -m gwinc.ifo -s ifo/all_compare.png
-  after_script:
-  - rm gitID.txt
-  cache:
-    key: "$CI_PROJECT_NAMESPACE:$CI_PROJECT_NAME:$CI_JOB_NAME"
-    untracked: true
+    - rm -f gwinc_test_report.pdf
+    - export MPLBACKEND=agg
+    - python3 -m gwinc.test -r gwinc_test_report.pdf
+  artifacts:
+    when: on_failure
+    paths:
+      - gwinc_test_report.pdf
+    expose_as: 'GWINC test failure report PDF'
+
+# create plots for the canonical IFOs
+ifo:
+  stage: docs
+  needs:
+    - test
+  image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+  script:
+      - mkdir -p ifo
+      - export PYTHONPATH=/inspiral_range
+      - for ifo in $(python3 -c "import gwinc; print(' '.join(gwinc.IFOS))"); do
+      -     python3 -m gwinc $ifo -s ifo/$ifo.png
+      -     python3 -m gwinc $ifo -s ifo/$ifo.h5
+      - done
+      - python3 -m gwinc.ifo -s ifo/all_compare.png
   artifacts:
     when: always
+    paths:
+      - ifo
     expire_in: 4w
+
+# generate the html doc web pages
+html:
+  stage: docs
+  # only:
+  #   - master
+  needs:
+    - test
+  image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+  script:
+    - rm -rf public
+    - apt-get install -y -qq python3-sphinx-rtd-theme
+    - cd docs
+    - make html
+    - cd ..
+    - mv ./build/sphinx/html public
+  artifacts:
+    when: always
     paths:
-    - ifo
-    - gwinc_test_report.pdf
+      - public
+    expire_in: 4w
 
 pages:
   stage: deploy
-  dependencies:
-  - test
+  # only:
+  #   - master
+  needs:
+    - job: ifo
+      artifacts: true
+    - job: html
+      artifacts: true
+  image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
   script:
-  - rm -rf public
-  - mv ifo public
-  - apt-get install -y -qq python3-sphinx-rtd-theme
-  - cd docs
-  - make html
-  - cd ..
-  - mv ./build/sphinx/html/* public/
-
+    - mv ifo public/
   artifacts:
+    when: always
     paths:
-    - public
-    expire_in: 4w
-  only:
-  - master
+      - public

IFO.md

@@ -2,43 +2,43 @@
 
 CI-generated plots and data for all IFOs included in pygwinc.
 
-![IFO compare](https://gwinc.docs.ligo.org/pygwinc/all_compare.png)
+![IFO compare](https://gwinc.docs.ligo.org/pygwinc/ifo/all_compare.png)
 
 ## aLIGO
 
 * [ifo.yaml](gwinc/ifo/aLIGO/ifo.yaml)
-* [aLIGO.h5](https://gwinc.docs.ligo.org/pygwinc/aLIGO.h5)
+* [aLIGO.h5](https://gwinc.docs.ligo.org/pygwinc/ifo/aLIGO.h5)
 
-![aLIGO](https://gwinc.docs.ligo.org/pygwinc/aLIGO.png)
+![aLIGO](https://gwinc.docs.ligo.org/pygwinc/ifo/aLIGO.png)
 
 
 ## A+
 
 * [ifo.yaml](gwinc/ifo/Aplus/ifo.yaml)
-* [Aplus.h5](https://gwinc.docs.ligo.org/pygwinc/Aplus.h5)
+* [Aplus.h5](https://gwinc.docs.ligo.org/pygwinc/ifo/Aplus.h5)
 
-![Aplus](https://gwinc.docs.ligo.org/pygwinc/Aplus.png)
+![Aplus](https://gwinc.docs.ligo.org/pygwinc/ifo/Aplus.png)
 
 
 ## Voyager
 
 * [ifo.yaml](gwinc/ifo/Voyager/ifo.yaml)
-* [Voyager.h5](https://gwinc.docs.ligo.org/pygwinc/Voyager.h5)
+* [Voyager.h5](https://gwinc.docs.ligo.org/pygwinc/ifo/Voyager.h5)
 
-![Voyager](https://gwinc.docs.ligo.org/pygwinc/Voyager.png)
+![Voyager](https://gwinc.docs.ligo.org/pygwinc/ifo/Voyager.png)
 
 
 ## Cosmic Explorer 1
 
 * [ifo.yaml](gwinc/ifo/CE1/ifo.yaml)
-* [CE1.h5](https://gwinc.docs.ligo.org/pygwinc/CE1.h5)
+* [CE1.h5](https://gwinc.docs.ligo.org/pygwinc/ifo/CE1.h5)
 
-![CE1](https://gwinc.docs.ligo.org/pygwinc/CE1.png)
+![CE1](https://gwinc.docs.ligo.org/pygwinc/ifo/CE1.png)
 
 
 ## Cosmic Explorer 2
 
 * [ifo.yaml](gwinc/ifo/CE2/ifo.yaml)
-* [CE2.h5](https://gwinc.docs.ligo.org/pygwinc/CE2.h5)
+* [CE2.h5](https://gwinc.docs.ligo.org/pygwinc/ifo/CE2.h5)
 
-![CE2](https://gwinc.docs.ligo.org/pygwinc/CE2.png)
+![CE2](https://gwinc.docs.ligo.org/pygwinc/ifo/CE2.png)

README.md

@@ -2,12 +2,14 @@
 
 # Python Gravitational Wave Interferometer Noise Calculator
 
-![gwinc](https://gwinc.docs.ligo.org/pygwinc/aLIGO.png)
+[![aLIGO](https://gwinc.docs.ligo.org/pygwinc/ifo/aLIGO.png "Canonical
+IFOs")](IFO.md)
 
 `pygwinc` is a multi-faceted tool for processing and plotting noise
 budgets for ground-based gravitational wave detectors.  It's primary
-feature is a collection of mostly analytic noise calculation functions
-for various sources of noise affecting detectors (`gwinc.noise`):
+feature is a collection of mostly analytic [noise calculation
+functions](#noise-functions) for various sources of noise affecting
+detectors (`gwinc.noise`):
 
 * quantum noise
 * mirror coating thermal noise
@@ -17,22 +19,20 @@ for various sources of noise affecting detectors (`gwinc.noise`):
 * Newtonian/gravity-gradient noise
 * residual gas noise
 
-See [noise functions](#noise-functions) below.
-
-`pygwinc` also includes a generalized noise budgeting tool
-(`gwinc.nb`) that allows users to create arbitrary noise budgets (for
-any experiment, not just ground-based GW detectors) using measured or
-analytically calculated data.  See the [budget
-interface](#Budget-interface) section below.
+`pygwinc` is also a generalized noise budgeting tool (`gwinc.nb`) that
+allows users to create arbitrary noise budgets (for any experiment,
+not just ground-based GW detectors) using measured or analytically
+calculated data.  See the [budget interface](#Budget-interface)
+section below.
 
 `pygwinc` includes canonical budgets for various well-known current
-and future detectors (`gwinc.ifo`):
+and future GW detectors (`gwinc.ifo`):
 
-* [aLIGO](https://gwinc.docs.ligo.org/pygwinc/aLIGO.png)
-* [A+](https://gwinc.docs.ligo.org/pygwinc/Aplus.png)
-* [Voyager](https://gwinc.docs.ligo.org/pygwinc/Voyager.png)
-* [Cosmic Explorer 1](https://gwinc.docs.ligo.org/pygwinc/CE1.png)
-* [Cosmic Explorer 2](https://gwinc.docs.ligo.org/pygwinc/CE2.png)
+* [aLIGO](https://gwinc.docs.ligo.org/pygwinc/ifo/aLIGO.png)
+* [A+](https://gwinc.docs.ligo.org/pygwinc/ifo/Aplus.png)
+* [Voyager](https://gwinc.docs.ligo.org/pygwinc/ifo/Voyager.png)
+* [Cosmic Explorer 1](https://gwinc.docs.ligo.org/pygwinc/ifo/CE1.png)
+* [Cosmic Explorer 2](https://gwinc.docs.ligo.org/pygwinc/ifo/CE2.png)
 
 See [IFO.md](IFO.md) for the latest CI-generated plots and hdf5 cached
 data.
@@ -325,13 +325,7 @@ traces = budget.run()
 ```
 
 The IFOs included in `gwinc.ifo` provide examples of the use of the
-budget interface:
-
-* [aLIGO](gwinc/ifo/aLIGO)
-* [Aplus](gwinc/ifo/Aplus)
-* [Voyager](gwinc/ifo/Voyager)
-* [CE1](master/gwinc/ifo/CE1)
-* [CE2](master/gwinc/ifo/CE2)
+budget interface (e.g. [gwinc.ifo.aLIGO](gwinc/ifo/aLIGO)).
 
 
 ### extracting single noise terms

gwinc/test/__main__.py

@@ -11,9 +11,6 @@ from collections import OrderedDict
 from collections.abc import Mapping
 from PyPDF2 import PdfFileReader, PdfFileWriter
 
-logging.basicConfig(format='%(message)s',
-                    level=os.getenv('LOG_LEVEL', logging.INFO))
-
 from .. import IFOS, load_budget
 from ..io import load_hdf5, save_hdf5
 
@@ -23,6 +20,11 @@ except ImportError:
     inspiral_range = None
 
 
+logging.basicConfig(
+    format='%(message)s',
+    level=os.getenv('LOG_LEVEL', logging.INFO))
+
+
 TOLERANCE = 1e-6
 CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cache')
 
@@ -106,7 +108,7 @@ def compare_traces(tracesA, tracesB, tolerance=TOLERANCE, skip=None):
 
 
 def plot_diffs(freq, diffs, tolerance,
-               name, labelA, labelB, fom_title='',
+               name, styleA, styleB, fom_title='',
                save=None):
     spec = (len(diffs)+1, 2)
     sharex = None
@@ -114,8 +116,8 @@ def plot_diffs(freq, diffs, tolerance,
         noiseA, noiseB, frac = diffs[nname]
 
         axl = plt.subplot2grid(spec, (i, 0), sharex=None)
-        axl.loglog(freq, np.sqrt(noiseA), label=labelA)
-        axl.loglog(freq, np.sqrt(noiseB), label=labelB)
+        axl.loglog(freq, np.sqrt(noiseA), **styleA)
+        axl.loglog(freq, np.sqrt(noiseB), **styleB)
         axl.grid()
         axl.legend(loc='upper right')
         axl.set_ylabel(nname)
@@ -136,7 +138,7 @@ def plot_diffs(freq, diffs, tolerance,
 
         plt.suptitle('''{} {}/{} noise comparison
 (noises that differ by more than {} ppm)
-{}'''.format(name, labelA, labelB, tolerance*1e6, fom_title))
+{}'''.format(name, styleA['label'], styleB['label'], tolerance*1e6, fom_title))
 
     axl.set_xlabel("frequency [Hz]")
     axr.set_xlabel("frequency [Hz]")
@@ -154,21 +156,28 @@ def plot_diffs(freq, diffs, tolerance,
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('--tolerance', '-t',  type=float, default=TOLERANCE,
-                        help='fractional tolerance [{}]'.format(TOLERANCE))
-    parser.add_argument('--skip', '-k', metavar='NOISE', action='append',
-                        help='traces to skip in comparison (multiple may be specified)')
-    parser.add_argument('--cache', '-c', metavar='PATH', default=CACHE_PATH,
-                        help='specify alternate IFO traces cache path')
+    parser.add_argument(
+        '--tolerance', '-t',  type=float, default=TOLERANCE,
+        help='fractional tolerance [{}]'.format(TOLERANCE))
+    parser.add_argument(
+        '--skip', '-k', metavar='NOISE', action='append',
+        help='traces to skip in comparison (multiple may be specified)')
+    parser.add_argument(
+        '--cache', '-c', metavar='PATH', default=CACHE_PATH,
+        help='specify alternate IFO traces cache path')
     rgroup = parser.add_mutually_exclusive_group()
-    rgroup.add_argument('--plot', '-p', action='store_true',
-                        help='plot differences')
-    rgroup.add_argument('--report', '-r', metavar='REPORT.pdf',
-                        help='create PDF report of test results (only created if differences found)')
-    rgroup.add_argument('--gen-cache', action='store_true',
-                        help='update/create IFO traces cache directory')
-    parser.add_argument('ifo', metavar='IFO', nargs='*',
-                        help='specific ifos to test (default all)')
+    rgroup.add_argument(
+        '--plot', '-p', action='store_true',
+        help='plot differences')
+    rgroup.add_argument(
+        '--report', '-r', metavar='REPORT.pdf',
+        help='create PDF report of test results (only created if differences found)')
+    rgroup.add_argument(
+        '--gen-cache', action='store_true',
+        help='update/create IFO traces cache directory')
+    parser.add_argument(
+        'ifo', metavar='IFO', nargs='*',
+        help='specific ifos to test (default all)')
     args = parser.parse_args()
 
     if args.gen_cache:
@@ -183,13 +192,13 @@ def main():
             path = os.path.join(args.cache, name+'.h5')
             save_hdf5(path, freq, traces)
         return
-        
+
     if args.report:
         base, ext = os.path.splitext(args.report)
         if ext != '.pdf':
             parser.error("Test reports only support PDF format.")
         outdir = tempfile.TemporaryDirectory()
-    
+
     # find all cached IFOs
     logging.info("loading cache {}...".format(args.cache))
     cached_ifos = {}
@@ -201,50 +210,57 @@ def main():
 
     # select
     if args.ifo:
-        ifos = {name:cached_ifos[name] for name in args.ifo}
+        ifos = args.ifo
     else:
-        ifos = cached_ifos
+        ifos = IFOS
 
-    labelA = 'cache'
-    labelB = 'head'
+    style_cache = dict(label='cache', linestyle='-')
+    style_head = dict(label='head', linestyle='--')
 
     fail = False
 
     # compare
-    for name, path in ifos.items():
+    for name in ifos:
         logging.info("{} tests...".format(name))
 
-        freq, tracesA, attrs = load_hdf5(path)
+        path = cached_ifos[name]
+
+        if not os.path.exists(path):
+            logging.warning("{} test cache not found".format(name))
+            fail |= True
+            continue
+
+        freq, traces_cache, attrs = load_hdf5(path)
 
         Budget = load_budget(name)
-        tracesB = Budget(freq).run()
+        traces_head = Budget(freq).run()
 
         if inspiral_range:
-            totalA = tracesA['Total'][0]
-            totalB = tracesB['Total'][0]
+            total_cache = traces_cache['Total'][0]
+            total_head = traces_head['Total'][0]
             range_func = inspiral_range.range
             H = inspiral_range.waveform.CBCWaveform(freq)
-            fomA = range_func(freq, totalA, H=H)
-            tracesA['int73'] = inspiral_range.int73(freq, totalA)[1], None
-            fomB = range_func(freq, totalB, H=H)
-            tracesB['int73'] = inspiral_range.int73(freq, totalB)[1], None
+            fom_cache = range_func(freq, total_cache, H=H)
+            traces_cache['int73'] = inspiral_range.int73(freq, total_cache)[1], None
+            fom_head = range_func(freq, total_head, H=H)
+            traces_head['int73'] = inspiral_range.int73(freq, total_head)[1], None
             fom_summary = """
 inspiral {func} {m1}/{m2} Msol:
-{labelA}: {fomA:.2f} Mpc
-{labelB}: {fomB:.2f} Mpc
+{label_cache}: {fom_cache:.2f} Mpc
+{label_head}: {fom_head:.2f} Mpc
 """.format(
                 func=range_func.__name__,
                 m1=H.params['m1'],
                 m2=H.params['m2'],
-                labelA=labelA,
-                fomA=fomA,
-                labelB=labelB,
-                fomB=fomB,
+                label_cache=style_cache['label'],
+                fom_cache=fom_cache,
+                label_head=style_head['label'],
+                fom_head=fom_head,
             )
         else:
             fom_summary = ''
 
-        diffs = compare_traces(tracesA, tracesB, args.tolerance, args.skip)
+        diffs = compare_traces(traces_cache, traces_head, args.tolerance, args.skip)
 
         if diffs:
             logging.warning("{} tests FAIL".format(name))
@@ -256,7 +272,7 @@ inspiral {func} {m1}/{m2} Msol:
                     save = None
                 plot_diffs(
                     freq, diffs, args.tolerance,
-                    name, labelA, labelB, fom_summary,
+                    name, style_cache, style_head, fom_summary,
                     save=save,
                 )
         else: