foo

.gitlab-ci.yml

 stages:
   - dist
   - test
-  - gen_cache
-  - update_cache
+  - review
   - docs
   - deploy
 
+# have to specify this so that all jobs execute for all commits
+# including merge requests
+workflow:
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+    - if: $CI_COMMIT_BRANCH
+
+variables:
+  GIT_STRATEGY: clone
+
 # build the docker image we will use in all the jobs, with all
-# dependencies pre-installed/configured
-.dependencies: &dependencies
+# dependencies pre-installed/configured.
+gwinc/base:
   stage: dist
   variables:
     IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_JOB_NAME:$CI_COMMIT_REF_NAME
@@ -19,20 +28,18 @@ stages:
       cat <<EOF > Dockerfile
       FROM igwn/base:buster
       RUN apt-get update -qq
-      RUN apt-get -y install --no-install-recommends git python3 python3-yaml python3-scipy python3-matplotlib python3-ipython lalsimulation-python3 python3-pypdf2 python3-h5py
+      RUN apt-get -y install --no-install-recommends git gitlab-cli python3 python3-yaml python3-scipy python3-matplotlib python3-ipython lalsimulation-python3 python3-pypdf2 python3-h5py
       RUN git clone https://gitlab-ci-token:ci_token@git.ligo.org/gwinc/inspiral_range.git
       EOF
     - docker build -t $IMAGE_TAG .
     - docker push $IMAGE_TAG
 
-# actually generate the docker image
-images/base:
-  <<: *dependencies
-
-# run the tests and generate the test report on failure
-test:
+# validate that the noises haven't changed relative to the reference
+# (the reference itself could have been updated, though, see
+# check_approval job)
+validate_noise:
   stage: test
-  image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+  image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
   script:
     - rm -f gwinc_test_report.pdf
     - export MPLBACKEND=agg
@@ -41,14 +48,59 @@ test:
     when: on_failure
     paths:
       - gwinc_test_report.pdf
-    expose_as: 'GWINC test failure report PDF'
+    expose_as: 'noise validation failure report'
+
+# this is a special job intended to run only for merge requests where
+# the test reference hash file has been updated, indicating that there
+# has been a noise change.  if the merge request has not yet been
+# approved, generate a report of noise changes relative to the target
+# branch and present that to the reviewers.  if the merge request is
+# approved, re-run this job, which will succeed if the MR is approved.
+check_approval:
+  stage: review
+  rules:
+    # - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
+    - if: $CI_MERGE_REQUEST_ID
+      changes:
+        - gwinc/test/ref_hash
+  image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
+  script:
+    - echo "NOISE REFERENCE CHANGE, checking approval..."
+    - |
+      cat <<EOF > check_approved.py
+      import sys
+      import gitlab
+      project_id = sys.argv[1]
+      mr_iid = sys.argv[2]
+      # this only works for public repos, otherwise need to specify
+      # private_token=
+      gl = gitlab.Gitlab('https://git.ligo.org')
+      project = gl.projects.get(project_id)
+      mr = project.mergerequests.get(mr_iid)
+      approvals = mr.approvals.get()
+      assert approvals.approved
+      EOF
+    - if ! python3 check_approved.py $CI_PROJECT_ID $CI_MERGE_REQUEST_IID ; then
+    -     old_hash=$(git cat-file -p origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME:gwinc/test/ref_hash)
+    -     if ! python3 -m gwinc.test --git-ref $old_hash -r gwinc_test_report.pdf ; then
+    -         echo "APPROVAL REQUIRED TO MERGE THIS BRANCH."
+    -         /bin/false
+    -     else
+    -         echo "Reference update did not cause appreciable noise change."
+    -     fi
+    - else
+    -     echo "Merge request approved, reference change accepted."
+    - fi
+  artifacts:
+    when: on_failure
+    paths:
+      - gwinc_test_report.pdf
+    expose_as: 'noise changes relative to target branch head APPROVAL REQUIRED TO MERGE'
 
 # create plots for the canonical IFOs
 ifo:
   stage: docs
-  needs:
-    - test
-  image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+  image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
   script:
       - mkdir -p ifo
       - export PYTHONPATH=/inspiral_range
@@ -67,9 +119,7 @@ html:
   stage: docs
   only:
     - master
-  needs:
-    - test
-  image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+  image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
   script:
     - rm -rf public
     - apt-get install -y -qq python3-sphinx-rtd-theme
@@ -82,6 +132,8 @@ html:
     paths:
       - public
 
+# the "pages" job has special meaning, as it's "public" artifact
+# becomes the directory served through gitlab static pages
 pages:
   stage: deploy
   only:
@@ -91,7 +143,7 @@ pages:
       artifacts: true
     - job: html
       artifacts: true
-  image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+  image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
   script:
     - mv ifo public/
   artifacts:

gwinc/test/__main__.py

 import os
 import sys
 import glob
+import shutil
 import signal
 import logging
 import tempfile
@@ -28,13 +29,64 @@ logging.basicConfig(
 
 FREQ = np.logspace(np.log10(5), np.log10(6000), 3000)
 TOLERANCE = 1e-6
+CACHE_LIMIT = 5
 
 
 def test_path(*args):
+    """Return path to package file."""
     return os.path.join(os.path.dirname(__file__), *args)
 
 
-def gen_cache(ref_hash, path):
+def git_ref_resolve_hash(git_ref):
+    """Resolve a git reference into its hash string."""
+    try:
+        return subprocess.run(
+            ['git', 'show', '-s', '--format=format:%H', git_ref],
+            capture_output=True, universal_newlines=True,
+        ).stdout
+    except subprocess.CalledProcessError:
+        return None
+
+
+def write_ref_hash(ref_hash):
+    """Write ref hash to reference file
+
+    """
+    with open(test_path('ref_hash'), 'w') as f:
+        f.write('{}\n'.format(ref_hash))
+
+
+def load_ref_hash():
+    """Load the current reference git hash.
+
+    """
+    try:
+        with open(test_path('ref_hash')) as f:
+            return f.read().strip()
+    except IOError:
+        return None
+
+
+def prune_cache_dir():
+    """Prune all but the N most recently accessed caches.
+
+    """
+    cache_dir = test_path('cache')
+    if not os.path.exists(cache_dir):
+        return
+    expired_paths = sorted(
+        [os.path.join(cache_dir, path) for path in os.listdir(cache_dir)],
+        key=lambda path: os.stat(path).st_atime,
+    )[CACHE_LIMIT:]
+    if not expired_paths:
+        return
+    logging.info("pruning {} old caches...".format(len(expired_paths)))
+    for path in expired_paths:
+        logging.debug("pruning {}...".format(path))
+        shutil.rmtree(path)
+
+
+def gen_cache_for_ref(ref_hash, path):
     """generate cache from git reference
 
     The ref_hash should be a git hash, and path should be the location
@@ -47,7 +99,8 @@ def gen_cache(ref_hash, path):
     """
     logging.info("creating new cache from reference {}...".format(ref_hash))
     subprocess.run(
-        [test_path('gen_cache.sh'), ref_hash, path]
+        [test_path('gen_cache.sh'), ref_hash, path],
+        check=True,
     )
 
 
@@ -65,7 +118,7 @@ def load_cache(path):
             ref_hash = f.read().strip()
     else:
         ref_hash = None
-    logging.info("cache git hash: {}".format(ref_hash))
+    logging.debug("cache hash: {}".format(ref_hash))
     cache['ref_hash'] = ref_hash
     cache['ifos'] = {}
     for f in sorted(os.listdir(path)):
@@ -213,31 +266,52 @@ def main():
         help='specific ifos to test (default all)')
     args = parser.parse_args()
 
+    # get the current hash of HEAD
+    head_hash = git_ref_resolve_hash('HEAD')
+    if not head_hash:
+        logging.warning("could not determine git HEAD hash.")
+
+    # update the reference if specified
     if args.update_ref:
         if args.update_ref == 'HEAD':
-            ref_hash = subprocess.run(
-                ['git', 'show', '-s', '--format=format:%H', 'HEAD'],
-                capture_output=True, universal_newlines=True,
-            ).stdout
+            if not head_hash:
+                sys.exit("Could not update reference to head.")
+            logging.info("updating reference to HEAD...")
+            ref_hash = head_hash
         else:
-            ref_hash = args.update_ref
-        logging.info("updating reference git hash to {}...".format(ref_hash))
-        with open(test_path('ref_hash'), 'w') as f:
-            f.write('{}\n'.format(ref_hash))
+            ref_hash = git_ref_resolve_hash(args.update_ref)
+        logging.info("updating reference git hash: {}".format(ref_hash))
+        write_ref_hash(ref_hash)
         sys.exit()
 
+    # get the reference hash
     if args.git_ref:
-        ref_hash = args.git_ref
-    elif os.path.exists(test_path('ref_hash')):
-        with open(test_path('ref_hash')) as f:
-            ref_hash = f.read().strip()
+        ref_hash = git_ref_resolve_hash(args.git_ref)
     else:
-        sys.exit("Unspecified reference git hash, could not run test.")
+        ref_hash = load_ref_hash()
+        if not ref_hash:
+            pass
+        try:
+            with open(test_path('ref_hash')) as f:
+                ref_hash = f.read().strip()
+        except IOError:
+            logging.warning("could not open reference")
+            sys.exit("Unspecified reference git hash, could not run test.")
+
+    logging.info("head hash: {}".format(head_hash))
+    logging.info("ref  hash: {}".format(ref_hash))
+
+    # don't bother test if hashes match
+    if ref_hash == head_hash:
+        logging.info("HEAD matches reference, not bothering to calculate.")
+        logging.info("Use --git-ref to compare against an arbitrary git commit.")
+        sys.exit()
 
     # load the cache
     cache_path = test_path('cache', ref_hash)
     if not os.path.exists(cache_path):
-        gen_cache(ref_hash, cache_path)
+        prune_cache_dir()
+        gen_cache_for_ref(ref_hash, cache_path)
     cache = load_cache(cache_path)
 
     if args.report: