From a7ce9bd84673b6ff92d58b24ad53055b47105348 Mon Sep 17 00:00:00 2001
From: Jameson Graef Rollins <jrollins@finestructure.net>
Date: Fri, 17 Apr 2020 11:42:38 -0700
Subject: [PATCH] foo
---
.gitlab-ci.yml | 92 +++++++++++++++++++++++++++--------
gwinc/test/__main__.py | 108 ++++++++++++++++++++++++++++++++++-------
2 files changed, 163 insertions(+), 37 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 62990d2..d79541d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,14 +1,23 @@
stages:
- dist
- test
- - gen_cache
- - update_cache
+ - review
- docs
- deploy
+# have to specify this so that all jobs execute for all commits
+# including merge requests
+workflow:
+ rules:
+ - if: $CI_MERGE_REQUEST_ID
+ - if: $CI_COMMIT_BRANCH
+
+variables:
+ GIT_STRATEGY: clone
+
# build the docker image we will use in all the jobs, with all
-# dependencies pre-installed/configured
-.dependencies: &dependencies
+# dependencies pre-installed/configured.
+gwinc/base:
stage: dist
variables:
IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_JOB_NAME:$CI_COMMIT_REF_NAME
@@ -19,20 +28,18 @@ stages:
cat <<EOF > Dockerfile
FROM igwn/base:buster
RUN apt-get update -qq
- RUN apt-get -y install --no-install-recommends git python3 python3-yaml python3-scipy python3-matplotlib python3-ipython lalsimulation-python3 python3-pypdf2 python3-h5py
+ RUN apt-get -y install --no-install-recommends git gitlab-cli python3 python3-yaml python3-scipy python3-matplotlib python3-ipython lalsimulation-python3 python3-pypdf2 python3-h5py
RUN git clone https://gitlab-ci-token:ci_token@git.ligo.org/gwinc/inspiral_range.git
EOF
- docker build -t $IMAGE_TAG .
- docker push $IMAGE_TAG
-# actually generate the docker image
-images/base:
- <<: *dependencies
-
-# run the tests and generate the test report on failure
-test:
+# validate that the noises haven't changed relative to the reference
+# (the reference itself could have been updated, though, see
+# check_approval job)
+validate_noise:
stage: test
- image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+ image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
script:
- rm -f gwinc_test_report.pdf
- export MPLBACKEND=agg
@@ -41,14 +48,59 @@ test:
when: on_failure
paths:
- gwinc_test_report.pdf
- expose_as: 'GWINC test failure report PDF'
+ expose_as: 'noise validation failure report'
+
+# this is a special job intended to run only for merge requests where
+# the test reference hash file has been updated, indicating that there
+# has been a noise change. if the merge request has not yet been
+# approved, generate a report of noise changes relative to the target
+# branch and present that to the reviewers. if the merge request is
+# approved, re-run this job, which will succeed if the MR is approved.
+check_approval:
+ stage: review
+ rules:
+ # - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
+ - if: $CI_MERGE_REQUEST_ID
+ changes:
+ - gwinc/test/ref_hash
+ image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
+ script:
+ - echo "NOISE REFERENCE CHANGE, checking approval..."
+ - |
+ cat <<EOF > check_approved.py
+ import sys
+ import gitlab
+ project_id = sys.argv[1]
+ mr_iid = sys.argv[2]
+ # this only works for public repos, otherwise need to specify
+ # private_token=
+ gl = gitlab.Gitlab('https://git.ligo.org')
+ project = gl.projects.get(project_id)
+ mr = project.mergerequests.get(mr_iid)
+ approvals = mr.approvals.get()
+ assert approvals.approved
+ EOF
+ - if ! python3 check_approved.py $CI_PROJECT_ID $CI_MERGE_REQUEST_IID ; then
+ - old_hash=$(git cat-file -p origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME:gwinc/test/ref_hash)
+ - if ! python3 -m gwinc.test --git-ref $old_hash -r gwinc_test_report.pdf ; then
+ - echo "APPROVAL REQUIRED TO MERGE THIS BRANCH."
+ - /bin/false
+ - else
+ - echo "Reference update did not cause appreciable noise change."
+ - fi
+ - else
+ - echo "Merge request approved, reference change accepted."
+ - fi
+ artifacts:
+ when: on_failure
+ paths:
+ - gwinc_test_report.pdf
+ expose_as: 'noise changes relative to target branch head APPROVAL REQUIRED TO MERGE'
# create plots for the canonical IFOs
ifo:
stage: docs
- needs:
- - test
- image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+ image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
script:
- mkdir -p ifo
- export PYTHONPATH=/inspiral_range
@@ -67,9 +119,7 @@ html:
stage: docs
only:
- master
- needs:
- - test
- image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+ image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
script:
- rm -rf public
- apt-get install -y -qq python3-sphinx-rtd-theme
@@ -82,6 +132,8 @@ html:
paths:
- public
+# the "pages" job has special meaning, as it's "public" artifact
+# becomes the directory served through gitlab static pages
pages:
stage: deploy
only:
@@ -91,7 +143,7 @@ pages:
artifacts: true
- job: html
artifacts: true
- image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME
+ image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME
script:
- mv ifo public/
artifacts:
diff --git a/gwinc/test/__main__.py b/gwinc/test/__main__.py
index c6900de..55b4e35 100644
--- a/gwinc/test/__main__.py
+++ b/gwinc/test/__main__.py
@@ -1,6 +1,7 @@
import os
import sys
import glob
+import shutil
import signal
import logging
import tempfile
@@ -28,13 +29,64 @@ logging.basicConfig(
FREQ = np.logspace(np.log10(5), np.log10(6000), 3000)
TOLERANCE = 1e-6
+CACHE_LIMIT = 5
def test_path(*args):
+ """Return path to package file."""
return os.path.join(os.path.dirname(__file__), *args)
-def gen_cache(ref_hash, path):
+def git_ref_resolve_hash(git_ref):
+ """Resolve a git reference into its hash string."""
+ try:
+ return subprocess.run(
+ ['git', 'show', '-s', '--format=format:%H', git_ref],
+ capture_output=True, universal_newlines=True,
+ ).stdout
+ except subprocess.CalledProcessError:
+ return None
+
+
+def write_ref_hash(ref_hash):
+ """Write ref hash to reference file
+
+ """
+ with open(test_path('ref_hash'), 'w') as f:
+ f.write('{}\n'.format(ref_hash))
+
+
+def load_ref_hash():
+ """Load the current reference git hash.
+
+ """
+ try:
+ with open(test_path('ref_hash')) as f:
+ return f.read().strip()
+ except IOError:
+ return None
+
+
+def prune_cache_dir():
+ """Prune all but the N most recently accessed caches.
+
+ """
+ cache_dir = test_path('cache')
+ if not os.path.exists(cache_dir):
+ return
+ expired_paths = sorted(
+ [os.path.join(cache_dir, path) for path in os.listdir(cache_dir)],
+ key=lambda path: os.stat(path).st_atime,
+ )[CACHE_LIMIT:]
+ if not expired_paths:
+ return
+ logging.info("pruning {} old caches...".format(len(expired_paths)))
+ for path in expired_paths:
+ logging.debug("pruning {}...".format(path))
+ shutil.rmtree(path)
+
+
+def gen_cache_for_ref(ref_hash, path):
"""generate cache from git reference
The ref_hash should be a git hash, and path should be the location
@@ -47,7 +99,8 @@ def gen_cache(ref_hash, path):
"""
logging.info("creating new cache from reference {}...".format(ref_hash))
subprocess.run(
- [test_path('gen_cache.sh'), ref_hash, path]
+ [test_path('gen_cache.sh'), ref_hash, path],
+ check=True,
)
@@ -65,7 +118,7 @@ def load_cache(path):
ref_hash = f.read().strip()
else:
ref_hash = None
- logging.info("cache git hash: {}".format(ref_hash))
+ logging.debug("cache hash: {}".format(ref_hash))
cache['ref_hash'] = ref_hash
cache['ifos'] = {}
for f in sorted(os.listdir(path)):
@@ -213,31 +266,52 @@ def main():
help='specific ifos to test (default all)')
args = parser.parse_args()
+ # get the current hash of HEAD
+ head_hash = git_ref_resolve_hash('HEAD')
+ if not head_hash:
+ logging.warning("could not determine git HEAD hash.")
+
+ # update the reference if specified
if args.update_ref:
if args.update_ref == 'HEAD':
- ref_hash = subprocess.run(
- ['git', 'show', '-s', '--format=format:%H', 'HEAD'],
- capture_output=True, universal_newlines=True,
- ).stdout
+ if not head_hash:
+ sys.exit("Could not update reference to head.")
+ logging.info("updating reference to HEAD...")
+ ref_hash = head_hash
else:
- ref_hash = args.update_ref
- logging.info("updating reference git hash to {}...".format(ref_hash))
- with open(test_path('ref_hash'), 'w') as f:
- f.write('{}\n'.format(ref_hash))
+ ref_hash = git_ref_resolve_hash(args.update_ref)
+ logging.info("updating reference git hash: {}".format(ref_hash))
+ write_ref_hash(ref_hash)
sys.exit()
+ # get the reference hash
if args.git_ref:
- ref_hash = args.git_ref
- elif os.path.exists(test_path('ref_hash')):
- with open(test_path('ref_hash')) as f:
- ref_hash = f.read().strip()
+ ref_hash = git_ref_resolve_hash(args.git_ref)
else:
- sys.exit("Unspecified reference git hash, could not run test.")
+ ref_hash = load_ref_hash()
+ if not ref_hash:
+ pass
+ try:
+ with open(test_path('ref_hash')) as f:
+ ref_hash = f.read().strip()
+ except IOError:
+ logging.warning("could not open reference")
+ sys.exit("Unspecified reference git hash, could not run test.")
+
+ logging.info("head hash: {}".format(head_hash))
+ logging.info("ref hash: {}".format(ref_hash))
+
+ # don't bother test if hashes match
+ if ref_hash == head_hash:
+ logging.info("HEAD matches reference, not bothering to calculate.")
+ logging.info("Use --git-ref to compare against an arbitrary git commit.")
+ sys.exit()
# load the cache
cache_path = test_path('cache', ref_hash)
if not os.path.exists(cache_path):
- gen_cache(ref_hash, cache_path)
+ prune_cache_dir()
+ gen_cache_for_ref(ref_hash, cache_path)
cache = load_cache(cache_path)
if args.report:
--
GitLab