From a7ce9bd84673b6ff92d58b24ad53055b47105348 Mon Sep 17 00:00:00 2001 From: Jameson Graef Rollins <jrollins@finestructure.net> Date: Fri, 17 Apr 2020 11:42:38 -0700 Subject: [PATCH] foo --- .gitlab-ci.yml | 92 +++++++++++++++++++++++++++-------- gwinc/test/__main__.py | 108 ++++++++++++++++++++++++++++++++++------- 2 files changed, 163 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62990d2..d79541d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,23 @@ stages: - dist - test - - gen_cache - - update_cache + - review - docs - deploy +# have to specify this so that all jobs execute for all commits +# including merge requests +workflow: + rules: + - if: $CI_MERGE_REQUEST_ID + - if: $CI_COMMIT_BRANCH + +variables: + GIT_STRATEGY: clone + # build the docker image we will use in all the jobs, with all -# dependencies pre-installed/configured -.dependencies: &dependencies +# dependencies pre-installed/configured. +gwinc/base: stage: dist variables: IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_JOB_NAME:$CI_COMMIT_REF_NAME @@ -19,20 +28,18 @@ stages: cat <<EOF > Dockerfile FROM igwn/base:buster RUN apt-get update -qq - RUN apt-get -y install --no-install-recommends git python3 python3-yaml python3-scipy python3-matplotlib python3-ipython lalsimulation-python3 python3-pypdf2 python3-h5py + RUN apt-get -y install --no-install-recommends git gitlab-cli python3 python3-yaml python3-scipy python3-matplotlib python3-ipython lalsimulation-python3 python3-pypdf2 python3-h5py RUN git clone https://gitlab-ci-token:ci_token@git.ligo.org/gwinc/inspiral_range.git EOF - docker build -t $IMAGE_TAG . - docker push $IMAGE_TAG -# actually generate the docker image -images/base: - <<: *dependencies - -# run the tests and generate the test report on failure -test: +# validate that the noises haven't changed relative to the reference +# (the reference itself could have been updated, though, see +# check_approval job) +validate_noise: stage: test - image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME + image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME script: - rm -f gwinc_test_report.pdf - export MPLBACKEND=agg @@ -41,14 +48,59 @@ test: when: on_failure paths: - gwinc_test_report.pdf - expose_as: 'GWINC test failure report PDF' + expose_as: 'noise validation failure report' + +# this is a special job intended to run only for merge requests where +# the test reference hash file has been updated, indicating that there +# has been a noise change. if the merge request has not yet been +# approved, generate a report of noise changes relative to the target +# branch and present that to the reviewers. if the merge request is +# approved, re-run this job, which will succeed if the MR is approved. +check_approval: + stage: review + rules: + # - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' + - if: $CI_MERGE_REQUEST_ID + changes: + - gwinc/test/ref_hash + image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME + script: + - echo "NOISE REFERENCE CHANGE, checking approval..." + - | + cat <<EOF > check_approved.py + import sys + import gitlab + project_id = sys.argv[1] + mr_iid = sys.argv[2] + # this only works for public repos, otherwise need to specify + # private_token= + gl = gitlab.Gitlab('https://git.ligo.org') + project = gl.projects.get(project_id) + mr = project.mergerequests.get(mr_iid) + approvals = mr.approvals.get() + assert approvals.approved + EOF + - if ! python3 check_approved.py $CI_PROJECT_ID $CI_MERGE_REQUEST_IID ; then + - old_hash=$(git cat-file -p origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME:gwinc/test/ref_hash) + - if ! python3 -m gwinc.test --git-ref $old_hash -r gwinc_test_report.pdf ; then + - echo "APPROVAL REQUIRED TO MERGE THIS BRANCH." + - /bin/false + - else + - echo "Reference update did not cause appreciable noise change." + - fi + - else + - echo "Merge request approved, reference change accepted." + - fi + artifacts: + when: on_failure + paths: + - gwinc_test_report.pdf + expose_as: 'noise changes relative to target branch head APPROVAL REQUIRED TO MERGE' # create plots for the canonical IFOs ifo: stage: docs - needs: - - test - image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME + image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME script: - mkdir -p ifo - export PYTHONPATH=/inspiral_range @@ -67,9 +119,7 @@ html: stage: docs only: - master - needs: - - test - image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME + image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME script: - rm -rf public - apt-get install -y -qq python3-sphinx-rtd-theme @@ -82,6 +132,8 @@ html: paths: - public +# the "pages" job has special meaning, as it's "public" artifact +# becomes the directory served through gitlab static pages pages: stage: deploy only: @@ -91,7 +143,7 @@ pages: artifacts: true - job: html artifacts: true - image: $CI_REGISTRY_IMAGE/images/base:$CI_COMMIT_REF_NAME + image: $CI_REGISTRY_IMAGE/gwinc/base:$CI_COMMIT_REF_NAME script: - mv ifo public/ artifacts: diff --git a/gwinc/test/__main__.py b/gwinc/test/__main__.py index c6900de..55b4e35 100644 --- a/gwinc/test/__main__.py +++ b/gwinc/test/__main__.py @@ -1,6 +1,7 @@ import os import sys import glob +import shutil import signal import logging import tempfile @@ -28,13 +29,64 @@ logging.basicConfig( FREQ = np.logspace(np.log10(5), np.log10(6000), 3000) TOLERANCE = 1e-6 +CACHE_LIMIT = 5 def test_path(*args): + """Return path to package file.""" return os.path.join(os.path.dirname(__file__), *args) -def gen_cache(ref_hash, path): +def git_ref_resolve_hash(git_ref): + """Resolve a git reference into its hash string.""" + try: + return subprocess.run( + ['git', 'show', '-s', '--format=format:%H', git_ref], + capture_output=True, universal_newlines=True, + ).stdout + except subprocess.CalledProcessError: + return None + + +def write_ref_hash(ref_hash): + """Write ref hash to reference file + + """ + with open(test_path('ref_hash'), 'w') as f: + f.write('{}\n'.format(ref_hash)) + + +def load_ref_hash(): + """Load the current reference git hash. + + """ + try: + with open(test_path('ref_hash')) as f: + return f.read().strip() + except IOError: + return None + + +def prune_cache_dir(): + """Prune all but the N most recently accessed caches. + + """ + cache_dir = test_path('cache') + if not os.path.exists(cache_dir): + return + expired_paths = sorted( + [os.path.join(cache_dir, path) for path in os.listdir(cache_dir)], + key=lambda path: os.stat(path).st_atime, + )[CACHE_LIMIT:] + if not expired_paths: + return + logging.info("pruning {} old caches...".format(len(expired_paths))) + for path in expired_paths: + logging.debug("pruning {}...".format(path)) + shutil.rmtree(path) + + +def gen_cache_for_ref(ref_hash, path): """generate cache from git reference The ref_hash should be a git hash, and path should be the location @@ -47,7 +99,8 @@ def gen_cache(ref_hash, path): """ logging.info("creating new cache from reference {}...".format(ref_hash)) subprocess.run( - [test_path('gen_cache.sh'), ref_hash, path] + [test_path('gen_cache.sh'), ref_hash, path], + check=True, ) @@ -65,7 +118,7 @@ def load_cache(path): ref_hash = f.read().strip() else: ref_hash = None - logging.info("cache git hash: {}".format(ref_hash)) + logging.debug("cache hash: {}".format(ref_hash)) cache['ref_hash'] = ref_hash cache['ifos'] = {} for f in sorted(os.listdir(path)): @@ -213,31 +266,52 @@ def main(): help='specific ifos to test (default all)') args = parser.parse_args() + # get the current hash of HEAD + head_hash = git_ref_resolve_hash('HEAD') + if not head_hash: + logging.warning("could not determine git HEAD hash.") + + # update the reference if specified if args.update_ref: if args.update_ref == 'HEAD': - ref_hash = subprocess.run( - ['git', 'show', '-s', '--format=format:%H', 'HEAD'], - capture_output=True, universal_newlines=True, - ).stdout + if not head_hash: + sys.exit("Could not update reference to head.") + logging.info("updating reference to HEAD...") + ref_hash = head_hash else: - ref_hash = args.update_ref - logging.info("updating reference git hash to {}...".format(ref_hash)) - with open(test_path('ref_hash'), 'w') as f: - f.write('{}\n'.format(ref_hash)) + ref_hash = git_ref_resolve_hash(args.update_ref) + logging.info("updating reference git hash: {}".format(ref_hash)) + write_ref_hash(ref_hash) sys.exit() + # get the reference hash if args.git_ref: - ref_hash = args.git_ref - elif os.path.exists(test_path('ref_hash')): - with open(test_path('ref_hash')) as f: - ref_hash = f.read().strip() + ref_hash = git_ref_resolve_hash(args.git_ref) else: - sys.exit("Unspecified reference git hash, could not run test.") + ref_hash = load_ref_hash() + if not ref_hash: + pass + try: + with open(test_path('ref_hash')) as f: + ref_hash = f.read().strip() + except IOError: + logging.warning("could not open reference") + sys.exit("Unspecified reference git hash, could not run test.") + + logging.info("head hash: {}".format(head_hash)) + logging.info("ref hash: {}".format(ref_hash)) + + # don't bother test if hashes match + if ref_hash == head_hash: + logging.info("HEAD matches reference, not bothering to calculate.") + logging.info("Use --git-ref to compare against an arbitrary git commit.") + sys.exit() # load the cache cache_path = test_path('cache', ref_hash) if not os.path.exists(cache_path): - gen_cache(ref_hash, cache_path) + prune_cache_dir() + gen_cache_for_ref(ref_hash, cache_path) cache = load_cache(cache_path) if args.report: -- GitLab