.bash_profile

+# .bash_profile for deployment on LSC DataGrid clusters.
+# Run at the start of a login shell.
+
+. ~/.bashrc
+
+# Warn the user to go through GitLab for normal deployment changes.
+whiptail --yesno "The low-latency pipeline is continuously deployed through \
+GitLab. Please use the GitLab interface to make any software updates or \
+configuration changes or to start, stop, or restart the pipeline. For \
+details, refer to:
+
+https://gwcelery.readthedocs.io/en/latest/deployment.html
+
+Interactive terminal sessions are for EMERGENCY USE ONLY. \
+Are you sure that you want to proceed?" 15 70 --defaultno \
+--title 'FOR EMERGENCY USE ONLY' \
+--backtitle "${USER}@$(hostname --fqdn)" || logout
+
+# If Poetry is installed, then enable poetry shell completion.
+if type -P poetry &> /dev/null
+then
+    . <(poetry completions bash)
+fi

.bashrc

+# .bash_profile for deployment on LSC DataGrid clusters.
+# Run at the start of an interactive shell (including a login shell).
+
+# Source global definitions.
+if [ -f /etc/bashrc ]; then
+    . /etc/bashrc
+fi
+
+# Create log directories.
+mkdir -p $HOME/.local/state/log
+
+# Add user site directory to the PATH. On Linux, this is usuall ~/.local/bin.
+export PATH="$(python3.11 -m site --user-base)/bin${PATH+:${PATH}}"
+
+# Disable OpenMP, MKL, and OpenBLAS threading by default.
+# In this environment, it will be enabled selectively by processes that use it.
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export OPENBLAS_NUM_THREADS=1
+
+# Use mpich for parameter estimation.
+module load mpi/mpich-x86_64
+
+# Unless the user has set `GSSAPIDelegateCredentials no` in their ~/.ssh/config
+# file, their Globus certificate will be copied in when they log in, shadowing
+# the robot certificate. Set these environment variables to override.
+#
+# Note: according to SSH_CONFIG(5), the default for `GSSAPIDelegateCredentials`
+# is `no`. That's obviously incorrect!
+export X509_USER_CERT="$HOME/.globus/usercert.pem"
+export X509_USER_KEY="$HOME/.globus/userkey.pem"
+
+# Configuration for GWCelery web applications.
+export FLASK_RUN_PORT=5556
+export FLASK_URL_PREFIX=/gwcelery
+export FLOWER_PORT=5555
+export FLOWER_URL_PREFIX=/flower
+
+# GWCelery configuration-dependent instance variables.
+case "${USER}" in
+emfollow)
+    export CELERY_CONFIG_MODULE="gwcelery.conf.production"
+    ;;
+emfollow-playground)
+    export CELERY_CONFIG_MODULE="gwcelery.conf.playground"
+    ;;
+emfollow-test)
+    export CELERY_CONFIG_MODULE="gwcelery.conf.test"
+    ;;
+emfollow-dev)
+    export CELERY_CONFIG_MODULE="gwcelery.conf.dev"
+    ;;
+esac
+
+# HTGETTOKENOPTS for passing through to condor
+export HTGETTOKENOPTS="--vaultserver vault.ligo.org --issuer igwn --role read-cvmfs-${USER} --credkey read-cvmfs-${USER}/robot/${USER}.ligo.caltech.edu --nooidc"

.config/logrotate.conf

+
+"~/.local/state/log/*.log" {
+    compress
+    dateext
+    missingok
+    rotate 4
+    weekly
+}

.gitattributes

-gwcelery/_version.py export-subst

.gitignore

+# Project files for various editors
+.idea
+.vscode
+
+# Files created by editing, developing, running tests, building docs
+.DS_Store
 .*.swp
 *~
 *.egg-info
@@ -7,12 +13,45 @@ __pycache__
 .cache
 .coverage*
 .pytest_cache
+celerybeat-schedule.*
 build
+_build
 dist
 htmlcov
+
+# Files created by running gwcelery inside the source directory
+appendonly.aof
 celerybeat-schedule
 *.pid
-*.log
 *.rdb
 *.sock
-.env
+
+# Files created if the source directory is also the home directory
+public_html
+.astropy
+.bash_history
+.bash_logout
+.condor
+.config
+.cupy
+.emacs
+.globus/krb5.keytab
+.globus/usercert.pem
+.globus/userkey.pem
+.ipython
+.keras
+.kshrc
+.lesshst
+.ligo.em_bright
+.local
+.lvk-userguide
+.netrc
+.nv
+.python_history
+.rediscli_history
+.rnd
+.texlive2018
+.vim
+.viminfo
+.Xauthority
+read-cvmfs.keytab

.gitlab-ci.yml

+include:
+  - project: computing/gitlab-ci-templates
+    file: debian.yml
+  - project: computing/gitlab-ci-templates
+    file: macports.yml
+  - project: computing/gitlab-ci-templates
+    file: python.yml
+  - project: computing/gitlab-ci-templates
+    file: workflow/sccb.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml
+
 variables:
   # Allow Celery to run as root, because everything runs as root under Docker.
   C_FORCE_ROOT: 'true'
-  # Checking out the source is the exception rather than the rule because
-  # most of our CI jobs run from tarballs or wheels or don't require the source
-  # at all.
-  GIT_STRATEGY: none
+  # Poetry version to install
+  POETRY_VERSION: 2.0.1
 
 stages:
-  - dist
+  - build
   - test
+  - docker
   - deploy
 
 # Build source distribution
-sdist:
-  image: python:3.6-slim
-  stage: dist
-  variables:
-    GIT_STRATEGY: fetch
-  before_script:
-    # Versioneer will need access to the git tool.
-    - apt-get -q update
-    - apt-get -yq install --no-install-recommends git
-  script:
-    - python setup.py sdist
-    - mv dist/* .
-  artifacts:
-    paths:
-      - '*.tar.gz'
+build:
+  stage: build
+  extends: .python:build
+
+# Install redis
+.install-redis: &install-redis
+  apt-get install -y -qq redis-server
 
-# Build binary distribution
-bdist:
-  image: python:3.6-slim
+.test-common:
   stage: test
-  script:
-    - tar --strip-components 1 -xf *.tar.*
-    - python setup.py bdist_wheel
-    - mv dist/* .
-  dependencies:
-    - sdist
+  extends:
+    - .debian:base
+    - .python:pytest
+  variables:
+    PYTEST_OPTIONS: "--cov-report=html"
   artifacts:
     paths:
-      - '*.whl'
+      - htmlcov/
 
-# Build Docker container for dependencies
-.dependencies: &dependencies
-  stage: dist
+# Run test suite using poetry environment and locked dependencies
+.test-poetry:
+  extends: .test-common
+  before_script:
+    - !reference [".debian:base", before_script]
+    - !reference [".python:pytest", before_script]
+    - *install-redis
+    - python3 -m pip install pipx
+    - export PATH="/root/.local/bin:$PATH"
+    - pipx ensurepath
+    - pipx install poetry==$POETRY_VERSION
+    - pipx inject poetry poetry-dynamic-versioning
+    - poetry config virtualenvs.create false
+    # Check that poetry.lock is consistent with pyproject.toml
+    - poetry check --lock
+    - poetry install -E test
+  needs: []
+test/poetry/python3.11:
+  extends: .test-poetry
+  image: python:3.11
+test/poetry/python3.12:
+  extends: .test-poetry
+  image: python:3.12
+
+# Run test suite using wheel and bleeding-edge dependencies
+.test-wheel:
+  extends: .test-common
   variables:
-    GIT_STRATEGY: fetch
-    IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_JOB_NAME:$CI_COMMIT_REF_NAME
-  script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - |
-      cat <<EOF > Dockerfile
-      FROM python:${CI_JOB_NAME#*python}
-      COPY requirements.txt .
-      RUN pip --no-cache-dir install -r requirements.txt
-      RUN rm -f requirements.txt
-      EOF
-    - docker build -t $IMAGE_TAG .
-    - docker push $IMAGE_TAG
-    - if [ "${CI_COMMIT_TAG:0:1}" = "v" ]; then docker tag $IMAGE_TAG ${IMAGE_TAG%:*}:latest; docker push ${IMAGE_TAG%:*}:latest; fi
-dependencies/python3.6:
-  <<: *dependencies
+    PYTEST_OPTIONS: "--pyargs $CI_PROJECT_NAME"
+  before_script:
+    - INSTALL_TARGET=$(echo *.whl)[test]
+    - !reference [".debian:base", before_script]
+    - !reference [".python:pytest", before_script]
+    - *install-redis
+  needs:
+    - build
+test/wheel/python3.11:
+  extends: .test-wheel
+  image: python:3.11
+test/wheel/python3.12:
+  extends: .test-wheel
+  image: python:3.12
 
-# Run unit tests and coverage measurement
-.test: &test
+.test-macos:
   stage: test
-  coverage: '/^TOTAL\s+.*\s+(\d+\.?\d*)%/'
-  before_script:
-    - apt-get update -qq && apt-get install -y -qq redis-server
-  script:
-    - tar --strip-components 1 -xf *.tar.*
-    - pip install pytest-cov
-    - python setup.py test --addopts='-vv --cov --cov-report=html --cov-report=term'
-  dependencies:
-    - sdist
+  extends:
+    - .python:pytest
+  variables:
+    PYTEST_OPTIONS: "--cov-report=html"
   artifacts:
     paths:
       - htmlcov/
-test/python3.6:
-  image: $CI_REGISTRY_IMAGE/dependencies/python3.6:$CI_COMMIT_REF_NAME
-  <<: *test
+  before_script:
+    - . /opt/local/share/macports/setupenv.bash
+    - python3.11 -m venv env
+    - source env/bin/activate
+    - pip install poetry poetry-dynamic-versioning
+    - poetry config virtualenvs.create false
+    # Check that poetry.lock is consistent with pyproject.toml
+    - poetry check --lock
+    - poetry install -E test
+  needs: []
+
+test/poetry/python3.11/macos/x86_64:
+  extends: .test-macos
+  tags:
+    - macos_monterey_x86_64
+  # FIXME: macOS x86_64 runner is flaky.
+  # https://git.ligo.org/computing/helpdesk/-/issues/6922
+  allow_failure: true
+
+test/poetry/python3.11/macos/arm64:
+  extends: .test-macos
+  tags:
+    - macos_sequoia_arm64
 
 lint:
-  image: $CI_REGISTRY_IMAGE/dependencies/python3.6:$CI_COMMIT_REF_NAME
   stage: test
+  extends: .python:flake8
+  needs: []
+  variables:
+    REQUIREMENTS: flake8-isort
+
+associate commits in Sentry:
+  stage: test
+  needs:
+    - build
+  only:
+    - main@emfollow/gwcelery
+    - tags@emfollow/gwcelery
+  image:
+    name: getsentry/sentry-cli
+    entrypoint: [""]
   script:
-    - tar --strip-components 1 -xf *.tar.*
-    - pip install flake8 'pep8-naming!=0.8.0'
-    - flake8 --show-source .
-  dependencies:
-    - sdist
-
-# Build docker container for application itself
-.docker: &docker
-  stage: deploy
-  script:
-    - IMAGE_TAG=$CI_REGISTRY_IMAGE/${CI_JOB_NAME#*/}:$CI_COMMIT_REF_NAME
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - |
-      cat <<EOF > Dockerfile
-      FROM $CI_REGISTRY_IMAGE/dependencies/${CI_JOB_NAME#*/}:$CI_COMMIT_REF_NAME
-      COPY *.whl .
-      RUN pip install *.whl
-      RUN rm -f *.whl
-      USER nobody
-      WORKDIR /tmp
-      ENTRYPOINT ["gwcelery"]
-      EOF
-    - docker build -t $IMAGE_TAG .
-    - docker push $IMAGE_TAG
-    - if [ "${CI_COMMIT_TAG:0:1}" = "v" ]; then docker tag $IMAGE_TAG ${IMAGE_TAG%:*}:latest; docker push ${IMAGE_TAG%:*}:latest; fi
-  dependencies:
-    - bdist
-docker/python3.6:
-  <<: *docker
+    - SENTRY_VERSION=$(echo *.tar.* | sed 's/\.tar\..*//')
+    - sentry-cli releases new ${SENTRY_VERSION}
+    - sentry-cli releases set-commits --auto ${SENTRY_VERSION}
 
 # Generate documentation
 doc:
-  image: python:3.6-slim
   stage: test
+  extends:
+    - .debian:base
+    - .python:sphinx
+  image: python:3.11
+  variables:
+    SOURCEDIR: doc
+    REQUIREMENTS: .[doc]
   before_script:
-    - apt-get update -qq && apt-get install -y -qq graphviz
+    - !reference [".debian:base", before_script]
+    - !reference [".python:sphinx", before_script]
+    - apt-get install -y -qq graphviz
+  needs: []
+
+# Create Docker image
+docker:
+  stage: build
   script:
-    - tar --strip-components 1 -xf *.tar.*
-    - python setup.py build_sphinx
-  dependencies:
-    - sdist
+    - IMAGE_TAG=$CI_REGISTRY_IMAGE:${CI_COMMIT_REF_NAME/\//-} # Replace "/" in release branch names with "-"
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build -t $IMAGE_TAG .
+    - docker push $IMAGE_TAG
+    - |
+      if [ "${CI_COMMIT_TAG:0:1}" = "v" ]
+      then
+        docker tag $IMAGE_TAG ${IMAGE_TAG%:*}:latest
+        docker push ${IMAGE_TAG%:*}:latest
+      fi
+  needs: []
+
+# SCCB request
+sccb:
+  stage: deploy
+  when: manual
+  only:
+    - tags@emfollow/gwcelery
+  before_script:
+    # poetry-dynamic-versioning needs to be installed to properly
+    # update the version information in pyproject.toml
+    - python -m pip install --upgrade pip
+    - python -m pip install pipx
+    - pipx ensurepath
+    - pipx install poetry==$POETRY_VERSION
+    - pipx inject poetry poetry-dynamic-versioning
+  needs: []
 
 # Publish coverage
 pages:
@@ -138,98 +197,235 @@ pages:
       - public
     expire_in: 30 days
   only:
-    - master
-  dependencies:
-    - test/python3.6
+    - main
+  needs:
+    - test/poetry/python3.11
 
-# Upload package to PyPI
+# Upload package to PyPI.
+# Place your PyPI API token in the repository's GitLab CI secrets.
 pypi:
   stage: deploy
-  image: python:3.6-slim
+  image: python:slim
   script:
-    - pip install twine
-    - twine upload *.tar.* *.whl
-  dependencies:
-    - sdist
-    - bdist
+    - python -m pip install twine
+    - twine upload --username __token__ --password $PYPI_API_TOKEN *.tar.* *.whl
+  needs:
+    - build
   only:
     - tags@emfollow/gwcelery
 
+# Create a release in GitLab
+release:
+  stage: deploy
+  image: python
+  variables:
+    GIT_STRATEGY: fetch
+  script:
+    - python -m pip install --upgrade pip poetry poetry-dynamic-versioning
+    - PACKAGE_NAME="$(basename $PWD)"
+    - PACKAGE_VERSION="$(poetry version --short)"
+    - CHANGES_FILENAME="$(echo CHANGES.*)"
+    - |
+      tee json <<EOF
+      {
+        "name": "${PACKAGE_VERSION}",
+        "tag_name": "${CI_COMMIT_TAG}",
+        "description": "Version ${PACKAGE_VERSION}",
+        "assets": {
+          "links": [
+            {
+              "name": "PyPI",
+              "url": "https://pypi.org/project/${PACKAGE_NAME}/${PACKAGE_VERSION}/"
+            },
+            {
+              "name": "Change log",
+              "url": "${CI_PROJECT_URL}/blob/v${VERSION}/${CHANGES_FILENAME}"
+            },
+            {
+              "name": "Documentation",
+              "url": "https://${PACKAGE_NAME}.readthedocs.io/en/v${PACKAGE_VERSION}/"
+            }
+          ]
+        }
+      }
+      EOF
+    - |
+      if [ -n "$CI_COMMIT_TAG" ]
+      then
+        curl --silent --show-error --fail \
+        --request POST --header "Private-Token: $GITLAB_ACCESS_TOKEN" \
+        --header "Content-Type: application/json" \
+        --data "@json" \
+        "$CI_API_V4_URL/projects/$CI_PROJECT_ID/releases"
+      fi
+
 .deploy-common: &deploy-common
+  image: containers.ligo.org/emfollow/ssh-kerberos
   stage: deploy
-  image: kroniak/ssh-client
   only:
-    - master@emfollow/gwcelery
+    - branches@emfollow/gwcelery
     - tags@emfollow/gwcelery
-  dependencies: []
-  before_script:
-    # Configure ssh known_hosts file and public/private key pair.
-    - mkdir -p ~/.ssh
-    - echo 'emfollow.ligo.caltech.edu,131.215.113.150 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAVKq+Y7sXfJVSEekUyw35FMUHiPgkryogaxMM0bsd2F4ZfXcH4x6HKQ+eByAJV53WeN9p17j+y69sV0/xNwjZc=' >> ~/.ssh/known_hosts
-    - echo -e "${ID_RSA}" > ~/.ssh/id_rsa
-    - echo "${ID_RSA_PUB}" > ~/.ssh/id_rsa.pub
-    - chmod 0600 ~/.ssh/id_rsa ~/.ssh/id_rsa.pub
-
-# Deploy on Caltech cluster
+  needs: []
+
+# Continuous deployment.
+# Note that all deployments except playground/CIT are manual.
+
 .deploy-start: &deploy-start
   <<: *deploy-common
-  script: |
-    ssh -v -T emfollow@emfollow.ligo.caltech.edu bash <<EOF
-    set -e
-    mkdir -p ${CI_ENVIRONMENT_NAME} && cd ${CI_ENVIRONMENT_NAME}
-    if [ -d .git ]; then git fetch; else git clone ${CI_PROJECT_URL} .; fi
-    git checkout $CI_COMMIT_SHA
-    echo CELERY_BROKER_URL=redis+socket://\${PWD}/redis.sock > .env
-    echo CELERY_CONFIG_MODULE=gwcelery.conf.${CI_ENVIRONMENT_NAME} >> .env
-    echo FLOWER_URL_PREFIX=/~\${USER}/${CI_ENVIRONMENT_NAME}/flower >> .env
-    echo FLASK_URL_PREFIX=/~\${USER}/${CI_ENVIRONMENT_NAME}/gwcelery >> .env
-    echo FLOWER_PORT=${FLOWER_PORT} >> .env
-    echo FLASK_PORT=${FLASK_PORT} >> .env
-    pipenv sync
-    pipenv run gwcelery condor resubmit
-    EOF
+  script:
+    # Update the repository.
+    - |
+      ssh -T $REMOTE_HOST bash <<EOF
+      set -e
+      if [ ! -d .git ]; then git init; fi
+      git fetch --tags ${CI_REPOSITORY_URL} $CI_COMMIT_SHA
+      git checkout -f $CI_COMMIT_SHA
+      EOF
+    # Reinstall and start (new session to reload bashrc).
+    - |
+      ssh -T $REMOTE_HOST bash <<EOF
+      . .bashrc
+      set -e
+      python3.11 -m ensurepip --upgrade
+      python3.11 -m pip install pipx
+      pipx ensurepath
+      pipx install poetry==$POETRY_VERSION
+      pipx inject poetry poetry-dynamic-versioning
+      # FIXME: It would be simpler to use `poetry install`, but it installs the
+      # project in editable mode, which is not appropriate for production
+      # deployments.
+      python3.11 -m pip install --no-deps -r <(poetry export --with dev)
+      python3.11 -m pip install --no-deps .
+      # FIXME: remove pip uninstall temporary directories left behind due to
+      # bug with pip on NFS. See https://github.com/pypa/pip/issues/6327
+      chmod -R ug+rwx .local/lib/python*/site-packages/~* || true
+      rm -rf .local/lib/python*/site-packages/~* || true
+      gwcelery condor resubmit
+      EOF
 
-# Deploy on Caltech cluster
 .deploy-stop: &deploy-stop
   <<: *deploy-common
   script: |
-    ssh -v -T emfollow@emfollow.ligo.caltech.edu bash <<EOF
+    ssh -T $REMOTE_HOST bash <<EOF
+    . .bashrc
     set -e
-    cd ${CI_ENVIRONMENT_NAME}
-    pipenv run gwcelery condor rm
+    gwcelery condor rm
     EOF
 
-deploy to production:
+deploy to production at CIT:
   <<: *deploy-start
+  when: manual
+  environment:
+    name: production/CIT
+    on_stop: stop deployment on production at CIT
+    url: https://emfollow.ligo.caltech.edu/flower
+  variables:
+    REMOTE_HOST: emfollow@emfollow.ligo.caltech.edu
+
+deploy to production at LHO:
+  <<: *deploy-start
+  when: manual
+  environment:
+    name: production/LHO
+    on_stop: stop deployment on production at LHO
+    url: https://emfollow.ligo-wa.caltech.edu/flower
+  variables:
+    REMOTE_HOST: emfollow@emfollow.ligo-wa.caltech.edu
+
+deploy to playground at CIT:
+  <<: *deploy-start
+  when: manual
+  environment:
+    name: playground/CIT
+    on_stop: stop deployment on playground at CIT
+    url: https://emfollow-playground.ligo.caltech.edu/flower
+  variables:
+    REMOTE_HOST: emfollow-playground@emfollow-playground.ligo.caltech.edu
+
+deploy to playground at LHO:
+  <<: *deploy-start
+  when: manual
+  environment:
+    name: playground/LHO
+    on_stop: stop deployment on playground at LHO
+    url: https://emfollow-playground.ligo-wa.caltech.edu/flower
+  variables:
+    REMOTE_HOST: emfollow-playground@emfollow-playground.ligo-wa.caltech.edu
+
+deploy to test at CIT:
+  <<: *deploy-start
+  environment:
+    name: test/CIT
+    on_stop: stop deployment on test at CIT
+    url: https://emfollow-test.ligo.caltech.edu/flower
   variables:
-    FLOWER_PORT: '5555'
-    FLASK_PORT: '5556'
+    REMOTE_HOST: emfollow-test@emfollow-test.ligo.caltech.edu
+
+deploy to test at LHO:
+  <<: *deploy-start
   when: manual
   environment:
-    name: production
-    on_stop: stop deployment on production
-    url: https://ldas-jobs.ligo.caltech.edu/~emfollow/production/flower
+    name: test/LHO
+    on_stop: stop deployment on test at LHO
+    url: https://emfollow-test.ligo-wa.caltech.edu/flower
+  variables:
+    REMOTE_HOST: emfollow-test@emfollow-test.ligo-wa.caltech.edu
 
-deploy to playground:
+deploy to dev at CIT:
   <<: *deploy-start
+  when: manual
+  environment:
+    name: dev/CIT
+    on_stop: stop deployment on dev at CIT
+    url: https://emfollow-dev.ligo.caltech.edu/flower
   variables:
-    FLOWER_PORT: '5557'
-    FLASK_PORT: '5558'
+    REMOTE_HOST: emfollow-dev@emfollow-dev.ligo.caltech.edu
+
+stop deployment on production at CIT:
+  <<: *deploy-stop
+  when: manual
   environment:
-    name: playground
-    on_stop: stop deployment on playground
-    url: https://ldas-jobs.ligo.caltech.edu/~emfollow/playground/flower
+    name: production/CIT
+    action: stop
+
+stop deployment on production at LHO:
+  <<: *deploy-stop
+  when: manual
+  environment:
+    name: production/LHO
+    action: stop
+
+stop deployment on playground at CIT:
+  <<: *deploy-stop
+  when: manual
+  environment:
+    name: playground/CIT
+    action: stop
+
+stop deployment on playground at LHO:
+  <<: *deploy-stop
+  when: manual
+  environment:
+    name: playground/LHO
+    action: stop
+
+stop deployment on test at CIT:
+  <<: *deploy-stop
+  when: manual
+  environment:
+    name: test/CIT
+    action: stop
 
-stop deployment on production:
+stop deployment on test at LHO:
   <<: *deploy-stop
   when: manual
   environment:
-    name: production
+    name: test/LHO
     action: stop
 
-stop deployment on playground:
+stop deployment on dev at CIT:
   <<: *deploy-stop
   when: manual
   environment:
-    name: playground
+    name: dev/CIT
     action: stop

.gitlab/issue_templates/AddNewExternalPipeline.md

+# Checklist
+
+## Policy
+
+- [ ] Determine whether the new experiment falls within an existing search, see this [documentation](https://gracedb.ligo.org/documentation/models.html#data-models) for a comprehensive list.
+- [ ] Determine which (if any) joint FAR method to use, either [untargeted](https://ligo-raven.readthedocs.io/en/latest/joint_far.html#untargeted-search-method) or [targeted](https://ligo-raven.readthedocs.io/en/latest/joint_far.html#targeted-search-method). The untargeted method is often used with highly significant external events (enough to be published based on their own sigificance) while the targeted method is often used with less significant events that have their own FARs available.
+- [ ] Determine what the relevant rates will be, either the *independent* rates of detection (i.e. not detected by any other experiment in the same search) or max FAR thresholds (both GW and external). See the following for examples: [adding subthreshold GRBs](https://dcc.ligo.org/cgi-bin/private/DocDB/ShowDocument?docid=T1900297); [Adding SVOM GRBs to the list of existing GRBs](https://dcc.ligo.org/LIGO-T2400116)
+- [ ] Determine which coincidence searches will need to be set up, i.e. which type(s) of GW candidate(s) associations we are looking for. Examples include CBCs and unmodeled GW Bursts created from supernovae.
+- [ ] If a new coincidence search, determine what the relevant new time window(s) should be. These should ideally be based on delay models. If the underlying systems that could produce such a coincidence are not as well understood, then a practical time window could be chosen that is roughly what we could expect (e.g. the CBC-FRB time windows are double that of CBC-GRB due to our ignorance of such systems). Otherwise use the time windows of the existing search.
+- [ ] Determine whether sky maps or sky map information (RA/dec/error radius) will be available. Determine whether these will be flattened or Multi-Order Coverage (MOC).
+- [ ] If a new search, determine whether there should be differences in considering which external event should be preferred (currently SNEWS > threshold > subthreshold, with tie breakers being whether the `NOT_GRB` label is present and the joint FAR).
+- [ ] Determine whether there are any special conditions to publish this event beyond the current conditions (see `raven.trigger_raven_alert`)
+
+## GraceDB
+
+- [ ] If the `search` and/or `pipeline` names do not yet exist in GraceDB, open an issue on [GraceDB Server](https://git.ligo.org/computing/gracedb/server/-/issues/new) and work with the developers to add this. This can be checked via the [documentation](https://gracedb.ligo.org/documentation/models.html#data-models).
+- [ ] If the VOEvent packet differs from previous examples and is not ingestible, open an issue on [GraceDB Server](https://git.ligo.org/computing/gracedb/server/-/issues/new) and work with the developers to fix this. *Note* we currently convert JSON packets to VOEvent in gwcelery, so until this is natively supported in GraceDB, an existing function in gwcelery [_kafka_to_voevent](https://git.ligo.org/emfollow/gwcelery/-/blob/218d378e665e5397677854b43f5e6719c42e0036/gwcelery/tasks/external_triggers.py#L726) may need to be modified (or a new equivalent function written if too different).
+
+## GWCelery
+
+- [ ] Add/modify the policy-determined variables to the [configuration file](https://git.ligo.org/emfollow/gwcelery/-/blob/main/gwcelery/conf/__init__.py), next to each relevant existing variable if applicable. This also includes pipeline variables used in other applications like detchar omegascans, etc.
+- [ ] If an existing search, add to the relevant listener. Add any unique differences compared to previous searches, especially if there are ways to either filter out events or mark them to not be considered in the search with the `NOT_GRB` label. If the search differs significantly from existing or different functionality is needed, create a new listener. Add logic to download sky maps if available or create sky maps if RA/dec/error is given by choosing the arguments for `external_triggers._create_replace_external_event_and_skymap`.
+- [ ] Do the same for the IGWN alert listener. Add `raven.coincidence_search`s determined in the above policy section.
+- [ ] If using Kafka, modify the credential map (at `~/.config/gwcelery/kafka_credential_map.json`) on each server (`production`, `playground`, `test`, `dev`) to include the key/credential pair for each new notice type.
+- [ ] If using ligo-raven < 4.0, modify `raven.calculate_coincidence_far` to get the correct joint FAR method by overriding the `ext_search` if necessary. If using ligo-raven >= 4.0, map to the correct `joint_far_method`.
+- [ ] If changing or adding time windows, modify `raven._time_window`.
+- [ ] If changing how the preferred external event is calculated, modify `raven.update_coinc_far` accordingly.
+- [ ] If change publishing conditions, modify `raven.trigger_raven_alert`.
+- [ ] If creating sky maps from RA/dec/error, add to list of pipelines in `external_skymaps.py`.
+- [ ] Add both O3 replay and MDC testing in [`first2years_external.py`](https://git.ligo.org/emfollow/gwcelery/-/blob/main/gwcelery/tasks/first2years_external.py).
+- [ ] Add [tests](https://git.ligo.org/emfollow/gwcelery/-/tree/main/gwcelery/tests) for all of the changes made. Ensure 100% testing coverage moving forward.
+
+/label ~External Triggers

.gitlab/issue_templates/CreateARelease.md

+**Git ref**: *insert git reference here*
+
+# Checklist
+
+## Basics
+
+1.  [ ] The CI pipeline succeeded, including all unit tests and code quality checks. *place link to pipeline here*
+2.  [ ] [CHANGES.rst](CHANGES.rst) lists all significant changes since the last release. It is free from spelling and grammatical errors.
+3.  [ ] The [latest Readthedocs documentation build](https://readthedocs.org/projects/gwcelery/builds/) passed and the [latest docs](https://gwcelery.readthedocs.io/en/latest/) are correctly rendered. Autodoc-generated API docs for tasks are shown.
+4.  [ ] If there is [milestone](https://git.ligo.org/emfollow/gwcelery/-/milestones) for this
+    release, then the list of issues and merge requests that have been
+    addressed is accurate. Any unaddressed issues and merge requests have been
+    moved to another milestone.
+5.  [ ] Check the versions of the following packages in the [`poetry.lock`](https://git.ligo.org/emfollow/gwcelery/-/blob/main/poetry.lock) file have been approved by the SCCB (i.e. either has the status:deploy or status:deployed label).
+    - [ ] [`bilby`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=bilby&first_page_size=100)
+    - [ ] [`bilby_pipe`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=bilby_pipe&first_page_size=100)
+    - [ ] [`gracedb-sdk`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=gracedb-sdk&first_page_size=100)
+    - [ ] [`gwdatafind`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=gwdatafind&first_page_size=100)
+    - [ ] [`gwpy`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=gwpy&first_page_size=100)
+    - [ ] [`gwskynet`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=gwskynet&first_page_size=100)
+    - [ ] [`igwn-alert`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=igwn-alert&first_page_size=100)
+    - [ ] [`igwn-alert-schema`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=igwn-gwalert-schema&first_page_size=20)
+    - [ ] [`lalsuite`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=lalsuite&first_page_size=100)
+    - [ ] [`ligo-followup-advocate`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=ligo-followup-advocate&first_page_size=100)
+    - [ ] [`ligo-gracedb`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=ligo-gracedb&first_page_size=100)
+    - [ ] [`ligo-raven`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=ligo-raven&first_page_size=100)
+    - [ ] [`ligo-segments`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=ligo-segments&first_page_size=20)
+    - [ ] [`ligo.em-bright`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=ligo.em-bright&first_page_size=20)
+    - [ ] [`ligo.skymap`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=ligo.skymap&first_page_size=100)
+    - [ ] [`lscsoft-glue`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=lscsoft-glue&first_page_size=100)
+    - [ ] [`pesummary`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=pesummary&first_page_size=100)
+    - [ ] [`python-ligo-lw`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=python-ligo-lw&first_page_size=100)
+    - [ ] [`rapidpe`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=Rapidpe&first_page_size=20)
+    - [ ] [`rapidpe-rift-pipe`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=RapidPE%20pipeline&first_page_size=20)
+    - [ ] [`RIFT`](https://git.ligo.org/computing/sccb/-/issues/?sort=updated_desc&state=all&search=rift&first_page_size=100)
+
+## Test deployment
+
+4.  [ ] Sentry does not show any new [unresolved issues on test](https://sentry.io/organizations/ligo-caltech/issues/?environment=test&groupStatsPeriod=14d&project=1425216&query=is%3Aunresolved&statsPeriod=14d) that indicate new bugs or regressions.
+5.  [ ] The test deployment has run for at least 10 minutes.
+6.  [ ] The [Flower monitor](https://emfollow-test.ligo.caltech.edu/flower) is reachable and shows no unexpected task failures.
+7.  [ ] The [Flask dashboard](https://emfollow-test.ligo.caltech.edu/gwcelery) is reachable.
+8.  [ ] The test deployment is [connected to IGWN Alert](https://emfollow-test.ligo.caltech.edu/flower/worker/gwcelery-worker%40emfollow-test.ligo.caltech.edu#tab-other) (in Flower, find the main gwcelery-worker, click Other, and look at the list of subscribed IGWN Alert topics).
+9.  [ ] The test deployment is [connected to GCN](https://emfollow-test.ligo.caltech.edu/flower/worker/gwcelery-voevent-worker%40emfollow-test.ligo.caltech.edu#tab-other) (in Flower, find the voevent gwcelery-worker, click Other, and look at the list of receiver peers).
+
+## Mock events
+
+10. [ ] The test deployment has [produced an MDC superevent](https://gracedb-test.ligo.org/latest/?query=MDC&query_type=S).
+11. [ ] The MDC superevent has the following annotations.
+    - [ ] `bayestar.multiorder.fits`
+    - [ ] `bayestar.fits.gz`
+    - [ ] `bayestar.png`
+    - [ ] `bayestar.volume.png`
+    - [ ] `bayestar.html`
+    - [ ] `p_astro.json`
+    - [ ] `p_astro.png`
+    - [ ] `em_bright.json`
+    - [ ] `em_bright.png`
+12. [ ] The MDC superevent has the following labels.
+    - [ ] `EMBRIGHT_READY`
+    - [ ] `GCN_PRELIM_SENT`
+    - [ ] `PASTRO_READY`
+    - [ ] `SKYMAP_READY`
+13. [ ] The MDC superevent has two automatic preliminary VOEvents, JSON packets, and Avro packets if `GCN_PRELIM_SENT` is applied.
+    - [ ] 2 preliminary VOEvents
+    - [ ] 2 preliminary JSON packets
+    - [ ] 2 preliminary Avro packets
+14. [ ] Issuing a manual preliminary alert from the [Flask dashboard](https://emfollow-test.ligo.caltech.edu/gwcelery) sends another preliminary alert.
+    - [ ] The alert **is sent** successfully if `ADVOK` or an `ADVNO` label is **not applied** this time.
+    - [ ] Alternatively, a preliminary alert is **blocked** due to presence of `ADVOK` or `ADVNO`.
+15. [ ] `DQR_REQUEST` label is applied to the superevent. The application happens at the time of launching the second preliminary alert.
+16. [ ] The MDC superevent has either an `ADVOK` or an `ADVNO` label.
+17. [ ] Issuing an `ADVOK` signoff through GraceDB results in an initial VOEvent.
+18. [ ] Issuing an `ADVNO` signoff through GraceDB results in a retraction VOEvent.
+19. [ ] Requesting an update alert through the [Flask dashboard](https://emfollow-test.ligo.caltech.edu/gwcelery) results in an update VOEvent.
+20. [ ] Test has recently [produced an MDC superevent with an external coincidence](https://gracedb-test.ligo.org/latest/?query=MDC+EM_COINC&query_type=S), i.e. with an `EM_COINC` label. Use the [Flask dashboard](https://emfollow-test.ligo.caltech.edu/gwcelery) to do this manually (note that joint events with Swift may not pass publishing conditions and or have a combined sky map, indicated by the lack of `RAVEN_ALERT` and `COMBINEDSKYMAP_READY` label respectively).
+21. [ ] The joint MDC superevent has the following annotations.
+    - [ ] `coincidence_far.json`
+    - [ ] `combined-ext.multiorder.fits` or `combined-ext.fits.gz`
+    - [ ] `combined-ext.png`
+    - [ ] `overlap_integral.png`
+22. [ ] The joint MDC superevent has the following labels.
+    - [ ] `EM_COINC`
+    - [ ] `RAVEN_ALERT`
+    - [ ] `COMBINEDSKYMAP_READY`
+    - [ ] `GCN_PRELIM_SENT`
+23. [ ] The joint MDC superevent is sending alerts with coincidence information.
+    - [ ] At least one VOEvent with `<Group name="External Coincidence">`.
+    - [ ] At least one Kafka JSON packet with an `external_coinc` field.
+    - [ ] At least one circular w/ `-emcoinc-` in filename.
+24. [ ] Issue a manual RAVEN alert using the [Flask dashboard](https://emfollow-test.ligo.caltech.edu/gwcelery) for a coincidence (i.e. has `EM_COINC` label) that has does not have the `RAVEN_ALERT` label yet. Choose a [recent joint coincidence that meets this criteria](https://gracedb-test.ligo.org/latest/?query=MDC+%7ERAVEN_ALERT+%26+EM_COINC&query_type=S&get_neighbors=&results_format=) and ensure that a `RAVEN_ALERT` label is applied to the associated superevent, external event, and preferred event.
+
+## Replay events
+
+24. [ ] [A Production superevent labeled `GCN_PRELIM_SENT`](https://gracedb-test.ligo.org/latest/?query=Production+GCN_PRELIM_SENT&query_type=S&get_neighbors=&results_format=) has the following parameter estimation annotations and the `PE_READY` label.
+    - [ ] `bilby_config.ini`
+    - [ ] `Bilby.posterior_samples.hdf5`
+    - [ ] `Bilby.multiorder.fits`
+    - [ ] `Bilby.html`
+    - [ ] `Bilby.fits.gz`
+    - [ ] `Bilby.png`
+    - [ ] `Bilby.volume.png`
+    - [ ] `PE_READY`
+    - [ ] Link to PEsummary page (log message in parameter estimation section)
+
+/label ~Release

.k5login

+emfollow-deployment/robot/runners.ligo.uwm.edu@LIGO.ORG

.readthedocs-requirements.txt

+poetry-dynamic-versioning

.readthedocs.yml

-requirements_file: docs-requirements.txt
+version: 2
+
+python:
+  install:
+    - requirements: .readthedocs-requirements.txt
+    - path: .
+      extra_requirements:
+        - doc
+
+build:
+  apt_packages:
+    - graphviz
+    - libkrb5-dev
+  os: ubuntu-22.04
+  tools:
+    python: '3.11'
+
+sphinx:
+  configuration: doc/conf.py

Dockerfile

+# build stage: build wheel + extract dependencies
+FROM python:3.11 as build
+
+ENV PIP_DEFAULT_TIMEOUT=100
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+ENV PIP_NO_CACHE_DIR=1
+ENV POETRY_VERSION=1.8.1
+
+WORKDIR /app
+
+COPY . ./
+
+RUN pip install --upgrade pip
+RUN pip install \
+    poetry==$POETRY_VERSION \
+	poetry-dynamic-versioning
+
+RUN poetry export -f requirements.txt -o requirements.txt
+RUN poetry build -f wheel
+
+# install stage: install gwcelery + dependencies
+FROM python:3.11 as install
+
+ENV PIP_DEFAULT_TIMEOUT=100
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+ENV PIP_NO_CACHE_DIR=1
+
+# install kerberos for ciecplib
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        libkrb5-dev && \
+	rm -rf /var/lib/apt /var/lib/dpkg /var/lib/cache /var/lib/log
+
+WORKDIR /app
+
+COPY --from=build /app/requirements.txt /app
+COPY --from=build /app/dist/*.whl /app
+
+# create virtualenv
+RUN python -m venv /opt/venv
+
+# ensure virtualenv python is preferred
+ENV PATH="/opt/venv/bin:$PATH"
+
+run pip install --upgrade pip
+RUN pip install -r requirements.txt
+RUN pip install --no-deps *.whl
+
+# final stage: installed app
+FROM python:3.11-slim as app
+
+COPY --from=install /opt/venv /opt/venv
+
+# install kerberos for ciecplib
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        libkrb5-dev && \
+	rm -rf /var/lib/apt /var/lib/dpkg /var/lib/cache /var/lib/log
+
+# create app user
+RUN groupadd 1000 && \
+    useradd --uid 1000 --gid 1000 -m gwcelery
+
+USER gwcelery
+
+WORKDIR /home/gwcelery
+
+# ensure virtualenv python is preferred
+ENV PATH="/opt/venv/bin:$PATH"
+
+# disable OpenMP, MKL, and OpenBLAS threading by default
+# it will be enabled selectively by processes that use it
+ENV OMP_NUM_THREADS=1
+ENV MKL_NUM_THREADS=1
+ENV OPENBLAS_NUM_THREADS=1
+
+# web application configuration
+ENV FLASK_RUN_PORT=5556
+ENV FLASK_URL_PREFIX=/gwcelery
+ENV FLOWER_PORT=5555
+ENV FLOWER_URL_PREFIX=/flower
+
+# set dev configuration by default
+ENV CELERY_CONFIG_MODULE=gwcelery.conf.dev
+
+ENTRYPOINT ["gwcelery"]

LICENSE.rst

@@ -5,7 +5,7 @@ License
 The `GWCelery Logo`_ is a composite of `Celery2`_ by Tiia Monto and `Lorentzian
 Wormhole`_ by Kes47 from Wikimedia Commons (`CC BY-SA 3.0`_).
 
-.. _`GWCelery Logo`: https://git.ligo.org/emfollow/gwcelery/blob/master/doc/_static/logo.png
+.. _`GWCelery Logo`: https://git.ligo.org/emfollow/gwcelery/blob/main/doc/_static/logo.png
 .. _`Celery2`: https://commons.wikimedia.org/wiki/File:Celery2.png
 .. _`Lorentzian Wormhole`: https://commons.wikimedia.org/wiki/File:Lorentzian_Wormhole.svg
 .. _`CC BY-SA 3.0`: https://creativecommons.org/licenses/by-sa/3.0/legalcode
@@ -351,4 +351,4 @@ Wormhole`_ by Kes47 from Wikimedia Commons (`CC BY-SA 3.0`_).
     proprietary programs.  If your program is a subroutine library, you may
     consider it more useful to permit linking proprietary applications with the
     library.  If this is what you want to do, use the GNU Lesser General
-    Public License instead of this License.
\ No newline at end of file
+    Public License instead of this License.

MANIFEST.in

-graft doc
-include *requirements.txt
-include versioneer.py
-include gwcelery/_version.py

Pipfile

-[[source]]
-name = "pypi"
-url = "https://pypi.org/simple"
-verify_ssl = true
-
-[packages]
-# Our package and all of its dependencies.
-gwcelery = {editable = true,path = "."}
-# Include IPython for a nicer ``gwcelery shell`` experience.
-ipython = "*"
-# Pull in flower from GitHub branch to pull in extra features.
-flower = {git = "https://github.com/lpsinger/flower",ref = "custom-stats"}
-
-[requires]
-python_version = "3.6"

README.rst

@@ -17,8 +17,8 @@ Features
 - `Easy installation with pip <https://gwcelery.readthedocs.io/en/latest/quickstart.html>`_
 - Lightning fast distributed task queue powered by
   `Celery <http://celeryproject.org>`_ and `Redis <https://redis.io>`_
-- Tasks are defined by `small, self-contained Python functions <https://git.ligo.org/emfollow/gwcelery/tree/master/gwcelery/tasks>`_
-- `Lightweight test suite <https://git.ligo.org/emfollow/gwcelery/tree/master/gwcelery/tests>`_ using mocks of external services
+- Tasks are defined by `small, self-contained Python functions <https://git.ligo.org/emfollow/gwcelery/tree/main/gwcelery/tasks>`_
+- `Lightweight test suite <https://git.ligo.org/emfollow/gwcelery/tree/main/gwcelery/tests>`_ using mocks of external services
 - `Continuous integration <https://git.ligo.org/emfollow/gwcelery/pipelines>`_
 - `One environment variable to switch from playground to production GraceDB server <https://gwcelery.readthedocs.io/en/latest/configuration.html>`_
 - `Browser-based monitoring console <https://gwcelery.readthedocs.io/en/latest/monitoring.html>`_

cron.sh

+#!/usr/bin/bash
+# Perform hourly maintenance activities.
+
+# Stop on error.
+set -e
+
+# This script is designed to be run by cron.
+# Set up the same environemnt variables as for a login shell.
+source "$HOME/.bashrc"
+
+kinit_keytab() {
+    local principal="$(klist -k "$1" | tail -n 1 | sed 's/.*\s//')"
+    kinit "${principal}" -k -t "$1"
+}
+
+# Renew GraceDB credentials.
+X509_USER_CERT="$HOME/.globus/usercert.pem"
+X509_USER_KEY="$HOME/.globus/userkey.pem"
+kinit_keytab "${HOME}/.globus/krb5.keytab"
+ecp-get-cert -k > /dev/null
+GRID_PROXY_PATH="$(ecp-cert-info -path)"
+cp "${GRID_PROXY_PATH}" "${X509_USER_CERT}"
+cp "${GRID_PROXY_PATH}" "${X509_USER_KEY}"
+
+# Renew CVMFS credentials.
+kinit_keytab "${HOME}/read-cvmfs.keytab"
+chronic htgettoken -v -a vault.ligo.org -i igwn -r read-cvmfs-${USER} --scopes=read:/virgo --credkey=read-cvmfs-${USER}/robot/${USER}.ligo.caltech.edu --nooidc
+
+# Rotate log files.
+/usr/sbin/logrotate --state ~/.local/state/logrotate.status ~/.config/logrotate.conf