.gitignore

@@ -7,3 +7,6 @@ config/settings/local.py
 docs/user_docs/build/*
 docs/admin_docs/build/*
 static_root/*
+.pytest_cache
+junit.xml
+.coverage

.gitlab-ci.yml

@@ -2,30 +2,205 @@
 image: docker:latest
 
 variables:
+  APT_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/apt"
   DOCKER_DRIVER: overlay
   DOCKER_BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
   DOCKER_LATEST: $CI_REGISTRY_IMAGE:latest
+  PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
 
 stages:
+  - test
   - branch
   - latest
 
 before_script:
   - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
 
+include:
+  # Container scanning
+  - component: $CI_SERVER_FQDN/computing/gitlab/components/container-scanning/container-scanning@~latest
+    inputs:
+      job_name: branch_scan
+
+  # Software scanning
+  - component: $CI_SERVER_FQDN/computing/gitlab/components/sast/sast@~latest
+    inputs:
+      run_advanced_sast: true
+  - component: $CI_SERVER_FQDN/computing/gitlab/components/secret-detection/secret-detection@~latest
+  - component: $CI_SERVER_FQDN/computing/gitlab/components/python/dependency-scanning@~latest
+
+# -- software scanning
+# overwrite some settings for the scanning jobs
+
+dependency_scanning:
+  stage: test
+  needs: []
+  variables:
+    DEBIAN_FRONTEND: "noninteractive"
+  before_script:
+    # install some underlying utilities using `apt` so that the dependency
+    # scanner can use pip to install everything else
+    - apt-get update -yqq
+    - apt-get install -yqq
+        libkrb5-dev
+        libldap-dev
+        libsasl2-dev
+
+.sast-analyzer:
+  stage: test
+  needs: []
+  before_script: []
+
+secret_detection:
+  stage: test
+  needs: []
+  before_script: []
+
+# -- testing
+
+.test: &test
+  image: igwn/base:bookworm
+  services:
+    - postgres:15.6
+    - memcached
+  variables:
+    AWS_SES_ACCESS_KEY_ID: "fake_aws_id"
+    AWS_SES_SECRET_ACCESS_KEY: "fake_aws_key"
+    DJANGO_ALERT_EMAIL_FROM: "fake_email"
+    DJANGO_DB_HOST: "postgres"
+    DJANGO_DB_PORT: "5432"
+    DJANGO_DB_NAME: "fake_name"
+    DJANGO_DB_USER: "runner"
+    DJANGO_DB_PASSWORD: ""
+    DJANGO_PRIMARY_FQDN: "fake_fqdn"
+    DJANGO_SECRET_KEY: "fake_key"
+    DJANGO_SETTINGS_MODULE: "config.settings.container.dev"
+    DJANGO_TWILIO_ACCOUNT_SID: "fake_sid"
+    DJANGO_TWILIO_AUTH_TOKEN: "fake_token"
+    DJANGO_DOCKER_MEMCACHED_ADDR: "memcached:11211"
+    EGAD_URL: "fake_url"
+    EGAD_API_KEY: "fake_key"
+    ENABLE_LVALERT_OVERSEER: "false"
+    ENABLE_IGWN_OVERSEER: "false"
+    LVALERT_OVERSEER_PORT: "2"
+    LVALERT_SERVER: "fake_server"
+    LVALERT_USER: "fake_user"
+    LVALERT_PASSWORD: "fake_password"
+    ENABLE_IGWN_OVERSEER: "false"
+    IGWN_ALERT_OVERSEER_PORT: "2"
+    IGWN_ALERT_SERVER: "fake_server"
+    IGWN_ALERT_USER: "fake_user"
+    IGWN_ALERT_PASSWORD: "fake_password"
+    POSTGRES_DB: "${DJANGO_DB_NAME}"
+    POSTGRES_USER: "${DJANGO_DB_USER}"
+    POSTGRES_PASSWORD: "${DJANGO_DB_PASSWORD}"
+    POSTGRES_HOST_AUTH_METHOD: trust
+  before_script:
+    # create apt cache directory
+    - mkdir -pv ${APT_CACHE_DIR}
+    # set python version
+    - PYTHON_VERSION="${CI_JOB_NAME##*:}"
+    - PYTHON_MAJOR="${PYTHON_VERSION:0:1}"
+    - PYTHON="python3"
+    # install build requirements
+    - apt-get -y install gnupg
+    - sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+    - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+    - apt-get -yqq update
+    - apt-get -o dir::cache::archives="${APT_CACHE_DIR}" install -yqq
+          git
+          gnupg
+          libldap2-dev
+          libsasl2-dev
+          libssl-dev
+          libxml2-dev
+          krb5-user
+          libkrb5-dev
+          libsasl2-modules-gssapi-mit
+          swig
+          pkg-config 
+          libpng-dev 
+          libfreetype6-dev 
+          libxslt-dev
+          ${PYTHON}-pip
+          postgresql-15
+          postgresql-client-15
+          libpq-dev
+    # upgrade pip (requirement for lalsuite)
+    - ${PYTHON} -m pip install --upgrade pip --break-system-packages
+    # install everything else from pip
+    - ${PYTHON} -m pip install -r requirements.txt --break-system-packages
+    # create logs path required for tests
+    - mkdir -pv ../logs/
+    # list packages
+    - ${PYTHON} -m pip list installed
+  script:
+    - PYTHONPATH=${PYTHONPATH}:${PWD}/gracedb ${PYTHON} -m pytest --cov-report term-missing --cov ./gracedb --junitxml=${CI_PROJECT_DIR}/junit.xml
+  after_script:
+    - rm -fvr ${PIP_CACHE_DIR}/log
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
+  artifacts:
+    reports:
+      junit: junit.xml
+  cache:
+    key: "${CI_JOB_NAME}"
+    paths:
+      - .cache/pip
+      - .cache/apt
+  coverage: '/^TOTAL\s+.*\s+(\d+\.?\d*)%/'
+  tags:
+      - executor-docker
+
+test:3.11:
+  <<: *test
+
+# -- docker
+
 branch_image:
   stage: branch
   script:
     - docker build --pull -t $DOCKER_BRANCH .
     - docker push $DOCKER_BRANCH
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
+  tags:
+      - executor-docker
+
+branch_scan:
+  stage: branch
+  needs: [branch_image]
+  # default rules spawn a merge request pipeline, we don't want that
+  rules:
+    - if: $CI_COMMIT_BRANCH
+  variables:
+    GIT_STRATEGY: fetch
+    # image to scan
+    CS_IMAGE: "$DOCKER_BRANCH"
+    # image to compare to
+    CS_DEFAULT_BRANCH_IMAGE: "$CI_REGISTRY/computing/gitlab/server:latest"
+    # path to Dockerfile for remediation
+    CS_DOCKERFILE_PATH: "Dockerfile"
+  before_script: []
 
 latest_image:
   stage: latest
   dependencies:
     - branch_image
-  only:
-    refs:
-      - master
   script:
+    - docker pull $DOCKER_BRANCH
     - docker tag $DOCKER_BRANCH $DOCKER_LATEST
     - docker push $DOCKER_LATEST
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
+  tags:
+      - executor-docker

.gitlab/issue_templates/Bug.md

+## Description of problem
+<!--
+Describe in detail what you are trying to do and what the result is.
+Exact timestamps, error tracebacks, and screenshots (if applicable) are very helpful.
+-->
+
+
+## Expected behavior
+<!-- What do you expect to happen instead? -->
+
+
+## Steps to reproduce
+<!-- Step-by-step procedure for reproducing the issue -->
+
+
+## Context/environment
+<!--
+Describe the environment you are working in:
+    * If using the ligo-gracedb client package, which version?
+    * Your operating system
+    * Your browser (web interface issues only)
+    * If you are experiencing this problem while working on a LIGO or Virgo computing cluster, which cluster are you using?
+-->
+
+
+## Suggested solutions
+<!-- Any ideas for how to resolve this problem? -->

.gitlab/issue_templates/Feature-request.md

+## Description of feature request
+<!--
+Describe your feature request!
+Is it a web interface change? Some underlying feature? An API resource?
+The more detail you can provide, the better.
+-->
+
+
+## Use cases
+<!-- List some specific cases where this feature will be useful -->
+
+
+## Benefits
+<!-- Describe the benefits of adding this feature -->
+
+
+## Drawbacks
+<!--
+Are there any drawbacks to adding this feature?
+Can you think of any ways in which this will negatively affect the service for any set of users?
+-->
+
+
+## Suggested solutions
+<!-- Do you have any ideas for how to implement this feature? -->

Dockerfile

-FROM ligo/base:stretch
+FROM debian:bookworm
 LABEL name="LIGO GraceDB Django application" \
-      maintainer="tanner.prestegard@ligo.org" \
-      date="20181206"
+      maintainer="alexander.pace@ligo.org" \
+      date="20240306"
 ARG SETTINGS_MODULE="config.settings.container.dev"
 
 COPY docker/SWITCHaai-swdistrib.gpg /etc/apt/trusted.gpg.d
-RUN echo 'deb http://pkg.switch.ch/switchaai/debian stretch main' > /etc/apt/sources.list.d/shibboleth.list
-RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
+COPY docker/backports.pref /etc/apt/preferences.d
 RUN apt-get update && \
+    apt-get -y install gnupg curl
+
+
+RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' > /etc/apt/sources.list.d/backports.list
+RUN echo 'deb http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main' > /etc/apt/sources.list.d/pgdg.list
+RUN echo 'deb [trusted=yes] https://hypatia.aei.mpg.de/lsc-amd64-bookworm ./' > /etc/apt/sources.list.d/lscsoft.list
+RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+RUN apt-get update && \
+    apt-get --assume-yes upgrade && \
     apt-get install --install-recommends --assume-yes \
         apache2 \
+        emacs-nox \
         gcc \
         git \
+        krb5-user \
+        libkrb5-dev \
+        libapache2-mod-shib \
         libapache2-mod-xsendfile \
-        libmariadbclient-dev \
         libldap2-dev \
+        libldap-2.5-0 \
         libsasl2-dev \
+        libsasl2-modules-gssapi-mit \
         libxml2-dev \
+        pkg-config \
+        libpng-dev \
+        libpq-dev \
+        libfreetype6-dev \
+        libxslt-dev \
         libsqlite3-dev \
         ligo-ca-certs \
-        mariadb-client \
-        nodejs \
         osg-ca-certs \
-        python2.7 \
-        python2.7-dev \
-        python-glue=1.60.0-3+deb9u0 \
-        python-glue-common=1.60.0-3+deb9u0 \
-        python-libxml2 \
-        python-pip \
-        python-voeventlib \
+        php \
+        php8.2-pgsql \
+        php8.2-mbstring \
+        postgresql-client-15 \
+        python3 \
+        python3-dev \
+        python3-libxml2 \
+        python3-pip \
         procps \
-        shibboleth \
-        supervisor \
+        redis \
+        shibboleth-sp-common \
+        shibboleth-sp-utils \
         libssl-dev \
         swig \
+        htop \
+        telnet \
         vim && \
     apt-get clean && \
-    npm install -g bower
+    curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
+    apt-get update && apt-get install --assume-yes yarn
+
+# Install AWS X-ray daemon
+RUN curl -O https://s3.us-east-2.amazonaws.com/aws-xray-assets.us-east-2/xray-daemon/aws-xray-daemon-3.x.deb
+RUN dpkg -i aws-xray-daemon-3.x.deb
+RUN rm aws-xray-daemon-3.x.deb
 
+# Docker scripts:
 COPY docker/entrypoint /usr/local/bin/entrypoint
 COPY docker/cleanup /usr/local/bin/cleanup
+
+# Supervisord configs:
 COPY docker/supervisord.conf /etc/supervisor/supervisord.conf
 COPY docker/supervisord-apache2.conf /etc/supervisor/conf.d/apache2.conf
-COPY docker/supervisord-lvalert-overseer.conf /etc/supervisor/conf.d/overseer.conf
+COPY docker/supervisord-igwn-alert-overseer.conf /etc/supervisor/conf.d/igwn-overseer.conf
 COPY docker/supervisord-shibd.conf /etc/supervisor/conf.d/shibd.conf
-COPY docker/shibboleth-ds /etc/shibboleth-ds
+COPY docker/supervisord-aws-xray.conf /etc/supervisor/conf.d/aws-xray.conf
+COPY docker/supervisord-qcluster.conf /etc/supervisor/conf.d/qcluster.conf
+
+# Apache configs:
 COPY docker/apache-config /etc/apache2/sites-available/gracedb.conf
+COPY docker/mpm_prefork.conf /etc/apache2/mods-enabled/mpm_prefork.conf
+
+# Enable mpm_event module:
+
+RUN rm /etc/apache2/mods-enabled/mpm_prefork.*
+RUN rm /etc/apache2/mods-enabled/php8.2.*
+RUN cp  /etc/apache2/mods-available/mpm_event.* /etc/apache2/mods-enabled/
+
+# Shibboleth configs and certs:
+COPY docker/shibboleth-ds /etc/shibboleth-ds
 COPY docker/login.ligo.org.cert.LIGOCA.pem /etc/shibboleth/login.ligo.org.cert.LIGOCA.pem
 COPY docker/inc-md-cert.pem /etc/shibboleth/inc-md-cert.pem
 COPY docker/check_shibboleth_status /usr/local/bin/check_shibboleth_status
@@ -60,15 +102,17 @@ ADD . /app/gracedb_project
 
 # install gracedb application itself
 WORKDIR /app/gracedb_project
-RUN bower install --allow-root
-RUN pip install --upgrade setuptools wheel && \
-    pip install -r requirements.txt
+RUN pip3 install --upgrade pip --break-system-packages
+RUN pip3 install -r requirements.txt --break-system-packages
+
+# install supervisor from pip
+RUN pip3 install supervisor --break-system-packages
 
 # Give pip-installed packages priority over distribution packages
-ENV PYTHONPATH /usr/local/lib/python2.7/dist-packages:$PYTHONPATH
+ENV PYTHONPATH /usr/local/lib/python3.11/dist-packages:$PYTHONPATH
 ENV ENABLE_SHIBD false
 ENV ENABLE_OVERSEER true
-ENV VIRTUAL_ENV dummy
+ENV VIRTUAL_ENV /dummy/
 
 # Expose port and run Gunicorn
 EXPOSE 8000
@@ -89,18 +133,31 @@ RUN DJANGO_SETTINGS_MODULE=${SETTINGS_MODULE} \
     DJANGO_SECRET_KEY=fake_key \
     DJANGO_PRIMARY_FQDN=fake_fqdn \
     DJANGO_ALERT_EMAIL_FROM=fake_email \
+    EGAD_URL=fake_url \
+    EGAD_API_KEY=fake_key \
     LVALERT_USER=fake_user \
     LVALERT_PASSWORD=fake_password \
     LVALERT_SERVER=fake_server \
     LVALERT_OVERSEER_PORT=2 \
+    IGWN_ALERT_USER=fake_user \
+    IGWN_ALERT_PASSWORD=fake_password \
+    IGWN_ALERT_SERVER=fake_server \
+    IGWN_ALERT_OVERSEER_PORT=2 \
+    IGWN_ALERT_GROUP=fake_group \
     DJANGO_TWILIO_ACCOUNT_SID=fake_sid \
     DJANGO_TWILIO_AUTH_TOKEN=fake_token \
-    python manage.py collectstatic --noinput
+    DJANGO_AWS_ELASTICACHE_ADDR=fake_address:11211 \
+    AWS_SES_ACCESS_KEY_ID=fake_aws_id \
+    AWS_SES_SECRET_ACCESS_KEY=fake_aws_key \
+    python3 manage.py collectstatic --noinput
 
 RUN rm -rf /app/logs/* /app/project_data/*
 
 RUN useradd -M -u 50001 -g www-data -s /bin/false gracedb
 
+#RUN groupadd -r xray
+#RUN useradd -M -u 50002 -g xray -s /bin/false xray
+
 # set secure file/directory permissions. In particular, ADD command at
 # beginning of recipe inherits umask of user running the build
 RUN chmod 0755 /usr/local/bin/entrypoint && \
@@ -110,5 +167,19 @@ RUN chmod 0755 /usr/local/bin/entrypoint && \
     find /app/gracedb_project -type d -exec chmod 0755 {} + && \
     find /app/gracedb_project -type f -exec chmod 0644 {} +
 
+# create and set scitoken key cache directory
+RUN mkdir /app/scitokens_cache && \
+    chown gracedb:www-data /app/scitokens_cache && \
+    chmod 0750 /app/scitokens_cache
+ENV XDG_CACHE_HOME /app/scitokens_cache
+
+# patch voeventparse for python3.10+:
+RUN sed -i 's/collections.Iterable/collections.abc.Iterable/g' /usr/local/lib/python3.11/dist-packages/voeventparse/voevent.py
+
+# Remove packages that expose security vulnerabilities and close out.
+# Edit: zlib1g* can't be removed because of a PrePend error
+RUN apt-get --assume-yes --purge autoremove wget libaom3 node-ip
+RUN apt-get clean
+
 ENTRYPOINT [ "/usr/local/bin/entrypoint" ]
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
+CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]

bower.json

 {
   "name": "gracedb",
   "dependencies": {
-    "dgrid": "0.4.0",
     "dijit": "1.10.4",
-    "dojox": "1.10.4",
-    "jquery": "3.2.1",
-    "jquery-ui": "1.12.1",
-    "moment-timezone": "0.5.0",
-    "moment": "2.11.1"
+    "dojox": "1.10.4"
   }
 }

config/gunicorn_config.py

@@ -6,6 +6,14 @@ from os.path import abspath, dirname, join
 import sys
 import multiprocessing
 
+# Useful function for getting environment variables
+def get_from_env(envvar, default_value=None, fail_if_not_found=True):
+    value = os.environ.get(envvar, default_value)
+    if (value == default_value and fail_if_not_found):
+        raise ImproperlyConfigured(
+            'Could not get environment variable {0}'.format(envvar))
+    return value
+
 # Parameters
 GUNICORN_PORT = 8080
 LOG_DIR = abspath(join(dirname(__file__), "..", "..", "logs"))
@@ -14,27 +22,143 @@ LOG_DIR = abspath(join(dirname(__file__), "..", "..", "logs"))
 # Bind to localhost on specified port
 bind = "127.0.0.1:{port}".format(port=GUNICORN_PORT)
 
-# Number of workers = 2*CPU + 1 (recommendation from Gunicorn documentation)
-workers = multiprocessing.cpu_count()*2 + 1
+# Number of workers -----------------------------------------------------------
+# 2*CPU + 1 (recommendation from Gunicorn documentation)
+# bumped to 4*CPU + 1 after testing. Maybe increase this number in the cloud
+# deployment? 
+
+workers  = int(get_from_env('GUNICORN_WORKERS',
+                   default_value=multiprocessing.cpu_count()*3 + 1,
+                   fail_if_not_found=False))
+
+# NOTE: it was found in extensive testing that threads > 1 are prone
+# to connection lockups. Leave this at 1 for safety until there are 
+# fixes in gunicorn.
+
+# Why not sync? The sync worker is prone to timeout for long requests,
+# like big queries. But gthread sends a heartbeat back to the main worker
+# to keep it alive. We could just set the timeout to a really large number
+# which would keep the long requests stable, but if there is a stuck worker, 
+# then they would be subject to that really long timeout. It's a tradeoff. 
+
+# All this goes away with async workers, but as of 3.2, django's ORM does support
+# async, and testing failed pretty catastrophically and unreliably. 
+
+threads = int(get_from_env('GUNICORN_THREADS',
+                   default_value=1,
+                   fail_if_not_found=False))
+
+# Worker connections. Limit the number of connections between apache<-->gunicorn
+
+worker_connections = workers * threads
 
-# Worker type
-worker_type = 'sync'
+# Worker class ----------------------------------------------------------------
+# sync by default, generally safe and low-resource:
+# https://docs.gunicorn.org/en/stable/design.html#sync-workers
 
-# Max requests settings - a worker restarts after handling this many
-# requests. May be useful if we have memory leak problems.
+worker_class = get_from_env('GUNICORN_WORKER_CLASS',
+                   default_value='gthread',
+                   fail_if_not_found=False)
+
+# Timeout ---------------------------------------------------------------------
+# If not specified, the timeout default is 30 seconds:
+# https://gunicorn-docs.readthedocs.io/en/stable/settings.html#worker-processes
+
+timeout = get_from_env('GUNICORN_TIMEOUT',
+                   default_value=30,
+                   fail_if_not_found=False)
+
+graceful_timeout = timeout
+
+# max_requests settings -------------------------------------------------------
+# The maximum number of requests a worker will process before restarting.
+# May be useful if we have memory leak problems.
 # The jitter is drawn from a uniform distribution:
 # randint(0, max_requests_jitter)
-#max_requests = 0
-#max_requests_jitter = 0
 
+max_requests = get_from_env('GUNICORN_MAX_REQUESTS',
+                   default_value=5000,
+                   fail_if_not_found=False)
+
+max_requests_jitter = get_from_env('GUNICORN_MAX_REQUESTS_JITTER',
+                   default_value=250,
+                   fail_if_not_found=False)
+
+# keepalive -------------------------------------------------------------------
+# The number of seconds to wait for requests on a Keep-Alive connection.
+# Generally set in the 1-5 seconds range for servers with direct connection
+# to the client (e.g. when you don’t have separate load balancer).
+# When Gunicorn is deployed behind a load balancer, it often makes sense to set
+# this to a higher value.
+
+# NOTE: force gunicorn to close its connection to apache after each request. 
+# This has been the source of so many 502's. Basically in periods of high activity,
+# gunicorn would hold on to open sockets with apache, and just deadlock itself:
+# https://github.com/benoitc/gunicorn/issues/2917
+
+keepalive = get_from_env('GUNICORN_KEEPALIVE',
+                   default_value=0,
+                   fail_if_not_found=False)
+
+# preload_app -----------------------------------------------------------------
+# Load application code before the worker processes are forked.
+
+# By preloading an application you can save some RAM resources as well as speed
+# up server boot times. Although, if you defer application loading to each 
+# worker process, you can reload your application code easily by restarting 
+# workers.
+
+# If you aren't going to make use of on-the-fly reloading, consider preloading 
+# your application code to reduce its memory footprint. So, turn this on in
+# production. This is default set to False for development, but
+# **TURN THIS TO TRUE FOR AWS DEPLOYMENT **
+
+preload_app = get_from_env('GUNICORN_PRELOAD_APP',
+                   default_value=True,
+                   fail_if_not_found=False)
+
+# Logging ---------------------------------------------------------------------
 # Access log
 accesslog = join(LOG_DIR, "gunicorn_access.log")
-access_log_format = '%(t)s %(h)s %(l)s %(u)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
+access_log_format = ('GUNICORN | %(h)s %(l)s %(u)s %(t)s '
+    '"%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"')
 
 # Error log
 errorlog = join(LOG_DIR, "gunicorn_error.log")
-loglevel = 'debug'
+
+# debug logging doesn't provide actual information. And this will 
+# eliminate the "Connection closed." messages while still giving info 
+# about worker restarts. 
+loglevel = 'info'
 capture_output = True
 
-#forwarded_allow_ips = '127.0.0.1'
-#proxy_allow_ips = '127.0.0.1'
+# using /dev/shm/ instead of /tmp for the temporary worker directory. See:
+# https://pythonspeed.com/articles/gunicorn-in-docker/
+# “in AWS an EBS root instance volume may sometimes hang for half a minute 
+# and during this time Gunicorn workers may completely block.”
+worker_tmp_dir='/dev/shm'
+
+
+# Override logger class to modify error format
+from gunicorn.glogging import Logger
+class CustomLogger(Logger):
+    error_fmt = 'GUNICORN | ' + Logger.error_fmt
+logger_class = CustomLogger
+
+def post_fork(server, worker):
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+def pre_fork(server, worker):
+    pass
+
+def pre_exec(server):
+    server.log.info("Forked child, re-executing.")
+
+def when_ready(server):
+    server.log.info("Server is ready. Spawning workers")
+
+def worker_int(worker):
+    worker.log.info("worker received INT or QUIT signal")
+
+def worker_abort(worker):
+    worker.log.info("worker received SIGABRT signal")

config/settings/base.py

-from cloghandler import ConcurrentRotatingFileHandler
+from concurrent_log_handler import ConcurrentRotatingFileHandler
 from datetime import datetime, timedelta
-import os, time, logging
+import os, time, logging, multiprocessing
 from os.path import abspath, dirname, join
 import socket
 from django.core.exceptions import ImproperlyConfigured
+from aws_xray_sdk.core.exceptions.exceptions import SegmentNotFoundException
 
 # Set up path to root of project
 BASE_DIR = abspath(join(dirname(__file__), "..", ".."))
@@ -21,18 +22,61 @@ def get_from_env(envvar, default_value=None, fail_if_not_found=True):
             'Could not get environment variable {0}'.format(envvar))
     return value
 
+def parse_envvar_bool(x):
+    return x.lower() in ['t', 'true', '1']
+
+# a sentry before_send function that filters aws SegmentNotFoundException's.
+# these exceptions are harmless and occur when performing management tasks
+# outside of the core gracedb app. but sentry picks it up and reports it as
+# an error which is ANNOYING.
+def before_send(event, hint):
+    if "exc_info" in hint:
+        exc_type, exc_value, tb = hint["exc_info"]
+        if isinstance(exc_value, (SegmentNotFoundException,)):
+            return None
+        return event
+
+# Maintenance mode
+MAINTENANCE_MODE = False
+MAINTENANCE_MODE_MESSAGE = None
+
+# Enable/Disable Information Banner:
+INFO_BANNER_ENABLED = False
+INFO_BANNER_MESSAGE = "TEST MESSAGE"
+
+# Beta reports page:
+BETA_REPORTS_LINK = False
+
+# Version ---------------------------------------------------------------------
+PROJECT_VERSION = '2.31.0'
+
 # Unauthenticated access ------------------------------------------------------
 # This variable should eventually control whether unauthenticated access is
 # allowed *ANYWHERE* on this service, except the home page, which is always
-# public. For now, it just controls the API.
-UNAUTHENTICATED_ACCESS = True
+# public. For now, it just controls the API and the public alerts page.
+# Update: make this updatable from the environment:
+UNAUTHENTICATED_ACCESS = parse_envvar_bool(
+    get_from_env('ENABLE_UNAUTHENTICATED_ACCESS',
+                 fail_if_not_found=False, default_value="true")
+)
+
 
 # Miscellaneous settings ------------------------------------------------------
 # Debug mode is off by default
 DEBUG = False
+# When debug mode is enabled, use custom reporter
+DEFAULT_EXCEPTION_REPORTER = 'core.utils.CustomExceptionReporter'
 
 # Number of results to show on latest page
-LATEST_RESULTS_NUMBER = 50
+LATEST_RESULTS_NUMBER = 25
+
+# Maximum number of log messages to display before throwing
+# a warning. NOTE: There should be a better way of doing this,
+# but just put in the hard cutoff for right now
+# Set to cover this:
+# https://gracedb.ligo.org/events/G184098
+TOO_MANY_LOG_ENTRIES = int(get_from_env('DJANGO_TOO_MANY_LOG_ENTRIES',
+    fail_if_not_found=False, default_value=2000))
 
 # Path to root URLconf
 ROOT_URLCONF = '{module}.urls'.format(module=os.path.basename(CONFIG_ROOT))
@@ -44,8 +88,9 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 # MANAGERS defines who gets broken link notifications when
 # BrokenLinkEmailsMiddleware is enabled
 ADMINS = [
-    ("Tanner Prestegard", "tanner.prestegard@ligo.org"),
     ("Alexander Pace", "alexander.pace@ligo.org"),
+    ("Duncan Meacher", "duncan.meacher@ligo.org"),
+    ("Daniel Wysocki", "daniel.wysocki@ligo.org"),
 ]
 MANAGERS = ADMINS
 
@@ -110,6 +155,24 @@ LOGOUT_REDIRECT_URL = 'home'
 SEND_XMPP_ALERTS = False
 SEND_PHONE_ALERTS = False
 SEND_EMAIL_ALERTS = False
+SEND_MATTERMOST_ALERTS = False
+
+# igwn-alert group settings. the default development group is 'lvalert-dev'
+# for the container deployments, the variable will be overwriten by the 
+# IGWN_ALERT_GROUP environment variable. 
+DEFAULT_IGWN_ALERT_GROUP = 'lvalert-dev'
+
+# enable/disable sending alerts to topics that have the search tag
+# for g/e-events. default to false, so only send to {group}_{pipeline}
+SEND_TO_SEARCH_TOPICS = parse_envvar_bool(
+    get_from_env('IGWN_ALERT_SEARCH_TOPICS',
+                 fail_if_not_found=False, default_value="false")
+)
+
+# overseer timeout:
+OVERSEER_TIMEOUT = float(get_from_env('IGWN_ALERT_OVERSEER_TIMEOUT',
+    fail_if_not_found=False, default_value=0.1))
+
 # Use LVAlert Overseer?
 USE_LVALERT_OVERSEER = True
 # For each LVAlert server, a separate instance of LVAlert Overseer
@@ -118,16 +181,17 @@ USE_LVALERT_OVERSEER = True
 #   listen_port: port which that instance of overseer is listening on
 LVALERT_OVERSEER_INSTANCES = [
     {
-        "lvalert_server": "lvalert-test.cgca.uwm.edu",
-        "listen_port": 8001,
+        "lvalert_server": "kafka://kafka.scimma.org/",
+        "listen_port": 8002,
+        "igwn_alert_group": DEFAULT_IGWN_ALERT_GROUP,
     },
 ]
 
 # Access and authorization ----------------------------------------------------
 # Some proper names related to authorization
-LVC_GROUP = 'Communities:LSCVirgoLIGOGroupMembers'
-LVEM_GROUP = 'gw-astronomy:LV-EM'
-LVEM_OBSERVERS_GROUP = 'gw-astronomy:LV-EM:Observers'
+LVC_GROUP = 'internal_users'
+LVEM_GROUP = 'lvem_users'
+LVEM_OBSERVERS_GROUP = 'lvem_observers'
 PUBLIC_GROUP = 'public_users'
 PRIORITY_USERS_GROUP = 'priority_users'
 
@@ -141,21 +205,13 @@ ACCESS_MANAGERS_GROUP = 'access_managers'
 EM_ADVOCATE_GROUP = 'em_advocates'
 # Superevent managers
 SUPEREVENT_MANAGERS_GROUP = 'superevent_managers'
+# RRT group name:
+RRT_MEMBERS_GROUP = 'rrt_members'
 
 # Analysis groups
 # Analysis group name for non-GW events
 EXTERNAL_ANALYSIS_GROUP = 'External'
 
-# Groups directly managed by GraceDB admins
-ADMIN_MANAGED_GROUPS = [EM_ADVOCATE_GROUP, EXEC_GROUP, PUBLIC_GROUP,
-    SUPEREVENT_MANAGERS_GROUP, ACCESS_MANAGERS_GROUP]
-# NOTE: soon, only superevent_managers will be fully managed by us. The other
-# groups will be managed by the LIGO auth infrastructure for human users. But
-# we will still have to still manage robot account membership in these groups:
-# access_managers, executives, and em_advocates.
-# NOTE: public_users is also managed by us and has only one member
-# (guardian.AnonymousUser)
-
 # Tag to apply to log messages to allow EM partners to view
 EXTERNAL_ACCESS_TAGNAME = 'lvem'
 PUBLIC_ACCESS_TAGNAME = 'public'
@@ -197,17 +253,30 @@ COINC_PIPELINES = [
                     'spiir',
                     'MBTAOnline',
                     'pycbc',
+                    'MBTA',
+                    'PyGRB',
                    ]
 GRB_PIPELINES = [
                     'Fermi',
                     'Swift',
+                    'INTEGRAL',
+                    'AGILE',
+                    'CHIME',
+                    'SVOM',
                 ]
 
-# SkyAlert stuff - used for VOEvents (?) --------------------------------------
-IVORN_PREFIX = "ivo://gwnet/LVC#"
-SKYALERT_ROLE          = "test"
-SKYALERT_DESCRIPTION   = "Report of a candidate gravitational wave event"
-SKYALERT_SUBMITTERS = ['Patrick Brady', 'Brian Moe']
+# List of pipelines that have been depreciated:
+DEPRECIATED_PIPELINES = [
+                          'X',
+                          'Q',
+                          'Omega',
+                        ]
+
+UNAPPROVED_PIPELINES = []
+
+# VOEvent stream --------------------------------------------------------------
+VOEVENT_STREAM = 'gwnet/LVC'
+
 
 # Stuff related to report/plot generation -------------------------------------
 
@@ -226,7 +295,7 @@ REPORT_INFO_URL_PREFIX = "/report_info/"
 REPORT_IFAR_IMAGE_DIR = PROJECT_DATA_DIR
 
 # Stuff for the new rates plot
-BINNED_COUNT_PIPELINES = ['gstlal', 'MBTAOnline', 'CWB', 'oLIB', 'spiir']
+BINNED_COUNT_PIPELINES = ['gstlal', 'MBTAOnline', 'MBTA', 'CWB', 'oLIB', 'spiir']
 BINNED_COUNT_FILE = join(PROJECT_DATA_DIR, "binned_counts.json")
 
 # Defaults for RSS feed
@@ -303,6 +372,11 @@ X509_INFOS_HEADER = 'HTTP_X_FORWARDED_TLS_CLIENT_CERT_INFOS'
 # Path to CA store for X509 certificate verification
 CAPATH = '/etc/grid-security/certificates'
 
+# SciTokens claims settings
+SCITOKEN_ISSUER = ['https://cilogon.org/igwn', 'https://test.cilogon.org/igwn', 'https://osdf.igwn.org/cit']
+SCITOKEN_AUDIENCE = ["ANY"]
+SCITOKEN_SCOPE = "gracedb.read"
+
 # List of authentication backends to use when attempting to authenticate
 # a user.  Will be used in this order.  Authentication for the API is
 # handled by the REST_FRAMEWORK dictionary.
@@ -314,6 +388,7 @@ AUTHENTICATION_BACKENDS = [
 
 # List of middleware classes to use.
 MIDDLEWARE = [
+    'core.middleware.maintenance.MaintenanceModeMiddleware',
     'events.middleware.PerformanceMiddleware',
     'core.middleware.accept.AcceptMiddleware',
     'core.middleware.api.ClientVersionMiddleware',
@@ -321,6 +396,7 @@ MIDDLEWARE = [
     'django.middleware.common.CommonMiddleware',
     'core.middleware.proxy.XForwardedForMiddleware',
     'user_sessions.middleware.SessionMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'ligoauth.middleware.ShibbolethWebAuthMiddleware',
@@ -337,28 +413,35 @@ SITE_ID=1
 INSTALLED_APPS = [
     'django.contrib.auth',
     'django.contrib.admin',
+    'django_ses',
     'django.contrib.contenttypes',
     'user_sessions',
     'django.contrib.sites',
     'django.contrib.staticfiles',
     'django.contrib.messages',
     'alerts',
+    'annotations',
     'api',
+    'core',
     'events',
     'ligoauth',
     'search',
     'superevents',
+    'gwtc',
     'rest_framework',
     'guardian',
     'django_twilio',
     'django_extensions',
     'django.contrib.sessions',
+    'computedfields',
+    'django_postgres_vacuum',
+    'django_q',
 ]
 
 # Aliases for django-extensions shell_plus
 SHELL_PLUS_MODEL_ALIASES = {
     # Two 'Group' models - auth.Group and gracedb.Group
-    'auth': {'Group': 'AuthGroup'},
+    'auth': {'Group': 'DjangoGroup'},
     # Superevents models which have the same name as
     # models in the events app
     'superevents': {
@@ -387,11 +470,12 @@ REST_FRAMEWORK = {
     ),
     'DEFAULT_THROTTLE_RATES': {
         'anon_burst': '300/minute',
-        'event_creation': '10/second',
-        'annotation'    : '10/second',
+        'event_creation': '50/second',
+        'annotation'    : '50/second',
     },
     'DEFAULT_AUTHENTICATION_CLASSES': (
         'api.backends.GraceDbAuthenticatedAuthentication',
+        'api.backends.GraceDbSciTokenAuthentication',
         'api.backends.GraceDbX509Authentication',
         'api.backends.GraceDbBasicAuthentication',
     ),
@@ -407,9 +491,6 @@ if UNAUTHENTICATED_ACCESS is True:
     REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'] = \
         ('rest_framework.permissions.IsAuthenticatedOrReadOnly',)
 
-# Location of packages installed by bower
-BOWER_DIR = join(BASE_DIR, "..", "bower_components")
-
 # Location of static components, CSS, JS, etc.
 STATIC_ROOT = join(BASE_DIR, "static_root")
 STATIC_URL = "/static/"
@@ -420,7 +501,6 @@ STATICFILES_FINDERS = [
 ]
 STATICFILES_DIRS = [
     join(PROJECT_ROOT, "static"),
-    BOWER_DIR,
 ]
 
 # Added in order to perform data migrations on Django apps
@@ -512,7 +592,7 @@ LOGGING = {
             'datefmt': LOG_DATEFMT,
         },
         'console': {
-            'format': ('DJANGO |%(asctime)s.%(msecs)03d | {host} | {ip} | '
+            'format': ('DJANGO | %(asctime)s.%(msecs)03d | {host} | {ip} | '
                 '%(name)s | %(levelname)s | %(filename)s, line %(lineno)s | '
                 '%(message)s').format(host=INTERNAL_HOSTNAME,
                 ip=INTERNAL_IP_ADDRESS),
@@ -603,10 +683,108 @@ LOGGING = {
             'propagate': True,
             'level': LOG_LEVEL,
         },
-       'django.request': {
+        'django.request': {
             'handlers': ['mail_admins'],
             'level': 'ERROR',
             'propagate': False,
         },
    },
 }
+
+# Turn off debug/error emails when in maintenance mode. 
+if MAINTENANCE_MODE:
+    LOGGING['loggers']['django.request']['handlers'].remove('mail_admins')
+
+# Turn off logging emails of django requests:
+# FIXME: figure out more reliable logging solution
+LOGGING['loggers']['django.request']['handlers'].remove('mail_admins')
+
+# Define some words for the instance stub:
+ENABLED = {True: "enabled", False: "disabled"}
+
+# Upgrading to django 3.2 produces warning: "Auto-created primary key used 
+# when not defining a primary key type, by default 'django.db.models.AutoField'.
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
+# Define window for neighbouring s events of a given g event.
+EVENT_SUPEREVENT_WINDOW_BEFORE = 100
+EVENT_SUPEREVENT_WINDOW_AFTER = 100
+
+# Define which observation periods to show on the public events page:
+# TODO: Group O4b and O4a under O4, once implemented.
+PUBLIC_PAGE_RUNS = ['O4', 'O4c', 'O4b', 'O4a', 'ER16', 'ER15', 'O3']
+
+# Define how long to cache the public page:
+PUBLIC_PAGE_CACHING = int(get_from_env('DJANGO_PUBLIC_PAGE_CACHING',
+    fail_if_not_found=False, default_value=300))
+
+# Define the number of results per page on the public page:
+PUBLIC_PAGE_RESULTS = int(get_from_env('DJANGO_PUBLIC_PAGE_RESULTS',
+    fail_if_not_found=False, default_value=15))
+
+# Define DATA_UPLOAD_MAX_MEMORY_SIZE for larger uploads:
+DATA_UPLOAD_MAX_MEMORY_SIZE = int(get_from_env('DJANGO_DATA_UPLOAD_MAX_MEMORY_SIZE',
+    fail_if_not_found=False, default_value=20*1024*1024))
+
+# Choose whether to use to Julian or Civil definition of year 
+# when displaying far's in /year:
+DISPLAY_CIVIL_YEAR_FAR = parse_envvar_bool(
+    get_from_env('DJANGO_DISPLAY_CIVIL_YEAR_FAR',
+                 fail_if_not_found=False, default_value="false")
+)
+
+if DISPLAY_CIVIL_YEAR_FAR:
+    DAYS_PER_YEAR = 365.0
+else:
+    DAYS_PER_YEAR = 365.25
+
+# Put in some setting for the redis queue backend
+ENABLE_REDIS_QUEUE = parse_envvar_bool(
+    get_from_env('DJANGO_ENABLE_REDIS_QUEUE',
+                 fail_if_not_found=False, default_value="false")
+)
+
+REDIS_QUEUE_ADDRESS = get_from_env('DJANGO_REDIS_QUEUE_ADDRESS',
+                 fail_if_not_found=False, default_value="127.0.0.1")
+
+REDIS_QUEUE_PORT = int(get_from_env('DJANGO_REDIS_QUEUE_PORT',
+                 fail_if_not_found=False, default_value="6379")
+)
+
+REDIS_QUEUE_DATABASE = int(get_from_env('DJANGO_REDIS_QUEUE_DATABASE',
+                 fail_if_not_found=False, default_value=0)
+)
+
+REDIS_QUEUE_WORKERS = int(get_from_env('DJANGO_REDIS_QUEUE_WORKERS',
+                   default_value=multiprocessing.cpu_count(),
+                   fail_if_not_found=False))
+
+REDIS_QUEUE_RETRY = int(get_from_env('DJANGO_REDIS_QUEUE_RETRY',
+                   default_value=40,
+                   fail_if_not_found=False))
+
+REDIS_QUEUE_TIMEOUT = int(get_from_env('DJANGO_REDIS_QUEUE_TIMEOUT',
+                   default_value=30,
+                   fail_if_not_found=False))
+
+REDIS_QUEUE_RECYCLE = int(get_from_env('DJANGO_REDIS_QUEUE_RECYCLE',
+                   default_value=500,
+                   fail_if_not_found=False))
+
+ENABLE_REDIS_CLUSTERED = parse_envvar_bool(
+    get_from_env('DJANGO_ENABLE_REDIS_CLUSTERED',
+                 fail_if_not_found=False, default_value="false")
+)
+
+# Define some defaults for the q-cluster parameters:
+Q_CLUSTER_NAME = 'gracedb-async-queue'
+Q_CLUSTER_LABEL = 'gracedb q cluster'
+
+if not ENABLE_REDIS_CLUSTERED:
+    Q_CLUSTER_NAME+=f'-{INTERNAL_HOSTNAME}'
+    Q_CLUSTER_LABEL+=f', {INTERNAL_HOSTNAME}'
+
+# Define MAX_DATATABLES_RESULTS to limit memory usage for web queries:
+MAX_DATATABLES_RESULTS = int(get_from_env('DJANGO_MAX_DATATABLES_RESULTS',
+    fail_if_not_found=False, default_value=1000))
+

config/settings/container/base.py

@@ -31,26 +31,87 @@ if SERVER_FQDN is None:
     raise ImproperlyConfigured('Could not get FQDN from envvars.')
 LIGO_FQDN = SERVER_FQDN
 
-# Get LVAlert server
-lvalert_server = os.environ.get('LVALERT_SERVER', None)
-if lvalert_server is None:
-    raise ImproperlyConfigured('Could not get LVAlert server from envvars.')
-
-# Get LVAlert Overseer listen port
-lvalert_overseer_port = os.environ.get('LVALERT_OVERSEER_PORT', None)
-if lvalert_overseer_port is None:
-    raise ImproperlyConfigured('Could not get LVAlert overseer port '
+
+## EGAD (External GraceDB Alert Dispatcher) configuration
+ENABLE_EGAD_EMAIL = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_EMAIL',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_KAFKA = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_KAFKA',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_MATTERMOST = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_MATTERMOST',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_PHONE = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_PHONE',
+                 fail_if_not_found=False, default_value="false")
+)
+
+ENABLE_EGAD = (
+    ENABLE_EGAD_EMAIL or ENABLE_EGAD_KAFKA
+    or ENABLE_EGAD_MATTERMOST or ENABLE_EGAD_PHONE
+)
+
+EGAD_URL = get_from_env('EGAD_URL',
+                        fail_if_not_found=ENABLE_EGAD, default_value=None)
+EGAD_API_KEY = get_from_env('EGAD_API_KEY',
+                   fail_if_not_found=ENABLE_EGAD, default_value=None)
+
+# Turn LVAlert on/off from the environment. Adding this
+# to turn lvalerts on/off from docker compose/update instead
+# of having to rebuild containers. If the environment variable
+# isn't set, then revert to the hardwired behavior:
+xmpp_env_var = get_from_env('SEND_LVALERT_XMPP_ALERTS',
+                   default_value=SEND_XMPP_ALERTS,
+                   fail_if_not_found=False)
+# Fix for other boolean values:
+if (isinstance(xmpp_env_var, str) and
+    xmpp_env_var.lower() in ['true','t','1']):
+    SEND_XMPP_ALERTS=True
+elif (isinstance(xmpp_env_var, str) and
+    xmpp_env_var.lower() in ['false','f','0']):
+    SEND_XMPP_ALERTS=False
+else:
+    SEND_XMPP_ALERTS = True
+
+# Get igwn_alert_overseer status:
+igwn_alert_on  = get_from_env(
+    'ENABLE_IGWN_OVERSEER',
+    default_value=False,
+    fail_if_not_found=False
+)
+if (isinstance(igwn_alert_on, str) and
+    igwn_alert_on.lower() in ['true', 't', '1']):
+        igwn_alert_overseer_on  = True
+else:
+    igwn_alert_overseer_on  = False
+
+# Get igwn-alert server
+igwn_alert_server = os.environ.get('IGWN_ALERT_SERVER', None)
+if igwn_alert_server is None:
+    raise ImproperlyConfigured('Could not get igwn-alert server from envvars.')
+
+# Get igwn-alert Overseer listen port
+igwn_alert_overseer_port = os.environ.get('IGWN_ALERT_OVERSEER_PORT', None)
+if igwn_alert_overseer_port is None:
+    raise ImproperlyConfigured('Could not get igwn-alert overseer port '
         'from envvars.')
 
-# Get LVAlert username
-lvalert_user = os.environ.get('LVALERT_USER', None)
-if lvalert_user is None:
-    raise ImproperlyConfigured('Could not get LVAlert username from envvars.')
+# Get igwn-alert group from envirnment:
+igwn_alert_group = os.environ.get('IGWN_ALERT_GROUP', DEFAULT_IGWN_ALERT_GROUP)
+
+# Get igwn-alert username
+igwn_alert_user = os.environ.get('IGWN_ALERT_USER', None)
+if igwn_alert_user is None:
+    raise ImproperlyConfigured('Could not get igwn-alert username from envvars.')
 
-# Get LVAlert password
-lvalert_password = os.environ.get('LVALERT_PASSWORD', None)
-if lvalert_password is None:
-    raise ImproperlyConfigured('Could not get LVAlert password from envvars.')
+# Get igwn-alert password
+igwn_alert_password = os.environ.get('IGWN_ALERT_PASSWORD', None)
+if igwn_alert_password is None:
+    raise ImproperlyConfigured('Could not get igwn-alert password from envvars.')
 
 # Get Twilio account information from environment
 TWILIO_ACCOUNT_SID = os.environ.get('DJANGO_TWILIO_ACCOUNT_SID', None)
@@ -61,48 +122,250 @@ TWILIO_AUTH_TOKEN = os.environ.get('DJANGO_TWILIO_AUTH_TOKEN', None)
 if TWILIO_AUTH_TOKEN is None:
     raise ImproperlyConfigured('Could not get Twilio auth token from envvars.')
 
+# Get maintenance mode settings from environment
+maintenance_mode = get_from_env(
+    'DJANGO_MAINTENANCE_MODE_ACTIVE',
+    default_value=False,
+    fail_if_not_found=False
+)
+
+# DB "cool-down" factor for when a db conflict is detected. This
+# factor scales a random number of seconds between zero and one.
+DB_SLEEP_FACTOR = get_from_env(
+        'DJANGO_DB_SLEEP_FACTOR',
+        default_value=1.0,
+        fail_if_not_found=False
+)
+# Fix the factor (str to float)
+try:
+    DB_SLEEP_FACTOR = float(DB_SLEEP_FACTOR)
+except:
+    DB_SLEEP_FACTOR = 1.0
+
+if (isinstance(maintenance_mode, str) and
+    maintenance_mode.lower() in ['true', 't', '1']):
+    MAINTENANCE_MODE = True
+MAINTENANCE_MODE_MESSAGE = \
+    get_from_env('DJANGO_MAINTENANCE_MODE_MESSAGE', fail_if_not_found=False)
+
+# Get info banner settings from environment
+info_banner_enabled = get_from_env(
+    'DJANGO_INFO_BANNER_ENABLED',
+    default_value=False,
+    fail_if_not_found=False
+    )
+# fix for other booleans:
+if (isinstance(info_banner_enabled, str) and
+    info_banner_enabled.lower() in ['true','t','1']):
+    INFO_BANNER_ENABLED = True
+INFO_BANNER_MESSAGE = \
+    get_from_env('DJANGO_INFO_BANNER_MESSAGE', fail_if_not_found=False)
+
+# Get reports page boolean:
+beta_reports_link = get_from_env(
+    'DJANGO_BETA_REPORTS_LINK',
+    default_value=False,
+    fail_if_not_found=False
+    )
+# fix for other booleans:
+if (isinstance(beta_reports_link, str) and
+    beta_reports_link.lower() in ['true','t','1']):
+    BETA_REPORTS_LINK = True
+
 # Get email settings from environment
-EMAIL_PORT = 587
-EMAIL_USE_TLS = True
-EMAIL_HOST = os.environ.get('SMTP_HOST', 'localhost')
-EMAIL_HOST_USER = os.environ.get('SMTP_USERNAME', '')
-EMAIL_HOST_PASSWORD = os.environ.get('SMTP_PASSWORD', '')
-ALERT_EMAIL_FROM = os.environ.get('DJANGO_ALERT_EMAIL_FROM', None)
-if ALERT_EMAIL_FROM is None:
-    raise ImproperlyConfigured(
-        "Could not get 'alert from email' from envvars.")
+EMAIL_BACKEND = 'django_ses.SESBackend'
+AWS_SES_ACCESS_KEY_ID = get_from_env('AWS_SES_ACCESS_KEY_ID')
+AWS_SES_SECRET_ACCESS_KEY = get_from_env('AWS_SES_SECRET_ACCESS_KEY')
+AWS_SES_REGION_NAME = get_from_env('AWS_SES_REGION_NAME',
+    default_value='us-west-2', fail_if_not_found=False)
+AWS_SES_REGION_ENDPOINT = get_from_env('AWS_SES_REGION_ENDPOINT',
+    default_value='email.us-west-2.amazonaws.com', fail_if_not_found=False)
+AWS_SES_AUTO_THROTTLE = 0.25
+ALERT_EMAIL_FROM = get_from_env('DJANGO_ALERT_EMAIL_FROM')
+
+# memcached settings. this variable should be set in the deployment to the 
+# same name as the service name in the docker deployment.
+
+DOCKER_MEMCACHED_ADDR = get_from_env('DJANGO_DOCKER_MEMCACHED_ADDR',
+                          default_value="memcached:11211",
+                          fail_if_not_found=False)
+DOCKER_MEMCACHED_SECONDS = get_from_env('DJANGO_DOCKER_MEMCACHED_SECONDS',
+                          default_value="15",
+                          fail_if_not_found=False)
+try:
+    CACHE_MIDDLEWARE_SECONDS = int(DOCKER_MEMCACHED_SECONDS)
+except:
+    CACHE_MIDDLEWARE_SECONDS = 15
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
+        'LOCATION': DOCKER_MEMCACHED_ADDR,
+        'OPTIONS': {
+            'ignore_exc': True,
+            }
+    },
+    # For API throttles
+    'throttles': {
+        'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
+        'LOCATION': 'api_throttle_cache', # Table name
+    },
+}
+
+if ENABLE_REDIS_QUEUE:
+    # For async alert follow-up:
+    CACHES.update({"async_followup": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": f"redis://{REDIS_QUEUE_ADDRESS}:{REDIS_QUEUE_PORT}/{REDIS_QUEUE_DATABASE}",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        }
+    }})
+
+
+    # Set queue backend for async django tasks:
+    # example django-redis connection
+    Q_CLUSTER = {
+        'name': Q_CLUSTER_NAME,
+        'label': Q_CLUSTER_LABEL,
+        'retry': REDIS_QUEUE_RETRY,
+        'timeout': REDIS_QUEUE_TIMEOUT,
+        'workers': REDIS_QUEUE_WORKERS,
+        'recycle': REDIS_QUEUE_RECYCLE,
+        'django_redis': 'async_followup'
+    }
+
+MIDDLEWARE = [
+    'django.middleware.cache.UpdateCacheMiddleware',
+    'core.middleware.maintenance.MaintenanceModeMiddleware',
+    'events.middleware.PerformanceMiddleware',
+    'core.middleware.accept.AcceptMiddleware',
+    'core.middleware.api.ClientVersionMiddleware',
+    'core.middleware.api.CliExceptionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'core.middleware.proxy.XForwardedForMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'user_sessions.middleware.SessionMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'ligoauth.middleware.ShibbolethWebAuthMiddleware',
+    'ligoauth.middleware.ControlRoomMiddleware',
+    'django.middleware.cache.FetchFromCacheMiddleware',
+]
+
+# Set up AWS X-ray patching if enabled
+ENABLE_AWS_XRAY = (
+    get_from_env("ENABLE_AWS_XRAY",
+                 default_value="false", fail_if_not_found=False).lower()
+    in ['true', 't', '1']
+)
+
+if ENABLE_AWS_XRAY:
+    # AWS X-ray middleware must be first in the list to measure timing
+    # accurately
+    MIDDLEWARE.insert(0, 'aws_xray_sdk.ext.django.middleware.XRayMiddleware')
+    # Include X-ray as an installed app in order to allow configuration beyond
+    # the default
+    INSTALLED_APPS.append('aws_xray_sdk.ext.django')
+    # Settings for AWS X-ray
+    XRAY_RECORDER = {
+        'AWS_XRAY_DAEMON_ADDRESS': '127.0.0.1:2000',
+        'AUTO_INSTRUMENT': True,
+        'AWS_XRAY_CONTEXT_MISSING': 'LOG_ERROR',
+        'PLUGINS': (),
+        'SAMPLING': True,
+        'SAMPLING_RULES': None,
+        'AWS_XRAY_TRACING_NAME': 'GraceDB',
+        'DYNAMIC_NAMING': None,
+        'STREAMING_THRESHOLD': None,
+    }
+
+# Priority server settings ----------------------------------------------------
+PRIORITY_SERVER = False
+is_priority_server = get_from_env('DJANGO_PRIORITY_SERVER', None,
+    fail_if_not_found=False)
+if (isinstance(is_priority_server, str) and
+    is_priority_server.lower() in ['true', 't']):
+    PRIORITY_SERVER = True
+
+# If priority server, only allow priority users to the API
+if PRIORITY_SERVER:
+    # Add custom permissions for the API
+    default_perms = list(REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'])
+    default_perms = ['api.permissions.IsPriorityUser'] + default_perms
+    REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'] = tuple(default_perms)
+
 
 # Database settings -----------------------------------------------------------
+
+# New postgresql database
+# Configured for the CI pipeline:
+# https://docs.gitlab.com/ee/ci/services/postgres.html
 DATABASES = {
     'default' : {
         'NAME': db_name,
-        'ENGINE': 'django.db.backends.mysql',
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
         'USER': db_user,
         'PASSWORD': db_password,
         'HOST': os.environ.get('DJANGO_DB_HOST', ''),
         'PORT': os.environ.get('DJANGO_DB_PORT', ''),
-        'OPTIONS': {
-            'init_command': 'SET storage_engine=MyISAM',
+        'CONN_MAX_AGE': 3600,
+        'TEST' : {
+            'NAME': 'gracedb_test_db',
         },
-    }
+    },
 }
 
 # Main server "hostname" - a little hacky but OK
 SERVER_HOSTNAME = SERVER_FQDN.split('.')[0]
 
-# LVAlert Overseer settings - get from environment
-LVALERT_OVERSEER_INSTANCES = [
-    {
-        "lvalert_server": lvalert_server,
-        "listen_port": int(lvalert_overseer_port),
-        "username": lvalert_user,
-        "password": lvalert_password,
-    },
-]
+# igwn_alert  Overseer settings - get from environment
+LVALERT_OVERSEER_INSTANCES = []
+LVALERT_OVERSEER_INSTANCES.append(
+{
+    "lvalert_server": igwn_alert_server,
+    "listen_port": int(igwn_alert_overseer_port),
+    "igwn_alert_group": igwn_alert_group,
+    "username": igwn_alert_user,
+    "password": igwn_alert_password,
+}
+)
+
+# Pull in remaining (phone/email) alert variables from 
+# the environment. Default to false.
+SEND_PHONE_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_PHONE_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+SEND_EMAIL_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_EMAIL_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+SEND_MATTERMOST_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_MATTERMOST_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+INSTANCE_STUB = """
+<li>Phone alerts (calls/SMS) are {0}</li>
+<li>Email alerts are {1}</li>
+<li><span class="text-monospace">igwn-alert</span> messages to <span class="text-monospace">{2}</span> are {3}</li>
+"""
+
+INSTANCE_LIST = INSTANCE_STUB.format(ENABLED[SEND_PHONE_ALERTS],
+                                ENABLED[SEND_EMAIL_ALERTS],
+                                LVALERT_OVERSEER_INSTANCES[0]['lvalert_server'],
+                                ENABLED[SEND_XMPP_ALERTS])
 
 # Use full client certificate to authenticate
 REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] = (
     'api.backends.GraceDbAuthenticatedAuthentication',
+    'api.backends.GraceDbSciTokenAuthentication',
     'api.backends.GraceDbX509FullCertAuthentication',
     'api.backends.GraceDbBasicAuthentication',
 )
@@ -129,3 +392,10 @@ EMBB_IGNORE_ADDRESSES = ['Mailer-Daemon@{fqdn}'.format(fqdn=SERVER_FQDN)]
 for key in LOGGING['loggers']:
     LOGGING['loggers'][key]['handlers'] = ['console']
 LOGGING['loggers']['django.request']['handlers'].append('mail_admins')
+
+# Turn off debug/error emails when in maintenance mode.
+if MAINTENANCE_MODE:
+    LOGGING['loggers']['django.request']['handlers'].remove('mail_admins')
+
+# Set SciToken accepted audience to server FQDN
+SCITOKEN_AUDIENCE = ["https://" + SERVER_FQDN, "https://" + LIGO_FQDN]

config/settings/container/dev.py

 # Settings for a test/dev GraceDB instance running in a container
 from .base import *
 
-CONFIG_NAME = "TEST"
+TIER = "dev"
+CONFIG_NAME = "DEV"
 
 # Debug settings
 DEBUG = True
@@ -28,6 +29,11 @@ INSTALLED_APPS += [
 # Add testserver to ALLOWED_HOSTS
 ALLOWED_HOSTS += ['testserver']
 
+# Enforce that phone and email alerts are off XXX: Set by deployment variables!
+#SEND_PHONE_ALERTS = False
+#SEND_EMAIL_ALERTS = False
+#SEND_MATTERMOST_ALERTS = True
+
 # Settings for django-silk profiler
 SILKY_AUTHENTICATION = True
 SILKY_AUTHORISATION = True
@@ -46,7 +52,45 @@ INTERNAL_IPS = [
     INTERNAL_IP_ADDRESS,
 ]
 
-# Adjust ADMINS for dev instances
-ADMINS = [
-    ("Tanner Prestegard", "tanner.prestegard@ligo.org"),
-]
+# Set up Sentry for error logging
+sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
+if sentry_dsn is not None:
+    USE_SENTRY = True
+
+    # Set up Sentry
+    import sentry_sdk
+    from sentry_sdk.integrations.django import DjangoIntegration
+    sentry_sdk.init(
+        environment='dev',
+        dsn=sentry_dsn,
+        integrations=[DjangoIntegration()],
+        before_send=before_send,
+    )
+
+    # Turn off default admin error emails
+    LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Development VM'
+
+# Add sub-bullet with igwn-alert group:
+group_sub_bullet = """<ul>
+<li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
+INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_TITLE = 'GraceDB Development Server'
+INSTANCE_INFO = """
+<h5>Development Instance</h5>
+<hr>
+<p>
+This GraceDB instance is designed for GraceDB maintainers to develop and
+test in the AWS cloud architecture. There is <b>no guarantee</b> that the
+behavior of this instance will mimic the production system at any time. 
+Events and associated data may change or be removed at any time. 
+</p>
+<ul>
+{}
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+</ul>
+""".format(INSTANCE_LIST)

config/settings/container/playground.py

+# Settings for a playground GraceDB instance (for user testing) running
+# in a container on AWS. These settings inherent from base.py) 
+# and overrides or adds to them.
+from .base import *
+
+TIER = "playground"
+CONFIG_NAME = "USER TESTING"
+
+# Debug settings
+DEBUG = False
+
+# Override EMBB email address
+# TP (8 Aug 2017): not sure why?
+EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
+
+# Enforce that phone and email alerts are off XXX: Set by deployment variables!
+#SEND_PHONE_ALERTS = False
+#SEND_EMAIL_ALERTS = False
+
+# Enable Mattermost alerts
+SEND_MATTERMOST_ALERTS = True
+
+# Add testserver to ALLOWED_HOSTS
+ALLOWED_HOSTS += ['testserver']
+
+# Set up Sentry for error logging
+sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
+if sentry_dsn is not None:
+    USE_SENTRY = True
+
+    # Set up Sentry
+    import sentry_sdk
+    from sentry_sdk.integrations.django import DjangoIntegration
+    sentry_sdk.init(
+        environment='playground',
+        dsn=sentry_dsn,
+        integrations=[DjangoIntegration()],
+        before_send=before_send,
+    )
+
+    # Turn off default admin error emails
+    LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Playground'
+
+# Add sub-bullet with igwn-alert group:
+group_sub_bullet = """<ul>
+<li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
+INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_INFO = """
+<h5>Playground instance</h5>
+<hr>
+<p>
+This GraceDB instance is designed for users to develop and test their own
+applications. It mimics the production instance in all but the following ways:
+</p>
+<ul>
+{}
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>Events and associated data will <b>not</b> be preserved indefinitely.
+A nightly cron job removes events older than 21 days.</li>
+</ul>
+""".format(INSTANCE_LIST)
+
+# Safety check on debug mode for playground
+if (DEBUG == True):
+    raise RuntimeError("Turn off debug mode for playground")

config/settings/container/production.py

 # Settings for a production GraceDB instance running in a container
-from django.core.exceptions import ImproperlyConfigured
 from .base import *
 
-DEBUG = False
-
-# Turn on alerts
-SEND_XMPP_ALERTS = True
-SEND_PHONE_ALERTS = True
-SEND_EMAIL_ALERTS = True
-
-# Priority server?
-PRIORITY_SERVER = False
-is_priority_server = os.environ.get('DJANGO_PRIORITY_SERVER', None)
-if (isinstance(is_priority_server, str) and
-    is_priority_server.lower() in ['true', 't']):
-    PRIORITY_SERVER = True
+TIER = "production"
 
-# If priority server, only allow priority users
-if PRIORITY_SERVER:
-    # Add custom permissions for API
-    default_perms = list(REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'])
-    default_perms = ['api.permissions.IsPriorityUser'] + default_perms
-    REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'] = tuple(default_perms)
+DEBUG = False
 
+# Turn on alerts: XXX: Set by deployment variables!
+#SEND_PHONE_ALERTS = True
+#SEND_EMAIL_ALERTS = True
+#SEND_MATTERMOST_ALERTS = True
 
 # TP, March 2019: for now, it looks infeasible to use multiple databases
 # since there are many operations which normal LVC users can do that
@@ -33,6 +19,7 @@ if PRIORITY_SERVER:
 # reworked properly.  I.e. this is a much bigger project than expected
 # so we're going to have to revisit it at some point.  We'll leave the
 # config here for now.
+# if not PRIORITY_SERVER:
 #    # If not a priority server, we use the read-only replica database
 #    # for reads and master for writes.
 #    # The username, password, and database name are all replicated
@@ -55,6 +42,49 @@ if PRIORITY_SERVER:
 #    # Set up database router
 #    DATABASE_ROUTERS = ['core.db.routers.NonPriorityRouter',]
 
+# Set up Sentry for error logging
+sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
+if sentry_dsn is not None:
+    USE_SENTRY = True
+
+    # Set up Sentry
+    import sentry_sdk
+    from sentry_sdk.integrations.django import DjangoIntegration
+    sentry_sdk.init(
+        environment='production',
+        dsn=sentry_dsn,
+        integrations=[DjangoIntegration()],
+        before_send=before_send,
+    )
+
+    # Turn off default admin error emails
+    LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB'
+
+# Add sub-bullet with igwn-alert group:
+group_sub_bullet = """<ul>
+<li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
+INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_INFO = """
+<h5>GraceDB Notifications</h5>
+<hr>
+<p>
+GraceDB notifies registered users of Gravitational-Wave candidate detections 
+in real-time during LIGO/Virgo/KAGRA observation periods. Current notifications 
+mechanisms are:
+</p>
+<ul>
+{}
+</ul>
+""".format(INSTANCE_LIST)
+
 # Safety check on debug mode for production
 if (DEBUG == True):
     raise RuntimeError("Turn off debug mode for production")
+
+# Hardcode pipelines not approved for production:
+UNAPPROVED_PIPELINES += ['aframe', 'GWAK']

config/settings/container/test.py

+# Settings for a test/dev GraceDB instance running in a container
+from .base import *
+
+TIER = "test"
+CONFIG_NAME = "TEST"
+
+# Debug settings
+DEBUG = True
+
+# Override EMBB email address
+# TP (8 Aug 2017): not sure why?
+EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
+
+# Add middleware
+debug_middleware = 'debug_toolbar.middleware.DebugToolbarMiddleware'
+MIDDLEWARE += [
+    debug_middleware,
+    #'silk.middleware.SilkyMiddleware',
+    #'core.middleware.profiling.ProfileMiddleware',
+    #'core.middleware.admin.AdminsOnlyMiddleware',
+]
+
+# Add to installed apps
+INSTALLED_APPS += [
+    'debug_toolbar',
+    #'silk'
+]
+
+# Add testserver to ALLOWED_HOSTS
+ALLOWED_HOSTS += ['testserver']
+
+# Settings for django-silk profiler
+SILKY_AUTHENTICATION = True
+SILKY_AUTHORISATION = True
+if 'silk' in INSTALLED_APPS:
+    # Needed to prevent RequestDataTooBig for files > 2.5 MB
+    # when silk is being used. This setting is typically used to
+    # prevent DOS attacks, so should not be changed in production.
+    DATA_UPLOAD_MAX_MEMORY_SIZE = 20*(1024**2)
+
+# Tuple of IPs which are marked as internal, useful for debugging.
+# Tanner (5 Dec. 2017): DON'T CHANGE THIS! Django Debug Toolbar exposes
+# some headers which we want to keep hidden.  So to be safe, we only allow
+# it to be used through this server.  You need to configure a SOCKS proxy
+# on your local machine to use DJDT (see admin docs).
+INTERNAL_IPS = [
+    INTERNAL_IP_ADDRESS,
+]
+
+# Enforce that phone and email alerts are off XXX: Set by deployment variables!
+#SEND_PHONE_ALERTS = False
+#SEND_EMAIL_ALERTS = False
+#SEND_MATTERMOST_ALERTS = True
+
+# Set up Sentry for error logging
+sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
+if sentry_dsn is not None:
+    USE_SENTRY = True
+
+    # Set up Sentry
+    import sentry_sdk
+    from sentry_sdk.integrations.django import DjangoIntegration
+    sentry_sdk.init(
+        environment='test',
+        dsn=sentry_dsn,
+        integrations=[DjangoIntegration()],
+        before_send=before_send,
+    )
+
+    # Turn off default admin error emails
+    LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Testing Server'
+
+# Add sub-bullet with igwn-alert group:
+group_sub_bullet = """<ul>
+<li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
+INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_INFO = """
+<h5>Testing Instance</h5>
+<hr>
+<p>
+This GraceDB instance is designed for Quality Assurance (QA) testing and
+validation for GraceDB and electromagnetic follow-up (EMFollow) developers.
+Software should meet QA milestones on the test instance before being moved 
+to Playground or Production. Note, on this GraceDB instance:
+</p>
+<ul>
+{}
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+</ul>
+""".format(INSTANCE_LIST)

config/settings/vm/base.py

@@ -10,13 +10,13 @@ import socket
 DATABASES = {
     'default' : {
         'NAME': 'gracedb',
-        'ENGINE': 'django.db.backends.mysql',
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
         'USER': 'gracedb',
         'PASSWORD': DB_PASSWORD,
-        'OPTIONS': {
-            'init_command': 'SET storage_engine=MyISAM',
-        },
-    }
+        'HOST':'127.0.0.1',
+        'PORT':'5432',
+        'CONN_MAX_AGE': 3600,
+    },
 }
 
 # Set up allowed hosts
@@ -38,3 +38,148 @@ EMBB_MAIL_ADDRESS = 'embb@{fqdn}.ligo.org'.format(fqdn=SERVER_FQDN)
 EMBB_SMTP_SERVER = 'localhost'
 EMBB_MAIL_ADMINS = [admin[1] for admin in ADMINS]
 EMBB_IGNORE_ADDRESSES = ['Mailer-Daemon@{fqdn}'.format(fqdn=SERVER_FQDN)]
+
+# Load modified caching middleware:
+# https://docs.djangoproject.com/en/2.2/ref/middleware/#middleware-ordering
+MIDDLEWARE = [
+    'django.middleware.cache.UpdateCacheMiddleware',
+    'django.middleware.gzip.GZipMiddleware',
+    'events.middleware.PerformanceMiddleware',
+    'core.middleware.accept.AcceptMiddleware',
+    'core.middleware.api.ClientVersionMiddleware',
+    'core.middleware.api.CliExceptionMiddleware',
+    'core.middleware.proxy.XForwardedForMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'user_sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'core.middleware.maintenance.MaintenanceModeMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'ligoauth.middleware.ShibbolethWebAuthMiddleware',
+    'ligoauth.middleware.ControlRoomMiddleware',
+    'django.middleware.cache.FetchFromCacheMiddleware',
+]
+
+# Set caches:
+CACHE_MIDDLEWARE_SECONDS = 5
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
+        'LOCATION': 'localhost:11211',
+        'TIMEOUT': 60,
+        'KEY_PREFIX': 'NULL',
+        'OPTIONS': {
+            'ignore_exc': True,
+            }
+    },
+    # For API throttles
+    'throttles': {
+        'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
+        'LOCATION': 'api_throttle_cache', # Table name
+    },    
+}
+
+# FIXME: hardwire this for now in the VMs for testing
+ENABLE_REDIS_QUEUE = True
+
+if ENABLE_REDIS_QUEUE:
+    # For async alert follow-up:
+    CACHES.update({"async_followup": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": f"redis://{REDIS_QUEUE_ADDRESS}:{REDIS_QUEUE_PORT}/{REDIS_QUEUE_DATABASE}",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        }
+    }})
+
+
+    # Set queue backend for async django tasks:
+    # example django-redis connection
+    Q_CLUSTER = {
+        'name': Q_CLUSTER_NAME,
+        'label': Q_CLUSTER_LABEL,
+        'retry': REDIS_QUEUE_RETRY,
+        'timeout': REDIS_QUEUE_TIMEOUT,
+        'workers': REDIS_QUEUE_WORKERS,
+        'recycle': REDIS_QUEUE_RECYCLE,
+        'django_redis': 'async_followup'
+    }
+
+# DB "cool-down" factor for when a db conflict is detected. This
+# factor scales a random number of seconds between zero and one.
+DB_SLEEP_FACTOR = get_from_env(
+        'DJANGO_DB_SLEEP_FACTOR',
+        default_value=1.0,
+        fail_if_not_found=False
+)
+# Fix the factor (str to float)
+try:
+    DB_SLEEP_FACTOR = float(DB_SLEEP_FACTOR)
+except:
+    DB_SLEEP_FACTOR = 1.0
+
+BETA_REPORTS_LINK = True
+
+## EGAD (External GraceDB Alert Dispatcher) configuration
+ENABLE_EGAD_EMAIL = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_EMAIL',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_KAFKA = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_KAFKA',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_MATTERMOST = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_MATTERMOST',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_PHONE = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_PHONE',
+                 fail_if_not_found=False, default_value="false")
+)
+
+ENABLE_EGAD = (
+    ENABLE_EGAD_EMAIL or ENABLE_EGAD_KAFKA
+    or ENABLE_EGAD_MATTERMOST or ENABLE_EGAD_PHONE
+)
+
+
+# Pull in remaining (phone/email) alert variables from
+# the environment. Default to false.
+SEND_PHONE_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_PHONE_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+SEND_EMAIL_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_EMAIL_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+SEND_MATTERMOST_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_MATTERMOST_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+INSTANCE_STUB = """
+<li>Phone alerts (calls/SMS) are {0}</li>
+<li>Email alerts are {1}</li>
+<li><span class="text-monospace">igwn-alert</span> messages to <span class="text-monospace">{2}</span> are {3}</li>
+"""
+
+INSTANCE_LIST = INSTANCE_STUB.format(ENABLED[SEND_PHONE_ALERTS],
+                                ENABLED[SEND_EMAIL_ALERTS],
+                                LVALERT_OVERSEER_INSTANCES[0]['lvalert_server'],
+                                ENABLED[SEND_XMPP_ALERTS])
+
+if (len(LVALERT_OVERSEER_INSTANCES) == 2):
+    IGWN_STUB = '<li><span class="text-monospace">igwn-alert</span> messages to <span class="text-monospace">{0}</span> are {1}</li>'
+    IGWN_LIST = IGWN_STUB.format(LVALERT_OVERSEER_INSTANCES[1]['lvalert_server'],
+                                ENABLED[SEND_XMPP_ALERTS])
+    INSTANCE_LIST = INSTANCE_LIST + IGWN_LIST
+
+# Set SciToken accepted audience to server FQDN
+SCITOKEN_AUDIENCE = ["https://" + SERVER_FQDN, "https://" + LIGO_FQDN]

config/settings/vm/dev.py

@@ -4,10 +4,13 @@
 import socket
 from .base import *
 
-CONFIG_NAME = "TEST"
+TIER = "dev"
+CONFIG_NAME = "DEV"
 
 # Debug settings
 DEBUG = True
+SEND_XMPP_ALERTS=True
+SEND_MATTERMOST_ALERTS=True
 
 # Override EMBB email address
 # TP (8 Aug 2017): not sure why?
@@ -25,7 +28,6 @@ MIDDLEWARE += [
 # Add to installed apps
 INSTALLED_APPS += [
     'debug_toolbar',
-    #'silk'
 ]
 
 # Add testserver to ALLOWED_HOSTS
@@ -48,8 +50,33 @@ if 'silk' in INSTALLED_APPS:
 INTERNAL_IPS = [
     INTERNAL_IP_ADDRESS,
 ]
+INSTANCE_TITLE = 'GraceDB Development VM'
 
-# Adjust ADMINS for dev instances
-ADMINS = [
-    ("Tanner Prestegard", "tanner.prestegard@ligo.org"),
-]
+# Add sub-bullet with igwn-alert group:
+if (len(LVALERT_OVERSEER_INSTANCES) == 2):
+    igwn_alert_group = os.environ.get('IGWN_ALERT_GROUP', 'lvalert-dev')
+    group_sub_bullet = """<ul>
+    <li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+    </ul>""".format(igwn_alert_group)
+    INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_INFO = """
+<h5>Development Instance</h5>
+<hr>
+<p>
+This GraceDB instance is designed for GraceDB maintainers to develop and
+test in the AWS cloud architecture. There is <b>no guarantee</b> that the
+behavior of this instance will mimic the production system at any time.
+Events and associated data may change or be removed at any time.
+</p>
+<ul>
+{}
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+</ul>
+""".format(INSTANCE_LIST)
+
+# Turn off public page caching for development and testing:
+PUBLIC_PAGE_CACHING = 0
+
+# Hardcode pipelines not approved for production (for vm testing)
+# UNAPPROVED_PIPELINES += ['aframe', 'GWAK']

config/settings/vm/playground.py

@@ -3,6 +3,7 @@
 # from base.py settings) and overrides or adds to them.
 from .base import *
 
+TIER = "playground"
 CONFIG_NAME = "USER TESTING"
 
 # Debug settings
@@ -15,6 +16,9 @@ EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
 # Turn on XMPP alerts
 SEND_XMPP_ALERTS = True
 
+# Turn on Mattermost alerts
+SEND_MATTERMOST_ALERTS = True
+
 # Enforce that phone and email alerts are off
 SEND_PHONE_ALERTS = False
 SEND_EMAIL_ALERTS = False
@@ -33,18 +37,17 @@ ALLOWED_HOSTS += ['testserver']
 # Home page stuff
 INSTANCE_TITLE = 'GraceDB Playground'
 INSTANCE_INFO = """
+<h3>Playground instance</h3>
 <p>
 This GraceDB instance is designed for users to develop and test their own
 applications. It mimics the production instance in all but the following ways:
 </p>
 <ul>
-<li>Phone and e-mail alerts are turned off</li>
-<li>Only LIGO logins are provided (no login via InCommon or Google)</li>
-<li>LVAlert messages are sent to lvalert-playground.cgca.uwm.edu</li>
+<li>Phone and e-mail alerts are turned off.</li>
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>LVAlert messages are sent to lvalert-playground.cgca.uwm.edu.</li>
 <li>Events and associated data will <b>not</b> be preserved indefinitely.
-A nightly cron job removes events older than 14 days.</li>
-<li><b>Note:</b> for O3 development, the above number has been updated to
-<b>112</b>.</li>
+A nightly cron job removes events older than 21 days.</li>
 </ul>
 """
 

config/settings/vm/production.py

@@ -3,6 +3,8 @@
 # settings) and overrides or adds to them.
 from .base import *
 
+TIER = "production"
+
 DEBUG = False
 
 # LVAlert Overseer settings
@@ -17,6 +19,7 @@ LVALERT_OVERSEER_INSTANCES = [
 SEND_XMPP_ALERTS = True
 SEND_PHONE_ALERTS = True
 SEND_EMAIL_ALERTS = True
+SEND_MATTERMOST_ALERTS = True
 
 # Safety check on debug mode for production
 if (DEBUG == True):

config/urls.py

 from django.conf import settings
-from django.conf.urls import url, include
+
+from django.urls import re_path, include
 
 from django.contrib import admin
-from django.contrib.auth.views import logout
+from django.contrib.auth.views import LogoutView
 from django.views.generic import TemplateView
 
 # Import feeds
@@ -10,7 +11,9 @@ import core.views
 from events.feeds import EventFeed, feedview
 import events.reports
 import events.views
-from ligoauth.views import pre_login, post_login, shib_logout, manage_password
+from ligoauth.views import (
+    manage_password, ShibLoginView, ShibPostLoginView
+)
 import search.views
 
 # Django admin auto-discover
@@ -22,54 +25,54 @@ feeds = {
 
 
 urlpatterns = [
-    url(r'^$', events.views.index, name="home"),
-    url(r'^navbar_only$', TemplateView.as_view(
+    re_path(r'^$', events.views.index, name="home"),
+    re_path(r'^navbar_only$', TemplateView.as_view(
         template_name='navbar_only.html'), name="navbar-only"),
-    url(r'^SPInfo', TemplateView.as_view(template_name='gracedb/spinfo.html'),
+    re_path(r'^SPInfo', TemplateView.as_view(template_name='gracedb/spinfo.html'),
          name="spinfo"),
-    url(r'^SPPrivacy', TemplateView.as_view(
+    re_path(r'^SPPrivacy', TemplateView.as_view(
         template_name='gracedb/spprivacy.html'), name="spprivacy"),
-    url(r'^DiscoveryService', TemplateView.as_view(
+    re_path(r'^DiscoveryService', TemplateView.as_view(
         template_name='discovery.html'), name="discovery"),
-    url(r'^events/', include('events.urls')),
-    url(r'^superevents/', include('superevents.urls')),
-    url(r'^alerts/', include('alerts.urls')),
-    url(r'^feeds/(?P<url>.*)/$', EventFeed()),
-    url(r'^feeds/$', feedview, name="feeds"),
-
-    url(r'^performance/$', events.views.performance, name="performance"),
-    url(r'^reports/$', events.reports.histo, name="reports"),
-    url(r'^reports/cbc_report/(?P<format>(json|flex))?$',
-        events.reports.cbc_report, name="cbc_report"),
-    url(r'^latest/$', search.views.latest, name="latest"),
+    re_path(r'^events/', include('events.urls')),
+    re_path(r'^superevents/', include('superevents.urls')),
+    re_path(r'^alerts/', include('alerts.urls')),
+    re_path(r'^feeds/(?P<url>.*)/$', EventFeed()),
+    re_path(r'^feeds/$', feedview, name="feeds"),
+
+    re_path(r'^other/$', TemplateView.as_view(template_name='other.html'),
+        name='other'),
+    re_path(r'^performance/$', events.views.performance, name="performance"),
+    re_path(r'^reports/$', events.reports.reports_page_context, name="reports"),
+    re_path(r'^latest/$', search.views.latest, name="latest"),
     #(r'^reports/(?P<path>.+)$', 'django.views.static.serve',
     #        {'document_root': settings.LATENCY_REPORT_DEST_DIR}),
-    url(r'^search/$', search.views.search, name="mainsearch"),
+    re_path(r'^search/$', search.views.search, name="mainsearch"),
 
     # Authentication
-    url(r'^login/$', pre_login, name='login'),
-    url(r'^post-login/$', post_login, name='post-login'),
-    url(r'^logout/$', shib_logout, name='logout'),
+    re_path(r'^login/$', ShibLoginView.as_view(), name='login'),
+    re_path(r'^post-login/$', ShibPostLoginView.as_view(), name='post-login'),
+    re_path(r'^logout/$', LogoutView.as_view(), name='logout'),
 
     # Password management
-    url('^manage-password/$', manage_password, name='manage-password'),
+    re_path('^manage-password/$', manage_password, name='manage-password'),
 
     # API URLs
-    url(r'^api/', include('api.urls')),
+    re_path(r'^api/', include('api.urls')),
     # Legacy API URLs - must be maintained!
-    url(r'^apibasic/', include('api.urls', namespace='legacy_apibasic')),
-    url(r'^apiweb/', include('api.urls', namespace='legacy_apiweb')),
+    re_path(r'^apibasic/', include('api.urls', namespace='legacy_apibasic')),
+    re_path(r'^apiweb/', include('api.urls', namespace='legacy_apiweb')),
 
     # Heartbeat URL
-    url(r'^heartbeat/$', core.views.heartbeat, name='heartbeat'),
+    re_path(r'^heartbeat/$', core.views.heartbeat, name='heartbeat'),
 
     # Uncomment the admin/doc line below and add 'django.contrib.admindocs'
     # to INSTALLED_APPS to enable admin documentation:
     # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
-    url(r'^admin/', admin.site.urls),
+    re_path(r'^admin/', admin.site.urls),
 
-    # sessions
-    #url(r'', include('user_sessions.urls', 'user_sessions')),
+    # Sessions
+    re_path(r'^', include('user_sessions.urls', 'user_sessions')),
 
 ]
 
@@ -79,12 +82,12 @@ urlpatterns = [
 if ('silk' in settings.INSTALLED_APPS):
     # Add django-silk
     urlpatterns = [
-        url(r'^silk/', include('silk.urls', namespace='silk'))
+        re_path(r'^silk/', include('silk.urls', namespace='silk'))
     ] + urlpatterns
 
 # Add django-debug-toolbar
 if settings.DEBUG and 'debug_toolbar' in settings.INSTALLED_APPS:
     import debug_toolbar
     urlpatterns = [
-        url(r'^__debug__/', include(debug_toolbar.urls)),
+        re_path(r'^__debug__/', include(debug_toolbar.urls)),
     ] + urlpatterns

docker/apache-config

@@ -9,6 +9,9 @@ ServerName ${DJANGO_PRIMARY_FQDN}
 
   ServerAdmin cgca-admins@uwm.edu
 
+  ## Log format
+  LogFormat "APACHE | %a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""
+
   ## Vhost docroot
   DocumentRoot "/var/www/html"
 
@@ -20,6 +23,13 @@ ServerName ${DJANGO_PRIMARY_FQDN}
     Require all granted
   </Directory>
 
+  # Improve proxy behavior with gunicorn:
+  # https://serverfault.com/questions/206738/intermittent-error-when-using-mod-proxy-to-do-reverse-proxy-to-soap-service#comment1327184_209006
+  # https://github.com/benoitc/gunicorn/issues/207
+
+  SetEnv force-proxy-request-1.0 1
+  SetEnv proxy-nokeepalive 1
+
   ## Custom fragment
   # gUnicorn edits
   Alias /shibboleth-ds/idpselect_config.js /etc/shibboleth-ds/idpselect_config.js
@@ -37,7 +47,33 @@ ServerName ${DJANGO_PRIMARY_FQDN}
   ProxyPass "/static" "!"
   ProxyPass "/documentation" "!"
   ProxyPass "/admin_docs" "!"
-  ProxyPass "/" "http://localhost:8080/"
+  ProxyPass "/" "http://localhost:8080/" timeout=120
+  ProxyPassReverse "/" "http://localhost:8080/"
+
+  # This section is for apache2 timeout and keepalive tuning parameters. 
+  # https://ioflood.com/blog/2020/02/21/what-is-apache-keepalive-timeout-how-to-optimize-this-critical-setting/
+  
+  # KeepAlive will... keep a connection alive for subsequent requests. 
+  # Turn this on.
+  KeepAlive On
+
+  # The maximum number of requests served to a client before terminating the connection.
+  # This can be large, possibly safely unlimited. (0 = unlimited)
+  MaxKeepAliveRequests 0
+
+  # The number of seconds Apache will wait for a subsequent request before closing the 
+  # connection. Once a request has been received, the timeout value specified by the 
+  # Timeout directive applies. Setting KeepAliveTimeout to a high value may cause 
+  # performance problems in heavily loaded servers. The higher the timeout, the more 
+  # server processes will be kept occupied waiting on connections with idle clients
+  KeepAliveTimeout 5
+
+  # Amount of time the server will wait for certain events before failing a 
+  # request. The TimeOut directive defines the length of time Apache will wait for
+  # I/O (e.g., when reading data from the client, when writing data to the client, etc.) 
+  # Default: 300s. Try setting this lower, then do a test like a long query with the API
+  # and in the browser and see what happens. 
+  Timeout 60
 
   # Unset certain headers to help prevent spoofing
   RequestHeader unset REMOTE_USER
@@ -55,6 +91,9 @@ ServerName ${DJANGO_PRIMARY_FQDN}
   # Set X_FORWARDED_PROTO to https
   RequestHeader set X_FORWARDED_PROTO "https"
 
+  # Increase the max allowable header size:
+  LimitRequestFieldSize 16384
+
   # Set up mod_xsendfile for serving static event files as directed by Django
   XSendFile On
   XSendFilePath /app/db_data/
@@ -79,7 +118,7 @@ ServerName ${DJANGO_PRIMARY_FQDN}
     Require all granted
   </Directory>
 
-  Alias /robots.txt /home/gracedb/gracedb_project/static_root/robots.txt
+  Alias /robots.txt /app/gracedb_project/static_root/robots.txt
 
   <Location /Shibboleth.sso>
     SetHandler shib
@@ -116,7 +155,7 @@ ServerName ${DJANGO_PRIMARY_FQDN}
     AuthType Shibboleth
     ShibRequestSetting requireSession true
     ShibUseHeaders On
-    Require shib-user tanner.prestegard@LIGO.ORG alexander.pace@LIGO.ORG patrick.brady@LIGO.ORG thomas.downes@LIGO.ORG
+    Require shib-user duncan.meacher@ligo.org alexander.pace@ligo.org daniel.wysocki@ligo.org patrick.brady@ligo.org
   </Location>
 
 </VirtualHost>