Compare revisions

366db43d · 139cdc0c · 52d2f12b · 860fb428 · 619080fa · 20bb87a9
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,5 @@ docs/user_docs/build/*
 docs/admin_docs/build/*
 static_root/*
 .pytest_cache
+junit.xml
+.coverage
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,60 +16,133 @@ stages:
 before_script:
  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY

+include:
+  # Container scanning
+  - component: $CI_SERVER_FQDN/computing/gitlab/components/container-scanning/container-scanning@~latest
+    inputs:
+      job_name: branch_scan
+
+  # Software scanning
+  - component: $CI_SERVER_FQDN/computing/gitlab/components/sast/sast@~latest
+    inputs:
+      run_advanced_sast: true
+  - component: $CI_SERVER_FQDN/computing/gitlab/components/secret-detection/secret-detection@~latest
+  - component: $CI_SERVER_FQDN/computing/gitlab/components/python/dependency-scanning@~latest
+
+# -- software scanning
+# overwrite some settings for the scanning jobs
+
+dependency_scanning:
+  stage: test
+  needs: []
+  variables:
+    DEBIAN_FRONTEND: "noninteractive"
+  before_script:
+    # install some underlying utilities using `apt` so that the dependency
+    # scanner can use pip to install everything else
+    - apt-get update -yqq
+    - apt-get install -yqq
+        libkrb5-dev
+        libldap-dev
+        libsasl2-dev
+
+.sast-analyzer:
+  stage: test
+  needs: []
+  before_script: []
+
+secret_detection:
+  stage: test
+  needs: []
+  before_script: []
+
+# -- testing
+
 .test: &test
-  image: ligo/base:stretch
+  image: igwn/base:bookworm
  services:
-    - mysql:5.5
+    - postgres:15.6
+    - memcached
  variables:
    AWS_SES_ACCESS_KEY_ID: "fake_aws_id"
    AWS_SES_SECRET_ACCESS_KEY: "fake_aws_key"
    DJANGO_ALERT_EMAIL_FROM: "fake_email"
-    DJANGO_DB_HOST: "mysql"
-    DJANGO_DB_PORT: "3306"
+    DJANGO_DB_HOST: "postgres"
+    DJANGO_DB_PORT: "5432"
    DJANGO_DB_NAME: "fake_name"
-    DJANGO_DB_USER: "root"
-    DJANGO_DB_PASSWORD: "fake_password"
+    DJANGO_DB_USER: "runner"
+    DJANGO_DB_PASSWORD: ""
    DJANGO_PRIMARY_FQDN: "fake_fqdn"
    DJANGO_SECRET_KEY: "fake_key"
    DJANGO_SETTINGS_MODULE: "config.settings.container.dev"
    DJANGO_TWILIO_ACCOUNT_SID: "fake_sid"
    DJANGO_TWILIO_AUTH_TOKEN: "fake_token"
+    DJANGO_DOCKER_MEMCACHED_ADDR: "memcached:11211"
+    EGAD_URL: "fake_url"
+    EGAD_API_KEY: "fake_key"
+    ENABLE_LVALERT_OVERSEER: "false"
+    ENABLE_IGWN_OVERSEER: "false"
    LVALERT_OVERSEER_PORT: "2"
    LVALERT_SERVER: "fake_server"
    LVALERT_USER: "fake_user"
    LVALERT_PASSWORD: "fake_password"
-    MYSQL_DB: "${DJANGO_DB_NAME}"
-    MYSQL_ROOT_PASSWORD: "${DJANGO_DB_PASSWORD}"
+    ENABLE_IGWN_OVERSEER: "false"
+    IGWN_ALERT_OVERSEER_PORT: "2"
+    IGWN_ALERT_SERVER: "fake_server"
+    IGWN_ALERT_USER: "fake_user"
+    IGWN_ALERT_PASSWORD: "fake_password"
+    POSTGRES_DB: "${DJANGO_DB_NAME}"
+    POSTGRES_USER: "${DJANGO_DB_USER}"
+    POSTGRES_PASSWORD: "${DJANGO_DB_PASSWORD}"
+    POSTGRES_HOST_AUTH_METHOD: trust
  before_script:
    # create apt cache directory
    - mkdir -pv ${APT_CACHE_DIR}
    # set python version
    - PYTHON_VERSION="${CI_JOB_NAME##*:}"
    - PYTHON_MAJOR="${PYTHON_VERSION:0:1}"
-    - if [[ "${PYTHON_MAJOR}" -eq 2 ]]; then PYTHON="python"; else PYTHON="python3"; fi
+    - PYTHON="python3"
    # install build requirements
+    - apt-get -y install gnupg
+    - sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+    - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
    - apt-get -yqq update
    - apt-get -o dir::cache::archives="${APT_CACHE_DIR}" install -yqq
          git
-          libmariadbclient-dev
+          gnupg
          libldap2-dev
          libsasl2-dev
          libssl-dev
          libxml2-dev
+          krb5-user
+          libkrb5-dev
+          libsasl2-modules-gssapi-mit
          swig
+          pkg-config 
+          libpng-dev 
+          libfreetype6-dev 
+          libxslt-dev
          ${PYTHON}-pip
-    # install voeventlib for python2
-    - if [[ "${PYTHON_MAJOR}" -eq 2 ]]; then apt-get -o dir::cache::archives="${APT_CACHE_DIR}" install -yqq python-voeventlib; fi
+          postgresql-15
+          postgresql-client-15
+          libpq-dev
+    # upgrade pip (requirement for lalsuite)
+    - ${PYTHON} -m pip install --upgrade pip --break-system-packages
    # install everything else from pip
-    - ${PYTHON} -m pip install -r requirements.txt
+    - ${PYTHON} -m pip install -r requirements.txt --break-system-packages
    # create logs path required for tests
    - mkdir -pv ../logs/
    # list packages
    - ${PYTHON} -m pip list installed
  script:
-    - PYTHONPATH=${PYTHONPATH}:${PWD}/gracedb ${PYTHON} -m pytest --cov ./gracedb --junitxml=${CI_PROJECT_DIR}/junit.xml
+    - PYTHONPATH=${PYTHONPATH}:${PWD}/gracedb ${PYTHON} -m pytest --cov-report term-missing --cov ./gracedb --junitxml=${CI_PROJECT_DIR}/junit.xml
  after_script:
    - rm -fvr ${PIP_CACHE_DIR}/log
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
  artifacts:
    reports:
      junit: junit.xml
@@ -78,27 +151,56 @@ before_script:
    paths:
      - .cache/pip
      - .cache/apt
+  coverage: '/^TOTAL\s+.*\s+(\d+\.?\d*)%/'
+  tags:
+      - executor-docker

-test:2.7:
+test:3.11:
  <<: *test

-test:3.7:
-  <<: *test
-  allow_failure: true
+# -- docker

 branch_image:
  stage: branch
  script:
    - docker build --pull -t $DOCKER_BRANCH .
    - docker push $DOCKER_BRANCH
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
+  tags:
+      - executor-docker
+
+branch_scan:
+  stage: branch
+  needs: [branch_image]
+  # default rules spawn a merge request pipeline, we don't want that
+  rules:
+    - if: $CI_COMMIT_BRANCH
+  variables:
+    GIT_STRATEGY: fetch
+    # image to scan
+    CS_IMAGE: "$DOCKER_BRANCH"
+    # image to compare to
+    CS_DEFAULT_BRANCH_IMAGE: "$CI_REGISTRY/computing/gitlab/server:latest"
+    # path to Dockerfile for remediation
+    CS_DOCKERFILE_PATH: "Dockerfile"
+  before_script: []

 latest_image:
  stage: latest
  dependencies:
    - branch_image
-  only:
-    refs:
-      - master
  script:
+    - docker pull $DOCKER_BRANCH
    - docker tag $DOCKER_BRANCH $DOCKER_LATEST
    - docker push $DOCKER_LATEST
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
+  tags:
+      - executor-docker
--- a/Dockerfile
+++ b/Dockerfile
-FROM ligo/base:stretch
+FROM debian:bookworm
 LABEL name="LIGO GraceDB Django application" \
-      maintainer="tanner.prestegard@ligo.org" \
-      date="20190430"
+      maintainer="alexander.pace@ligo.org" \
+      date="20240306"
 ARG SETTINGS_MODULE="config.settings.container.dev"

 COPY docker/SWITCHaai-swdistrib.gpg /etc/apt/trusted.gpg.d
 COPY docker/backports.pref /etc/apt/preferences.d
-RUN echo 'deb http://pkg.switch.ch/switchaai/debian stretch main' > /etc/apt/sources.list.d/shibboleth.list
-RUN echo 'deb http://deb.debian.org/debian stretch-backports main' > /etc/apt/sources.list.d/backports.list
-RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
 RUN apt-get update && \
+    apt-get -y install gnupg curl
+
+
+RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' > /etc/apt/sources.list.d/backports.list
+RUN echo 'deb http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main' > /etc/apt/sources.list.d/pgdg.list
+RUN echo 'deb [trusted=yes] https://hypatia.aei.mpg.de/lsc-amd64-bookworm ./' > /etc/apt/sources.list.d/lscsoft.list
+RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+RUN apt-get update && \
+    apt-get --assume-yes upgrade && \
    apt-get install --install-recommends --assume-yes \
        apache2 \
+        emacs-nox \
        gcc \
        git \
+        krb5-user \
+        libkrb5-dev \
+        libapache2-mod-shib \
        libapache2-mod-xsendfile \
-        libmariadbclient-dev \
        libldap2-dev \
+        libldap-2.5-0 \
        libsasl2-dev \
+        libsasl2-modules-gssapi-mit \
        libxml2-dev \
+        pkg-config \
+        libpng-dev \
+        libpq-dev \
+        libfreetype6-dev \
+        libxslt-dev \
        libsqlite3-dev \
        ligo-ca-certs \
-        mariadb-client \
-        nodejs \
        osg-ca-certs \
-        python2.7 \
-        python2.7-dev \
-        python-libxml2 \
-        python-pip \
-        python-voeventlib \
+        php \
+        php8.2-pgsql \
+        php8.2-mbstring \
+        postgresql-client-15 \
+        python3 \
+        python3-dev \
+        python3-libxml2 \
+        python3-pip \
        procps \
-        shibboleth \
-        supervisor \
+        redis \
+        shibboleth-sp-common \
+        shibboleth-sp-utils \
        libssl-dev \
        swig \
+        htop \
+        telnet \
        vim && \
    apt-get clean && \
-    npm install -g bower
+    curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
+    apt-get update && apt-get install --assume-yes yarn
+
+# Install AWS X-ray daemon
+RUN curl -O https://s3.us-east-2.amazonaws.com/aws-xray-assets.us-east-2/xray-daemon/aws-xray-daemon-3.x.deb
+RUN dpkg -i aws-xray-daemon-3.x.deb
+RUN rm aws-xray-daemon-3.x.deb

+# Docker scripts:
 COPY docker/entrypoint /usr/local/bin/entrypoint
 COPY docker/cleanup /usr/local/bin/cleanup
+
+# Supervisord configs:
 COPY docker/supervisord.conf /etc/supervisor/supervisord.conf
 COPY docker/supervisord-apache2.conf /etc/supervisor/conf.d/apache2.conf
-COPY docker/supervisord-lvalert-overseer.conf /etc/supervisor/conf.d/overseer.conf
+COPY docker/supervisord-igwn-alert-overseer.conf /etc/supervisor/conf.d/igwn-overseer.conf
 COPY docker/supervisord-shibd.conf /etc/supervisor/conf.d/shibd.conf
-COPY docker/shibboleth-ds /etc/shibboleth-ds
+COPY docker/supervisord-aws-xray.conf /etc/supervisor/conf.d/aws-xray.conf
+COPY docker/supervisord-qcluster.conf /etc/supervisor/conf.d/qcluster.conf
+
+# Apache configs:
 COPY docker/apache-config /etc/apache2/sites-available/gracedb.conf
+COPY docker/mpm_prefork.conf /etc/apache2/mods-enabled/mpm_prefork.conf
+
+# Enable mpm_event module:
+
+RUN rm /etc/apache2/mods-enabled/mpm_prefork.*
+RUN rm /etc/apache2/mods-enabled/php8.2.*
+RUN cp  /etc/apache2/mods-available/mpm_event.* /etc/apache2/mods-enabled/
+
+# Shibboleth configs and certs:
+COPY docker/shibboleth-ds /etc/shibboleth-ds
 COPY docker/login.ligo.org.cert.LIGOCA.pem /etc/shibboleth/login.ligo.org.cert.LIGOCA.pem
 COPY docker/inc-md-cert.pem /etc/shibboleth/inc-md-cert.pem
 COPY docker/check_shibboleth_status /usr/local/bin/check_shibboleth_status
@@ -60,15 +102,17 @@ ADD . /app/gracedb_project

 # install gracedb application itself
 WORKDIR /app/gracedb_project
-RUN bower install --allow-root
-RUN pip install --upgrade setuptools wheel && \
-    pip install -r requirements.txt
+RUN pip3 install --upgrade pip --break-system-packages
+RUN pip3 install -r requirements.txt --break-system-packages
+
+# install supervisor from pip
+RUN pip3 install supervisor --break-system-packages

 # Give pip-installed packages priority over distribution packages
-ENV PYTHONPATH /usr/local/lib/python2.7/dist-packages:$PYTHONPATH
+ENV PYTHONPATH /usr/local/lib/python3.11/dist-packages:$PYTHONPATH
 ENV ENABLE_SHIBD false
 ENV ENABLE_OVERSEER true
-ENV VIRTUAL_ENV dummy
+ENV VIRTUAL_ENV /dummy/

 # Expose port and run Gunicorn
 EXPOSE 8000
@@ -89,20 +133,31 @@ RUN DJANGO_SETTINGS_MODULE=${SETTINGS_MODULE} \
    DJANGO_SECRET_KEY=fake_key \
    DJANGO_PRIMARY_FQDN=fake_fqdn \
    DJANGO_ALERT_EMAIL_FROM=fake_email \
+    EGAD_URL=fake_url \
+    EGAD_API_KEY=fake_key \
    LVALERT_USER=fake_user \
    LVALERT_PASSWORD=fake_password \
    LVALERT_SERVER=fake_server \
    LVALERT_OVERSEER_PORT=2 \
+    IGWN_ALERT_USER=fake_user \
+    IGWN_ALERT_PASSWORD=fake_password \
+    IGWN_ALERT_SERVER=fake_server \
+    IGWN_ALERT_OVERSEER_PORT=2 \
+    IGWN_ALERT_GROUP=fake_group \
    DJANGO_TWILIO_ACCOUNT_SID=fake_sid \
    DJANGO_TWILIO_AUTH_TOKEN=fake_token \
+    DJANGO_AWS_ELASTICACHE_ADDR=fake_address:11211 \
    AWS_SES_ACCESS_KEY_ID=fake_aws_id \
    AWS_SES_SECRET_ACCESS_KEY=fake_aws_key \
-    python manage.py collectstatic --noinput
+    python3 manage.py collectstatic --noinput

 RUN rm -rf /app/logs/* /app/project_data/*

 RUN useradd -M -u 50001 -g www-data -s /bin/false gracedb

+#RUN groupadd -r xray
+#RUN useradd -M -u 50002 -g xray -s /bin/false xray
+
 # set secure file/directory permissions. In particular, ADD command at
 # beginning of recipe inherits umask of user running the build
 RUN chmod 0755 /usr/local/bin/entrypoint && \
@@ -112,5 +167,19 @@ RUN chmod 0755 /usr/local/bin/entrypoint && \
    find /app/gracedb_project -type d -exec chmod 0755 {} + && \
    find /app/gracedb_project -type f -exec chmod 0644 {} +

+# create and set scitoken key cache directory
+RUN mkdir /app/scitokens_cache && \
+    chown gracedb:www-data /app/scitokens_cache && \
+    chmod 0750 /app/scitokens_cache
+ENV XDG_CACHE_HOME /app/scitokens_cache
+
+# patch voeventparse for python3.10+:
+RUN sed -i 's/collections.Iterable/collections.abc.Iterable/g' /usr/local/lib/python3.11/dist-packages/voeventparse/voevent.py
+
+# Remove packages that expose security vulnerabilities and close out.
+# Edit: zlib1g* can't be removed because of a PrePend error
+RUN apt-get --assume-yes --purge autoremove wget libaom3 node-ip
+RUN apt-get clean
+
 ENTRYPOINT [ "/usr/local/bin/entrypoint" ]
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
+CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
--- a/bower.json
+++ b/bower.json
 {
  "name": "gracedb",
  "dependencies": {
-    "dgrid": "0.4.0",
    "dijit": "1.10.4",
-    "dojox": "1.10.4",
-    "jquery": "3.2.1",
-    "jquery-ui": "1.12.1",
-    "moment-timezone": "0.5.0",
-    "moment": "2.11.1"
+    "dojox": "1.10.4"
  }
 }
--- a/config/gunicorn_config.py
+++ b/config/gunicorn_config.py
@@ -6,6 +6,14 @@ from os.path import abspath, dirname, join
 import sys
 import multiprocessing

+# Useful function for getting environment variables
+def get_from_env(envvar, default_value=None, fail_if_not_found=True):
+    value = os.environ.get(envvar, default_value)
+    if (value == default_value and fail_if_not_found):
+        raise ImproperlyConfigured(
+            'Could not get environment variable {0}'.format(envvar))
+    return value
+
 # Parameters
 GUNICORN_PORT = 8080
 LOG_DIR = abspath(join(dirname(__file__), "..", "..", "logs"))
@@ -14,18 +22,100 @@ LOG_DIR = abspath(join(dirname(__file__), "..", "..", "logs"))
 # Bind to localhost on specified port
 bind = "127.0.0.1:{port}".format(port=GUNICORN_PORT)

-# Number of workers = 2*CPU + 1 (recommendation from Gunicorn documentation)
-workers = multiprocessing.cpu_count()*2 + 1
+# Number of workers -----------------------------------------------------------
+# 2*CPU + 1 (recommendation from Gunicorn documentation)
+# bumped to 4*CPU + 1 after testing. Maybe increase this number in the cloud
+# deployment? 
+
+workers  = int(get_from_env('GUNICORN_WORKERS',
+                   default_value=multiprocessing.cpu_count()*3 + 1,
+                   fail_if_not_found=False))
+
+# NOTE: it was found in extensive testing that threads > 1 are prone
+# to connection lockups. Leave this at 1 for safety until there are 
+# fixes in gunicorn.
+
+# Why not sync? The sync worker is prone to timeout for long requests,
+# like big queries. But gthread sends a heartbeat back to the main worker
+# to keep it alive. We could just set the timeout to a really large number
+# which would keep the long requests stable, but if there is a stuck worker, 
+# then they would be subject to that really long timeout. It's a tradeoff. 
+
+# All this goes away with async workers, but as of 3.2, django's ORM does support
+# async, and testing failed pretty catastrophically and unreliably. 
+
+threads = int(get_from_env('GUNICORN_THREADS',
+                   default_value=1,
+                   fail_if_not_found=False))
+
+# Worker connections. Limit the number of connections between apache<-->gunicorn
+
+worker_connections = workers * threads

-# Worker class
-worker_class = 'sync'
+# Worker class ----------------------------------------------------------------
+# sync by default, generally safe and low-resource:
+# https://docs.gunicorn.org/en/stable/design.html#sync-workers

-# Max requests settings - a worker restarts after handling this many
-# requests. May be useful if we have memory leak problems.
+worker_class = get_from_env('GUNICORN_WORKER_CLASS',
+                   default_value='gthread',
+                   fail_if_not_found=False)
+
+# Timeout ---------------------------------------------------------------------
+# If not specified, the timeout default is 30 seconds:
+# https://gunicorn-docs.readthedocs.io/en/stable/settings.html#worker-processes
+
+timeout = get_from_env('GUNICORN_TIMEOUT',
+                   default_value=30,
+                   fail_if_not_found=False)
+
+graceful_timeout = timeout
+
+# max_requests settings -------------------------------------------------------
+# The maximum number of requests a worker will process before restarting.
+# May be useful if we have memory leak problems.
 # The jitter is drawn from a uniform distribution:
 # randint(0, max_requests_jitter)
-#max_requests = 0
-#max_requests_jitter = 0
+
+max_requests = get_from_env('GUNICORN_MAX_REQUESTS',
+                   default_value=5000,
+                   fail_if_not_found=False)
+
+max_requests_jitter = get_from_env('GUNICORN_MAX_REQUESTS_JITTER',
+                   default_value=250,
+                   fail_if_not_found=False)
+
+# keepalive -------------------------------------------------------------------
+# The number of seconds to wait for requests on a Keep-Alive connection.
+# Generally set in the 1-5 seconds range for servers with direct connection
+# to the client (e.g. when you don’t have separate load balancer).
+# When Gunicorn is deployed behind a load balancer, it often makes sense to set
+# this to a higher value.
+
+# NOTE: force gunicorn to close its connection to apache after each request. 
+# This has been the source of so many 502's. Basically in periods of high activity,
+# gunicorn would hold on to open sockets with apache, and just deadlock itself:
+# https://github.com/benoitc/gunicorn/issues/2917
+
+keepalive = get_from_env('GUNICORN_KEEPALIVE',
+                   default_value=0,
+                   fail_if_not_found=False)
+
+# preload_app -----------------------------------------------------------------
+# Load application code before the worker processes are forked.
+
+# By preloading an application you can save some RAM resources as well as speed
+# up server boot times. Although, if you defer application loading to each 
+# worker process, you can reload your application code easily by restarting 
+# workers.
+
+# If you aren't going to make use of on-the-fly reloading, consider preloading 
+# your application code to reduce its memory footprint. So, turn this on in
+# production. This is default set to False for development, but
+# **TURN THIS TO TRUE FOR AWS DEPLOYMENT **
+
+preload_app = get_from_env('GUNICORN_PRELOAD_APP',
+                   default_value=True,
+                   fail_if_not_found=False)

 # Logging ---------------------------------------------------------------------
 # Access log
@@ -35,11 +125,40 @@ access_log_format = ('GUNICORN | %(h)s %(l)s %(u)s %(t)s '

 # Error log
 errorlog = join(LOG_DIR, "gunicorn_error.log")
-loglevel = 'debug'
+
+# debug logging doesn't provide actual information. And this will 
+# eliminate the "Connection closed." messages while still giving info 
+# about worker restarts. 
+loglevel = 'info'
 capture_output = True

+# using /dev/shm/ instead of /tmp for the temporary worker directory. See:
+# https://pythonspeed.com/articles/gunicorn-in-docker/
+# “in AWS an EBS root instance volume may sometimes hang for half a minute 
+# and during this time Gunicorn workers may completely block.”
+worker_tmp_dir='/dev/shm'
+
+
 # Override logger class to modify error format
 from gunicorn.glogging import Logger
 class CustomLogger(Logger):
    error_fmt = 'GUNICORN | ' + Logger.error_fmt
 logger_class = CustomLogger
+
+def post_fork(server, worker):
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+def pre_fork(server, worker):
+    pass
+
+def pre_exec(server):
+    server.log.info("Forked child, re-executing.")
+
+def when_ready(server):
+    server.log.info("Server is ready. Spawning workers")
+
+def worker_int(worker):
+    worker.log.info("worker received INT or QUIT signal")
+
+def worker_abort(worker):
+    worker.log.info("worker received SIGABRT signal")
--- a/config/settings/base.py
+++ b/config/settings/base.py
-from cloghandler import ConcurrentRotatingFileHandler
+from concurrent_log_handler import ConcurrentRotatingFileHandler
 from datetime import datetime, timedelta
-import os, time, logging
+import os, time, logging, multiprocessing
 from os.path import abspath, dirname, join
 import socket
 from django.core.exceptions import ImproperlyConfigured
+from aws_xray_sdk.core.exceptions.exceptions import SegmentNotFoundException

 # Set up path to root of project
 BASE_DIR = abspath(join(dirname(__file__), "..", ".."))
@@ -21,21 +22,61 @@ def get_from_env(envvar, default_value=None, fail_if_not_found=True):
            'Could not get environment variable {0}'.format(envvar))
    return value

+def parse_envvar_bool(x):
+    return x.lower() in ['t', 'true', '1']
+
+# a sentry before_send function that filters aws SegmentNotFoundException's.
+# these exceptions are harmless and occur when performing management tasks
+# outside of the core gracedb app. but sentry picks it up and reports it as
+# an error which is ANNOYING.
+def before_send(event, hint):
+    if "exc_info" in hint:
+        exc_type, exc_value, tb = hint["exc_info"]
+        if isinstance(exc_value, (SegmentNotFoundException,)):
+            return None
+        return event
+
+# Maintenance mode
+MAINTENANCE_MODE = False
+MAINTENANCE_MODE_MESSAGE = None
+
+# Enable/Disable Information Banner:
+INFO_BANNER_ENABLED = False
+INFO_BANNER_MESSAGE = "TEST MESSAGE"
+
+# Beta reports page:
+BETA_REPORTS_LINK = False
+
 # Version ---------------------------------------------------------------------
-PROJECT_VERSION = '2.5.1'
+PROJECT_VERSION = '2.31.0'

 # Unauthenticated access ------------------------------------------------------
 # This variable should eventually control whether unauthenticated access is
 # allowed *ANYWHERE* on this service, except the home page, which is always
-# public. For now, it just controls the API.
-UNAUTHENTICATED_ACCESS = True
+# public. For now, it just controls the API and the public alerts page.
+# Update: make this updatable from the environment:
+UNAUTHENTICATED_ACCESS = parse_envvar_bool(
+    get_from_env('ENABLE_UNAUTHENTICATED_ACCESS',
+                 fail_if_not_found=False, default_value="true")
+)
+

 # Miscellaneous settings ------------------------------------------------------
 # Debug mode is off by default
 DEBUG = False
+# When debug mode is enabled, use custom reporter
+DEFAULT_EXCEPTION_REPORTER = 'core.utils.CustomExceptionReporter'

 # Number of results to show on latest page
-LATEST_RESULTS_NUMBER = 50
+LATEST_RESULTS_NUMBER = 25
+
+# Maximum number of log messages to display before throwing
+# a warning. NOTE: There should be a better way of doing this,
+# but just put in the hard cutoff for right now
+# Set to cover this:
+# https://gracedb.ligo.org/events/G184098
+TOO_MANY_LOG_ENTRIES = int(get_from_env('DJANGO_TOO_MANY_LOG_ENTRIES',
+    fail_if_not_found=False, default_value=2000))

 # Path to root URLconf
 ROOT_URLCONF = '{module}.urls'.format(module=os.path.basename(CONFIG_ROOT))
@@ -47,8 +88,9 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 # MANAGERS defines who gets broken link notifications when
 # BrokenLinkEmailsMiddleware is enabled
 ADMINS = [
-    ("Tanner Prestegard", "tanner.prestegard@ligo.org"),
    ("Alexander Pace", "alexander.pace@ligo.org"),
+    ("Duncan Meacher", "duncan.meacher@ligo.org"),
+    ("Daniel Wysocki", "daniel.wysocki@ligo.org"),
 ]
 MANAGERS = ADMINS

@@ -113,6 +155,24 @@ LOGOUT_REDIRECT_URL = 'home'
 SEND_XMPP_ALERTS = False
 SEND_PHONE_ALERTS = False
 SEND_EMAIL_ALERTS = False
+SEND_MATTERMOST_ALERTS = False
+
+# igwn-alert group settings. the default development group is 'lvalert-dev'
+# for the container deployments, the variable will be overwriten by the 
+# IGWN_ALERT_GROUP environment variable. 
+DEFAULT_IGWN_ALERT_GROUP = 'lvalert-dev'
+
+# enable/disable sending alerts to topics that have the search tag
+# for g/e-events. default to false, so only send to {group}_{pipeline}
+SEND_TO_SEARCH_TOPICS = parse_envvar_bool(
+    get_from_env('IGWN_ALERT_SEARCH_TOPICS',
+                 fail_if_not_found=False, default_value="false")
+)
+
+# overseer timeout:
+OVERSEER_TIMEOUT = float(get_from_env('IGWN_ALERT_OVERSEER_TIMEOUT',
+    fail_if_not_found=False, default_value=0.1))
+
 # Use LVAlert Overseer?
 USE_LVALERT_OVERSEER = True
 # For each LVAlert server, a separate instance of LVAlert Overseer
@@ -121,16 +181,17 @@ USE_LVALERT_OVERSEER = True
 #   listen_port: port which that instance of overseer is listening on
 LVALERT_OVERSEER_INSTANCES = [
    {
-        "lvalert_server": "lvalert-test.cgca.uwm.edu",
-        "listen_port": 8001,
+        "lvalert_server": "kafka://kafka.scimma.org/",
+        "listen_port": 8002,
+        "igwn_alert_group": DEFAULT_IGWN_ALERT_GROUP,
    },
 ]

 # Access and authorization ----------------------------------------------------
 # Some proper names related to authorization
-LVC_GROUP = 'Communities:LSCVirgoLIGOGroupMembers'
-LVEM_GROUP = 'gw-astronomy:LV-EM'
-LVEM_OBSERVERS_GROUP = 'gw-astronomy:LV-EM:Observers'
+LVC_GROUP = 'internal_users'
+LVEM_GROUP = 'lvem_users'
+LVEM_OBSERVERS_GROUP = 'lvem_observers'
 PUBLIC_GROUP = 'public_users'
 PRIORITY_USERS_GROUP = 'priority_users'

@@ -144,21 +205,13 @@ ACCESS_MANAGERS_GROUP = 'access_managers'
 EM_ADVOCATE_GROUP = 'em_advocates'
 # Superevent managers
 SUPEREVENT_MANAGERS_GROUP = 'superevent_managers'
+# RRT group name:
+RRT_MEMBERS_GROUP = 'rrt_members'

 # Analysis groups
 # Analysis group name for non-GW events
 EXTERNAL_ANALYSIS_GROUP = 'External'

-# Groups directly managed by GraceDB admins
-ADMIN_MANAGED_GROUPS = [EM_ADVOCATE_GROUP, EXEC_GROUP, PUBLIC_GROUP,
-    SUPEREVENT_MANAGERS_GROUP, ACCESS_MANAGERS_GROUP, 'raven_users']
-# NOTE: soon, only superevent_managers will be fully managed by us. The other
-# groups will be managed by the LIGO auth infrastructure for human users. But
-# we will still have to still manage robot account membership in these groups:
-# access_managers, executives, and em_advocates.
-# NOTE: public_users is also managed by us and has only one member
-# (guardian.AnonymousUser)
-
 # Tag to apply to log messages to allow EM partners to view
 EXTERNAL_ACCESS_TAGNAME = 'lvem'
 PUBLIC_ACCESS_TAGNAME = 'public'
@@ -200,17 +253,30 @@ COINC_PIPELINES = [
                    'spiir',
                    'MBTAOnline',
                    'pycbc',
+                    'MBTA',
+                    'PyGRB',
                   ]
 GRB_PIPELINES = [
                    'Fermi',
                    'Swift',
+                    'INTEGRAL',
+                    'AGILE',
+                    'CHIME',
+                    'SVOM',
                ]

-# SkyAlert stuff - used for VOEvents (?) --------------------------------------
-IVORN_PREFIX = "ivo://gwnet/LVC#"
-SKYALERT_ROLE          = "test"
-SKYALERT_DESCRIPTION   = "Report of a candidate gravitational wave event"
-SKYALERT_SUBMITTERS = ['Patrick Brady', 'Brian Moe']
+# List of pipelines that have been depreciated:
+DEPRECIATED_PIPELINES = [
+                          'X',
+                          'Q',
+                          'Omega',
+                        ]
+
+UNAPPROVED_PIPELINES = []
+
+# VOEvent stream --------------------------------------------------------------
+VOEVENT_STREAM = 'gwnet/LVC'
+

 # Stuff related to report/plot generation -------------------------------------

@@ -229,7 +295,7 @@ REPORT_INFO_URL_PREFIX = "/report_info/"
 REPORT_IFAR_IMAGE_DIR = PROJECT_DATA_DIR

 # Stuff for the new rates plot
-BINNED_COUNT_PIPELINES = ['gstlal', 'MBTAOnline', 'CWB', 'oLIB', 'spiir']
+BINNED_COUNT_PIPELINES = ['gstlal', 'MBTAOnline', 'MBTA', 'CWB', 'oLIB', 'spiir']
 BINNED_COUNT_FILE = join(PROJECT_DATA_DIR, "binned_counts.json")

 # Defaults for RSS feed
@@ -306,6 +372,11 @@ X509_INFOS_HEADER = 'HTTP_X_FORWARDED_TLS_CLIENT_CERT_INFOS'
 # Path to CA store for X509 certificate verification
 CAPATH = '/etc/grid-security/certificates'

+# SciTokens claims settings
+SCITOKEN_ISSUER = ['https://cilogon.org/igwn', 'https://test.cilogon.org/igwn', 'https://osdf.igwn.org/cit']
+SCITOKEN_AUDIENCE = ["ANY"]
+SCITOKEN_SCOPE = "gracedb.read"
+
 # List of authentication backends to use when attempting to authenticate
 # a user.  Will be used in this order.  Authentication for the API is
 # handled by the REST_FRAMEWORK dictionary.
@@ -317,6 +388,7 @@ AUTHENTICATION_BACKENDS = [

 # List of middleware classes to use.
 MIDDLEWARE = [
+    'core.middleware.maintenance.MaintenanceModeMiddleware',
    'events.middleware.PerformanceMiddleware',
    'core.middleware.accept.AcceptMiddleware',
    'core.middleware.api.ClientVersionMiddleware',
@@ -324,6 +396,7 @@ MIDDLEWARE = [
    'django.middleware.common.CommonMiddleware',
    'core.middleware.proxy.XForwardedForMiddleware',
    'user_sessions.middleware.SessionMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'ligoauth.middleware.ShibbolethWebAuthMiddleware',
@@ -347,23 +420,28 @@ INSTALLED_APPS = [
    'django.contrib.staticfiles',
    'django.contrib.messages',
    'alerts',
+    'annotations',
    'api',
    'core',
    'events',
    'ligoauth',
    'search',
    'superevents',
+    'gwtc',
    'rest_framework',
    'guardian',
    'django_twilio',
    'django_extensions',
    'django.contrib.sessions',
+    'computedfields',
+    'django_postgres_vacuum',
+    'django_q',
 ]

 # Aliases for django-extensions shell_plus
 SHELL_PLUS_MODEL_ALIASES = {
    # Two 'Group' models - auth.Group and gracedb.Group
-    'auth': {'Group': 'AuthGroup'},
+    'auth': {'Group': 'DjangoGroup'},
    # Superevents models which have the same name as
    # models in the events app
    'superevents': {
@@ -392,11 +470,12 @@ REST_FRAMEWORK = {
    ),
    'DEFAULT_THROTTLE_RATES': {
        'anon_burst': '300/minute',
-        'event_creation': '10/second',
-        'annotation'    : '10/second',
+        'event_creation': '50/second',
+        'annotation'    : '50/second',
    },
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'api.backends.GraceDbAuthenticatedAuthentication',
+        'api.backends.GraceDbSciTokenAuthentication',
        'api.backends.GraceDbX509Authentication',
        'api.backends.GraceDbBasicAuthentication',
    ),
@@ -412,9 +491,6 @@ if UNAUTHENTICATED_ACCESS is True:
    REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'] = \
        ('rest_framework.permissions.IsAuthenticatedOrReadOnly',)

-# Location of packages installed by bower
-BOWER_DIR = join(BASE_DIR, "..", "bower_components")
-
 # Location of static components, CSS, JS, etc.
 STATIC_ROOT = join(BASE_DIR, "static_root")
 STATIC_URL = "/static/"
@@ -425,7 +501,6 @@ STATICFILES_FINDERS = [
 ]
 STATICFILES_DIRS = [
    join(PROJECT_ROOT, "static"),
-    BOWER_DIR,
 ]

 # Added in order to perform data migrations on Django apps
@@ -615,3 +690,101 @@ LOGGING = {
        },
   },
 }
+
+# Turn off debug/error emails when in maintenance mode. 
+if MAINTENANCE_MODE:
+    LOGGING['loggers']['django.request']['handlers'].remove('mail_admins')
+
+# Turn off logging emails of django requests:
+# FIXME: figure out more reliable logging solution
+LOGGING['loggers']['django.request']['handlers'].remove('mail_admins')
+
+# Define some words for the instance stub:
+ENABLED = {True: "enabled", False: "disabled"}
+
+# Upgrading to django 3.2 produces warning: "Auto-created primary key used 
+# when not defining a primary key type, by default 'django.db.models.AutoField'.
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
+# Define window for neighbouring s events of a given g event.
+EVENT_SUPEREVENT_WINDOW_BEFORE = 100
+EVENT_SUPEREVENT_WINDOW_AFTER = 100
+
+# Define which observation periods to show on the public events page:
+# TODO: Group O4b and O4a under O4, once implemented.
+PUBLIC_PAGE_RUNS = ['O4', 'O4c', 'O4b', 'O4a', 'ER16', 'ER15', 'O3']
+
+# Define how long to cache the public page:
+PUBLIC_PAGE_CACHING = int(get_from_env('DJANGO_PUBLIC_PAGE_CACHING',
+    fail_if_not_found=False, default_value=300))
+
+# Define the number of results per page on the public page:
+PUBLIC_PAGE_RESULTS = int(get_from_env('DJANGO_PUBLIC_PAGE_RESULTS',
+    fail_if_not_found=False, default_value=15))
+
+# Define DATA_UPLOAD_MAX_MEMORY_SIZE for larger uploads:
+DATA_UPLOAD_MAX_MEMORY_SIZE = int(get_from_env('DJANGO_DATA_UPLOAD_MAX_MEMORY_SIZE',
+    fail_if_not_found=False, default_value=20*1024*1024))
+
+# Choose whether to use to Julian or Civil definition of year 
+# when displaying far's in /year:
+DISPLAY_CIVIL_YEAR_FAR = parse_envvar_bool(
+    get_from_env('DJANGO_DISPLAY_CIVIL_YEAR_FAR',
+                 fail_if_not_found=False, default_value="false")
+)
+
+if DISPLAY_CIVIL_YEAR_FAR:
+    DAYS_PER_YEAR = 365.0
+else:
+    DAYS_PER_YEAR = 365.25
+
+# Put in some setting for the redis queue backend
+ENABLE_REDIS_QUEUE = parse_envvar_bool(
+    get_from_env('DJANGO_ENABLE_REDIS_QUEUE',
+                 fail_if_not_found=False, default_value="false")
+)
+
+REDIS_QUEUE_ADDRESS = get_from_env('DJANGO_REDIS_QUEUE_ADDRESS',
+                 fail_if_not_found=False, default_value="127.0.0.1")
+
+REDIS_QUEUE_PORT = int(get_from_env('DJANGO_REDIS_QUEUE_PORT',
+                 fail_if_not_found=False, default_value="6379")
+)
+
+REDIS_QUEUE_DATABASE = int(get_from_env('DJANGO_REDIS_QUEUE_DATABASE',
+                 fail_if_not_found=False, default_value=0)
+)
+
+REDIS_QUEUE_WORKERS = int(get_from_env('DJANGO_REDIS_QUEUE_WORKERS',
+                   default_value=multiprocessing.cpu_count(),
+                   fail_if_not_found=False))
+
+REDIS_QUEUE_RETRY = int(get_from_env('DJANGO_REDIS_QUEUE_RETRY',
+                   default_value=40,
+                   fail_if_not_found=False))
+
+REDIS_QUEUE_TIMEOUT = int(get_from_env('DJANGO_REDIS_QUEUE_TIMEOUT',
+                   default_value=30,
+                   fail_if_not_found=False))
+
+REDIS_QUEUE_RECYCLE = int(get_from_env('DJANGO_REDIS_QUEUE_RECYCLE',
+                   default_value=500,
+                   fail_if_not_found=False))
+
+ENABLE_REDIS_CLUSTERED = parse_envvar_bool(
+    get_from_env('DJANGO_ENABLE_REDIS_CLUSTERED',
+                 fail_if_not_found=False, default_value="false")
+)
+
+# Define some defaults for the q-cluster parameters:
+Q_CLUSTER_NAME = 'gracedb-async-queue'
+Q_CLUSTER_LABEL = 'gracedb q cluster'
+
+if not ENABLE_REDIS_CLUSTERED:
+    Q_CLUSTER_NAME+=f'-{INTERNAL_HOSTNAME}'
+    Q_CLUSTER_LABEL+=f', {INTERNAL_HOSTNAME}'
+
+# Define MAX_DATATABLES_RESULTS to limit memory usage for web queries:
+MAX_DATATABLES_RESULTS = int(get_from_env('DJANGO_MAX_DATATABLES_RESULTS',
+    fail_if_not_found=False, default_value=1000))
+
--- a/config/settings/container/base.py
+++ b/config/settings/container/base.py
@@ -31,26 +31,87 @@ if SERVER_FQDN is None:
    raise ImproperlyConfigured('Could not get FQDN from envvars.')
 LIGO_FQDN = SERVER_FQDN

-# Get LVAlert server
-lvalert_server = os.environ.get('LVALERT_SERVER', None)
-if lvalert_server is None:
-    raise ImproperlyConfigured('Could not get LVAlert server from envvars.')
-
-# Get LVAlert Overseer listen port
-lvalert_overseer_port = os.environ.get('LVALERT_OVERSEER_PORT', None)
-if lvalert_overseer_port is None:
-    raise ImproperlyConfigured('Could not get LVAlert overseer port '
+
+## EGAD (External GraceDB Alert Dispatcher) configuration
+ENABLE_EGAD_EMAIL = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_EMAIL',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_KAFKA = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_KAFKA',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_MATTERMOST = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_MATTERMOST',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_PHONE = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_PHONE',
+                 fail_if_not_found=False, default_value="false")
+)
+
+ENABLE_EGAD = (
+    ENABLE_EGAD_EMAIL or ENABLE_EGAD_KAFKA
+    or ENABLE_EGAD_MATTERMOST or ENABLE_EGAD_PHONE
+)
+
+EGAD_URL = get_from_env('EGAD_URL',
+                        fail_if_not_found=ENABLE_EGAD, default_value=None)
+EGAD_API_KEY = get_from_env('EGAD_API_KEY',
+                   fail_if_not_found=ENABLE_EGAD, default_value=None)
+
+# Turn LVAlert on/off from the environment. Adding this
+# to turn lvalerts on/off from docker compose/update instead
+# of having to rebuild containers. If the environment variable
+# isn't set, then revert to the hardwired behavior:
+xmpp_env_var = get_from_env('SEND_LVALERT_XMPP_ALERTS',
+                   default_value=SEND_XMPP_ALERTS,
+                   fail_if_not_found=False)
+# Fix for other boolean values:
+if (isinstance(xmpp_env_var, str) and
+    xmpp_env_var.lower() in ['true','t','1']):
+    SEND_XMPP_ALERTS=True
+elif (isinstance(xmpp_env_var, str) and
+    xmpp_env_var.lower() in ['false','f','0']):
+    SEND_XMPP_ALERTS=False
+else:
+    SEND_XMPP_ALERTS = True
+
+# Get igwn_alert_overseer status:
+igwn_alert_on  = get_from_env(
+    'ENABLE_IGWN_OVERSEER',
+    default_value=False,
+    fail_if_not_found=False
+)
+if (isinstance(igwn_alert_on, str) and
+    igwn_alert_on.lower() in ['true', 't', '1']):
+        igwn_alert_overseer_on  = True
+else:
+    igwn_alert_overseer_on  = False
+
+# Get igwn-alert server
+igwn_alert_server = os.environ.get('IGWN_ALERT_SERVER', None)
+if igwn_alert_server is None:
+    raise ImproperlyConfigured('Could not get igwn-alert server from envvars.')
+
+# Get igwn-alert Overseer listen port
+igwn_alert_overseer_port = os.environ.get('IGWN_ALERT_OVERSEER_PORT', None)
+if igwn_alert_overseer_port is None:
+    raise ImproperlyConfigured('Could not get igwn-alert overseer port '
        'from envvars.')

-# Get LVAlert username
-lvalert_user = os.environ.get('LVALERT_USER', None)
-if lvalert_user is None:
-    raise ImproperlyConfigured('Could not get LVAlert username from envvars.')
+# Get igwn-alert group from envirnment:
+igwn_alert_group = os.environ.get('IGWN_ALERT_GROUP', DEFAULT_IGWN_ALERT_GROUP)

-# Get LVAlert password
-lvalert_password = os.environ.get('LVALERT_PASSWORD', None)
-if lvalert_password is None:
-    raise ImproperlyConfigured('Could not get LVAlert password from envvars.')
+# Get igwn-alert username
+igwn_alert_user = os.environ.get('IGWN_ALERT_USER', None)
+if igwn_alert_user is None:
+    raise ImproperlyConfigured('Could not get igwn-alert username from envvars.')
+
+# Get igwn-alert password
+igwn_alert_password = os.environ.get('IGWN_ALERT_PASSWORD', None)
+if igwn_alert_password is None:
+    raise ImproperlyConfigured('Could not get igwn-alert password from envvars.')

 # Get Twilio account information from environment
 TWILIO_ACCOUNT_SID = os.environ.get('DJANGO_TWILIO_ACCOUNT_SID', None)
@@ -61,6 +122,56 @@ TWILIO_AUTH_TOKEN = os.environ.get('DJANGO_TWILIO_AUTH_TOKEN', None)
 if TWILIO_AUTH_TOKEN is None:
    raise ImproperlyConfigured('Could not get Twilio auth token from envvars.')

+# Get maintenance mode settings from environment
+maintenance_mode = get_from_env(
+    'DJANGO_MAINTENANCE_MODE_ACTIVE',
+    default_value=False,
+    fail_if_not_found=False
+)
+
+# DB "cool-down" factor for when a db conflict is detected. This
+# factor scales a random number of seconds between zero and one.
+DB_SLEEP_FACTOR = get_from_env(
+        'DJANGO_DB_SLEEP_FACTOR',
+        default_value=1.0,
+        fail_if_not_found=False
+)
+# Fix the factor (str to float)
+try:
+    DB_SLEEP_FACTOR = float(DB_SLEEP_FACTOR)
+except:
+    DB_SLEEP_FACTOR = 1.0
+
+if (isinstance(maintenance_mode, str) and
+    maintenance_mode.lower() in ['true', 't', '1']):
+    MAINTENANCE_MODE = True
+MAINTENANCE_MODE_MESSAGE = \
+    get_from_env('DJANGO_MAINTENANCE_MODE_MESSAGE', fail_if_not_found=False)
+
+# Get info banner settings from environment
+info_banner_enabled = get_from_env(
+    'DJANGO_INFO_BANNER_ENABLED',
+    default_value=False,
+    fail_if_not_found=False
+    )
+# fix for other booleans:
+if (isinstance(info_banner_enabled, str) and
+    info_banner_enabled.lower() in ['true','t','1']):
+    INFO_BANNER_ENABLED = True
+INFO_BANNER_MESSAGE = \
+    get_from_env('DJANGO_INFO_BANNER_MESSAGE', fail_if_not_found=False)
+
+# Get reports page boolean:
+beta_reports_link = get_from_env(
+    'DJANGO_BETA_REPORTS_LINK',
+    default_value=False,
+    fail_if_not_found=False
+    )
+# fix for other booleans:
+if (isinstance(beta_reports_link, str) and
+    beta_reports_link.lower() in ['true','t','1']):
+    BETA_REPORTS_LINK = True
+
 # Get email settings from environment
 EMAIL_BACKEND = 'django_ses.SESBackend'
 AWS_SES_ACCESS_KEY_ID = get_from_env('AWS_SES_ACCESS_KEY_ID')
@@ -72,6 +183,102 @@ AWS_SES_REGION_ENDPOINT = get_from_env('AWS_SES_REGION_ENDPOINT',
 AWS_SES_AUTO_THROTTLE = 0.25
 ALERT_EMAIL_FROM = get_from_env('DJANGO_ALERT_EMAIL_FROM')

+# memcached settings. this variable should be set in the deployment to the 
+# same name as the service name in the docker deployment.
+
+DOCKER_MEMCACHED_ADDR = get_from_env('DJANGO_DOCKER_MEMCACHED_ADDR',
+                          default_value="memcached:11211",
+                          fail_if_not_found=False)
+DOCKER_MEMCACHED_SECONDS = get_from_env('DJANGO_DOCKER_MEMCACHED_SECONDS',
+                          default_value="15",
+                          fail_if_not_found=False)
+try:
+    CACHE_MIDDLEWARE_SECONDS = int(DOCKER_MEMCACHED_SECONDS)
+except:
+    CACHE_MIDDLEWARE_SECONDS = 15
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
+        'LOCATION': DOCKER_MEMCACHED_ADDR,
+        'OPTIONS': {
+            'ignore_exc': True,
+            }
+    },
+    # For API throttles
+    'throttles': {
+        'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
+        'LOCATION': 'api_throttle_cache', # Table name
+    },
+}
+
+if ENABLE_REDIS_QUEUE:
+    # For async alert follow-up:
+    CACHES.update({"async_followup": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": f"redis://{REDIS_QUEUE_ADDRESS}:{REDIS_QUEUE_PORT}/{REDIS_QUEUE_DATABASE}",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        }
+    }})
+
+
+    # Set queue backend for async django tasks:
+    # example django-redis connection
+    Q_CLUSTER = {
+        'name': Q_CLUSTER_NAME,
+        'label': Q_CLUSTER_LABEL,
+        'retry': REDIS_QUEUE_RETRY,
+        'timeout': REDIS_QUEUE_TIMEOUT,
+        'workers': REDIS_QUEUE_WORKERS,
+        'recycle': REDIS_QUEUE_RECYCLE,
+        'django_redis': 'async_followup'
+    }
+
+MIDDLEWARE = [
+    'django.middleware.cache.UpdateCacheMiddleware',
+    'core.middleware.maintenance.MaintenanceModeMiddleware',
+    'events.middleware.PerformanceMiddleware',
+    'core.middleware.accept.AcceptMiddleware',
+    'core.middleware.api.ClientVersionMiddleware',
+    'core.middleware.api.CliExceptionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'core.middleware.proxy.XForwardedForMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'user_sessions.middleware.SessionMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'ligoauth.middleware.ShibbolethWebAuthMiddleware',
+    'ligoauth.middleware.ControlRoomMiddleware',
+    'django.middleware.cache.FetchFromCacheMiddleware',
+]
+
+# Set up AWS X-ray patching if enabled
+ENABLE_AWS_XRAY = (
+    get_from_env("ENABLE_AWS_XRAY",
+                 default_value="false", fail_if_not_found=False).lower()
+    in ['true', 't', '1']
+)
+
+if ENABLE_AWS_XRAY:
+    # AWS X-ray middleware must be first in the list to measure timing
+    # accurately
+    MIDDLEWARE.insert(0, 'aws_xray_sdk.ext.django.middleware.XRayMiddleware')
+    # Include X-ray as an installed app in order to allow configuration beyond
+    # the default
+    INSTALLED_APPS.append('aws_xray_sdk.ext.django')
+    # Settings for AWS X-ray
+    XRAY_RECORDER = {
+        'AWS_XRAY_DAEMON_ADDRESS': '127.0.0.1:2000',
+        'AUTO_INSTRUMENT': True,
+        'AWS_XRAY_CONTEXT_MISSING': 'LOG_ERROR',
+        'PLUGINS': (),
+        'SAMPLING': True,
+        'SAMPLING_RULES': None,
+        'AWS_XRAY_TRACING_NAME': 'GraceDB',
+        'DYNAMIC_NAMING': None,
+        'STREAMING_THRESHOLD': None,
+    }

 # Priority server settings ----------------------------------------------------
 PRIORITY_SERVER = False
@@ -90,38 +297,75 @@ if PRIORITY_SERVER:


 # Database settings -----------------------------------------------------------
+
+# New postgresql database
+# Configured for the CI pipeline:
+# https://docs.gitlab.com/ee/ci/services/postgres.html
 DATABASES = {
    'default' : {
        'NAME': db_name,
-        'ENGINE': 'django.db.backends.mysql',
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'USER': db_user,
        'PASSWORD': db_password,
        'HOST': os.environ.get('DJANGO_DB_HOST', ''),
        'PORT': os.environ.get('DJANGO_DB_PORT', ''),
-        'OPTIONS': {
-            'init_command': 'SET storage_engine=MyISAM',
-            # NOTE: for mysql>=5.7 this will need to be changed to
-            #'init_command': 'SET default_storage_engine=MyISAM',
+        'CONN_MAX_AGE': 3600,
+        'TEST' : {
+            'NAME': 'gracedb_test_db',
        },
-    }
+    },
 }

 # Main server "hostname" - a little hacky but OK
 SERVER_HOSTNAME = SERVER_FQDN.split('.')[0]

-# LVAlert Overseer settings - get from environment
-LVALERT_OVERSEER_INSTANCES = [
-    {
-        "lvalert_server": lvalert_server,
-        "listen_port": int(lvalert_overseer_port),
-        "username": lvalert_user,
-        "password": lvalert_password,
-    },
-]
+# igwn_alert  Overseer settings - get from environment
+LVALERT_OVERSEER_INSTANCES = []
+LVALERT_OVERSEER_INSTANCES.append(
+{
+    "lvalert_server": igwn_alert_server,
+    "listen_port": int(igwn_alert_overseer_port),
+    "igwn_alert_group": igwn_alert_group,
+    "username": igwn_alert_user,
+    "password": igwn_alert_password,
+}
+)
+
+# Pull in remaining (phone/email) alert variables from 
+# the environment. Default to false.
+SEND_PHONE_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_PHONE_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+SEND_EMAIL_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_EMAIL_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+SEND_MATTERMOST_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_MATTERMOST_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+INSTANCE_STUB = """
+<li>Phone alerts (calls/SMS) are {0}</li>
+<li>Email alerts are {1}</li>
+<li><span class="text-monospace">igwn-alert</span> messages to <span class="text-monospace">{2}</span> are {3}</li>
+"""
+
+INSTANCE_LIST = INSTANCE_STUB.format(ENABLED[SEND_PHONE_ALERTS],
+                                ENABLED[SEND_EMAIL_ALERTS],
+                                LVALERT_OVERSEER_INSTANCES[0]['lvalert_server'],
+                                ENABLED[SEND_XMPP_ALERTS])

 # Use full client certificate to authenticate
 REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] = (
    'api.backends.GraceDbAuthenticatedAuthentication',
+    'api.backends.GraceDbSciTokenAuthentication',
    'api.backends.GraceDbX509FullCertAuthentication',
    'api.backends.GraceDbBasicAuthentication',
 )
@@ -148,3 +392,10 @@ EMBB_IGNORE_ADDRESSES = ['Mailer-Daemon@{fqdn}'.format(fqdn=SERVER_FQDN)]
 for key in LOGGING['loggers']:
    LOGGING['loggers'][key]['handlers'] = ['console']
 LOGGING['loggers']['django.request']['handlers'].append('mail_admins')
+
+# Turn off debug/error emails when in maintenance mode.
+if MAINTENANCE_MODE:
+    LOGGING['loggers']['django.request']['handlers'].remove('mail_admins')
+
+# Set SciToken accepted audience to server FQDN
+SCITOKEN_AUDIENCE = ["https://" + SERVER_FQDN, "https://" + LIGO_FQDN]
--- a/config/settings/container/dev.py
+++ b/config/settings/container/dev.py
 # Settings for a test/dev GraceDB instance running in a container
 from .base import *

-CONFIG_NAME = "TEST"
+TIER = "dev"
+CONFIG_NAME = "DEV"

 # Debug settings
 DEBUG = True
@@ -28,6 +29,11 @@ INSTALLED_APPS += [
 # Add testserver to ALLOWED_HOSTS
 ALLOWED_HOSTS += ['testserver']

+# Enforce that phone and email alerts are off XXX: Set by deployment variables!
+#SEND_PHONE_ALERTS = False
+#SEND_EMAIL_ALERTS = False
+#SEND_MATTERMOST_ALERTS = True
+
 # Settings for django-silk profiler
 SILKY_AUTHENTICATION = True
 SILKY_AUTHORISATION = True
@@ -46,12 +52,6 @@ INTERNAL_IPS = [
    INTERNAL_IP_ADDRESS,
 ]

-# Adjust ADMINS for dev instances
-ADMINS = [
-    ("Tanner Prestegard", "tanner.prestegard@ligo.org"),
-]
-
-
 # Set up Sentry for error logging
 sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
 if sentry_dsn is not None:
@@ -61,10 +61,36 @@ if sentry_dsn is not None:
    import sentry_sdk
    from sentry_sdk.integrations.django import DjangoIntegration
    sentry_sdk.init(
-        environment='test',
+        environment='dev',
        dsn=sentry_dsn,
-        integrations=[DjangoIntegration()]
+        integrations=[DjangoIntegration()],
+        before_send=before_send,
    )

    # Turn off default admin error emails
    LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Development VM'
+
+# Add sub-bullet with igwn-alert group:
+group_sub_bullet = """<ul>
+<li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
+INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_TITLE = 'GraceDB Development Server'
+INSTANCE_INFO = """
+<h5>Development Instance</h5>
+<hr>
+<p>
+This GraceDB instance is designed for GraceDB maintainers to develop and
+test in the AWS cloud architecture. There is <b>no guarantee</b> that the
+behavior of this instance will mimic the production system at any time. 
+Events and associated data may change or be removed at any time. 
+</p>
+<ul>
+{}
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+</ul>
+""".format(INSTANCE_LIST)
--- a/config/settings/container/playground.py
+++ b/config/settings/container/playground.py
+# Settings for a playground GraceDB instance (for user testing) running
+# in a container on AWS. These settings inherent from base.py) 
+# and overrides or adds to them.
+from .base import *
+
+TIER = "playground"
+CONFIG_NAME = "USER TESTING"
+
+# Debug settings
+DEBUG = False
+
+# Override EMBB email address
+# TP (8 Aug 2017): not sure why?
+EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
+
+# Enforce that phone and email alerts are off XXX: Set by deployment variables!
+#SEND_PHONE_ALERTS = False
+#SEND_EMAIL_ALERTS = False
+
+# Enable Mattermost alerts
+SEND_MATTERMOST_ALERTS = True
+
+# Add testserver to ALLOWED_HOSTS
+ALLOWED_HOSTS += ['testserver']
+
+# Set up Sentry for error logging
+sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
+if sentry_dsn is not None:
+    USE_SENTRY = True
+
+    # Set up Sentry
+    import sentry_sdk
+    from sentry_sdk.integrations.django import DjangoIntegration
+    sentry_sdk.init(
+        environment='playground',
+        dsn=sentry_dsn,
+        integrations=[DjangoIntegration()],
+        before_send=before_send,
+    )
+
+    # Turn off default admin error emails
+    LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Playground'
+
+# Add sub-bullet with igwn-alert group:
+group_sub_bullet = """<ul>
+<li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
+INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_INFO = """
+<h5>Playground instance</h5>
+<hr>
+<p>
+This GraceDB instance is designed for users to develop and test their own
+applications. It mimics the production instance in all but the following ways:
+</p>
+<ul>
+{}
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>Events and associated data will <b>not</b> be preserved indefinitely.
+A nightly cron job removes events older than 21 days.</li>
+</ul>
+""".format(INSTANCE_LIST)
+
+# Safety check on debug mode for playground
+if (DEBUG == True):
+    raise RuntimeError("Turn off debug mode for playground")
--- a/config/settings/container/production.py
+++ b/config/settings/container/production.py
 # Settings for a production GraceDB instance running in a container
 from .base import *

+TIER = "production"
+
 DEBUG = False

-# Turn on alerts
-SEND_XMPP_ALERTS = True
-SEND_PHONE_ALERTS = True
-SEND_EMAIL_ALERTS = True
+# Turn on alerts: XXX: Set by deployment variables!
+#SEND_PHONE_ALERTS = True
+#SEND_EMAIL_ALERTS = True
+#SEND_MATTERMOST_ALERTS = True

 # TP, March 2019: for now, it looks infeasible to use multiple databases
 # since there are many operations which normal LVC users can do that
@@ -51,12 +53,38 @@ if sentry_dsn is not None:
    sentry_sdk.init(
        environment='production',
        dsn=sentry_dsn,
-        integrations=[DjangoIntegration()]
+        integrations=[DjangoIntegration()],
+        before_send=before_send,
    )

    # Turn off default admin error emails
    LOGGING['loggers']['django.request']['handlers'] = []

+# Home page stuff
+INSTANCE_TITLE = 'GraceDB'
+
+# Add sub-bullet with igwn-alert group:
+group_sub_bullet = """<ul>
+<li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
+INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_INFO = """
+<h5>GraceDB Notifications</h5>
+<hr>
+<p>
+GraceDB notifies registered users of Gravitational-Wave candidate detections 
+in real-time during LIGO/Virgo/KAGRA observation periods. Current notifications 
+mechanisms are:
+</p>
+<ul>
+{}
+</ul>
+""".format(INSTANCE_LIST)
+
 # Safety check on debug mode for production
 if (DEBUG == True):
    raise RuntimeError("Turn off debug mode for production")
+
+# Hardcode pipelines not approved for production:
+UNAPPROVED_PIPELINES += ['aframe', 'GWAK']
--- a/config/settings/container/test.py
+++ b/config/settings/container/test.py
+# Settings for a test/dev GraceDB instance running in a container
+from .base import *
+
+TIER = "test"
+CONFIG_NAME = "TEST"
+
+# Debug settings
+DEBUG = True
+
+# Override EMBB email address
+# TP (8 Aug 2017): not sure why?
+EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
+
+# Add middleware
+debug_middleware = 'debug_toolbar.middleware.DebugToolbarMiddleware'
+MIDDLEWARE += [
+    debug_middleware,
+    #'silk.middleware.SilkyMiddleware',
+    #'core.middleware.profiling.ProfileMiddleware',
+    #'core.middleware.admin.AdminsOnlyMiddleware',
+]
+
+# Add to installed apps
+INSTALLED_APPS += [
+    'debug_toolbar',
+    #'silk'
+]
+
+# Add testserver to ALLOWED_HOSTS
+ALLOWED_HOSTS += ['testserver']
+
+# Settings for django-silk profiler
+SILKY_AUTHENTICATION = True
+SILKY_AUTHORISATION = True
+if 'silk' in INSTALLED_APPS:
+    # Needed to prevent RequestDataTooBig for files > 2.5 MB
+    # when silk is being used. This setting is typically used to
+    # prevent DOS attacks, so should not be changed in production.
+    DATA_UPLOAD_MAX_MEMORY_SIZE = 20*(1024**2)
+
+# Tuple of IPs which are marked as internal, useful for debugging.
+# Tanner (5 Dec. 2017): DON'T CHANGE THIS! Django Debug Toolbar exposes
+# some headers which we want to keep hidden.  So to be safe, we only allow
+# it to be used through this server.  You need to configure a SOCKS proxy
+# on your local machine to use DJDT (see admin docs).
+INTERNAL_IPS = [
+    INTERNAL_IP_ADDRESS,
+]
+
+# Enforce that phone and email alerts are off XXX: Set by deployment variables!
+#SEND_PHONE_ALERTS = False
+#SEND_EMAIL_ALERTS = False
+#SEND_MATTERMOST_ALERTS = True
+
+# Set up Sentry for error logging
+sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
+if sentry_dsn is not None:
+    USE_SENTRY = True
+
+    # Set up Sentry
+    import sentry_sdk
+    from sentry_sdk.integrations.django import DjangoIntegration
+    sentry_sdk.init(
+        environment='test',
+        dsn=sentry_dsn,
+        integrations=[DjangoIntegration()],
+        before_send=before_send,
+    )
+
+    # Turn off default admin error emails
+    LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Testing Server'
+
+# Add sub-bullet with igwn-alert group:
+group_sub_bullet = """<ul>
+<li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
+INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_INFO = """
+<h5>Testing Instance</h5>
+<hr>
+<p>
+This GraceDB instance is designed for Quality Assurance (QA) testing and
+validation for GraceDB and electromagnetic follow-up (EMFollow) developers.
+Software should meet QA milestones on the test instance before being moved 
+to Playground or Production. Note, on this GraceDB instance:
+</p>
+<ul>
+{}
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+</ul>
+""".format(INSTANCE_LIST)
--- a/config/settings/vm/base.py
+++ b/config/settings/vm/base.py
@@ -10,13 +10,13 @@ import socket
 DATABASES = {
    'default' : {
        'NAME': 'gracedb',
-        'ENGINE': 'django.db.backends.mysql',
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'USER': 'gracedb',
        'PASSWORD': DB_PASSWORD,
-        'OPTIONS': {
-            'init_command': 'SET storage_engine=MyISAM',
-        },
-    }
+        'HOST':'127.0.0.1',
+        'PORT':'5432',
+        'CONN_MAX_AGE': 3600,
+    },
 }

 # Set up allowed hosts
@@ -38,3 +38,148 @@ EMBB_MAIL_ADDRESS = 'embb@{fqdn}.ligo.org'.format(fqdn=SERVER_FQDN)
 EMBB_SMTP_SERVER = 'localhost'
 EMBB_MAIL_ADMINS = [admin[1] for admin in ADMINS]
 EMBB_IGNORE_ADDRESSES = ['Mailer-Daemon@{fqdn}'.format(fqdn=SERVER_FQDN)]
+
+# Load modified caching middleware:
+# https://docs.djangoproject.com/en/2.2/ref/middleware/#middleware-ordering
+MIDDLEWARE = [
+    'django.middleware.cache.UpdateCacheMiddleware',
+    'django.middleware.gzip.GZipMiddleware',
+    'events.middleware.PerformanceMiddleware',
+    'core.middleware.accept.AcceptMiddleware',
+    'core.middleware.api.ClientVersionMiddleware',
+    'core.middleware.api.CliExceptionMiddleware',
+    'core.middleware.proxy.XForwardedForMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'user_sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'core.middleware.maintenance.MaintenanceModeMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'ligoauth.middleware.ShibbolethWebAuthMiddleware',
+    'ligoauth.middleware.ControlRoomMiddleware',
+    'django.middleware.cache.FetchFromCacheMiddleware',
+]
+
+# Set caches:
+CACHE_MIDDLEWARE_SECONDS = 5
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
+        'LOCATION': 'localhost:11211',
+        'TIMEOUT': 60,
+        'KEY_PREFIX': 'NULL',
+        'OPTIONS': {
+            'ignore_exc': True,
+            }
+    },
+    # For API throttles
+    'throttles': {
+        'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
+        'LOCATION': 'api_throttle_cache', # Table name
+    },    
+}
+
+# FIXME: hardwire this for now in the VMs for testing
+ENABLE_REDIS_QUEUE = True
+
+if ENABLE_REDIS_QUEUE:
+    # For async alert follow-up:
+    CACHES.update({"async_followup": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": f"redis://{REDIS_QUEUE_ADDRESS}:{REDIS_QUEUE_PORT}/{REDIS_QUEUE_DATABASE}",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        }
+    }})
+
+
+    # Set queue backend for async django tasks:
+    # example django-redis connection
+    Q_CLUSTER = {
+        'name': Q_CLUSTER_NAME,
+        'label': Q_CLUSTER_LABEL,
+        'retry': REDIS_QUEUE_RETRY,
+        'timeout': REDIS_QUEUE_TIMEOUT,
+        'workers': REDIS_QUEUE_WORKERS,
+        'recycle': REDIS_QUEUE_RECYCLE,
+        'django_redis': 'async_followup'
+    }
+
+# DB "cool-down" factor for when a db conflict is detected. This
+# factor scales a random number of seconds between zero and one.
+DB_SLEEP_FACTOR = get_from_env(
+        'DJANGO_DB_SLEEP_FACTOR',
+        default_value=1.0,
+        fail_if_not_found=False
+)
+# Fix the factor (str to float)
+try:
+    DB_SLEEP_FACTOR = float(DB_SLEEP_FACTOR)
+except:
+    DB_SLEEP_FACTOR = 1.0
+
+BETA_REPORTS_LINK = True
+
+## EGAD (External GraceDB Alert Dispatcher) configuration
+ENABLE_EGAD_EMAIL = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_EMAIL',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_KAFKA = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_KAFKA',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_MATTERMOST = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_MATTERMOST',
+                 fail_if_not_found=False, default_value="false")
+)
+ENABLE_EGAD_PHONE = parse_envvar_bool(
+    get_from_env('ENABLE_EGAD_PHONE',
+                 fail_if_not_found=False, default_value="false")
+)
+
+ENABLE_EGAD = (
+    ENABLE_EGAD_EMAIL or ENABLE_EGAD_KAFKA
+    or ENABLE_EGAD_MATTERMOST or ENABLE_EGAD_PHONE
+)
+
+
+# Pull in remaining (phone/email) alert variables from
+# the environment. Default to false.
+SEND_PHONE_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_PHONE_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+SEND_EMAIL_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_EMAIL_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+SEND_MATTERMOST_ALERTS = parse_envvar_bool(get_from_env(
+    'SEND_MATTERMOST_ALERTS',
+    default_value='False',
+    fail_if_not_found=False
+    ))
+
+INSTANCE_STUB = """
+<li>Phone alerts (calls/SMS) are {0}</li>
+<li>Email alerts are {1}</li>
+<li><span class="text-monospace">igwn-alert</span> messages to <span class="text-monospace">{2}</span> are {3}</li>
+"""
+
+INSTANCE_LIST = INSTANCE_STUB.format(ENABLED[SEND_PHONE_ALERTS],
+                                ENABLED[SEND_EMAIL_ALERTS],
+                                LVALERT_OVERSEER_INSTANCES[0]['lvalert_server'],
+                                ENABLED[SEND_XMPP_ALERTS])
+
+if (len(LVALERT_OVERSEER_INSTANCES) == 2):
+    IGWN_STUB = '<li><span class="text-monospace">igwn-alert</span> messages to <span class="text-monospace">{0}</span> are {1}</li>'
+    IGWN_LIST = IGWN_STUB.format(LVALERT_OVERSEER_INSTANCES[1]['lvalert_server'],
+                                ENABLED[SEND_XMPP_ALERTS])
+    INSTANCE_LIST = INSTANCE_LIST + IGWN_LIST
+
+# Set SciToken accepted audience to server FQDN
+SCITOKEN_AUDIENCE = ["https://" + SERVER_FQDN, "https://" + LIGO_FQDN]
--- a/config/settings/vm/dev.py
+++ b/config/settings/vm/dev.py
@@ -4,10 +4,13 @@
 import socket
 from .base import *

-CONFIG_NAME = "TEST"
+TIER = "dev"
+CONFIG_NAME = "DEV"

 # Debug settings
 DEBUG = True
+SEND_XMPP_ALERTS=True
+SEND_MATTERMOST_ALERTS=True

 # Override EMBB email address
 # TP (8 Aug 2017): not sure why?
@@ -25,7 +28,6 @@ MIDDLEWARE += [
 # Add to installed apps
 INSTALLED_APPS += [
    'debug_toolbar',
-    #'silk'
 ]

 # Add testserver to ALLOWED_HOSTS
@@ -48,8 +50,33 @@ if 'silk' in INSTALLED_APPS:
 INTERNAL_IPS = [
    INTERNAL_IP_ADDRESS,
 ]
+INSTANCE_TITLE = 'GraceDB Development VM'

-# Adjust ADMINS for dev instances
-ADMINS = [
-    ("Tanner Prestegard", "tanner.prestegard@ligo.org"),
-]
+# Add sub-bullet with igwn-alert group:
+if (len(LVALERT_OVERSEER_INSTANCES) == 2):
+    igwn_alert_group = os.environ.get('IGWN_ALERT_GROUP', 'lvalert-dev')
+    group_sub_bullet = """<ul>
+    <li> Messages are sent to group: <span class="text-monospace"> {0}  </span></li>
+    </ul>""".format(igwn_alert_group)
+    INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
+
+INSTANCE_INFO = """
+<h5>Development Instance</h5>
+<hr>
+<p>
+This GraceDB instance is designed for GraceDB maintainers to develop and
+test in the AWS cloud architecture. There is <b>no guarantee</b> that the
+behavior of this instance will mimic the production system at any time.
+Events and associated data may change or be removed at any time.
+</p>
+<ul>
+{}
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+</ul>
+""".format(INSTANCE_LIST)
+
+# Turn off public page caching for development and testing:
+PUBLIC_PAGE_CACHING = 0
+
+# Hardcode pipelines not approved for production (for vm testing)
+# UNAPPROVED_PIPELINES += ['aframe', 'GWAK']
--- a/config/settings/vm/playground.py
+++ b/config/settings/vm/playground.py
@@ -3,6 +3,7 @@
 # from base.py settings) and overrides or adds to them.
 from .base import *

+TIER = "playground"
 CONFIG_NAME = "USER TESTING"

 # Debug settings
@@ -15,6 +16,9 @@ EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
 # Turn on XMPP alerts
 SEND_XMPP_ALERTS = True

+# Turn on Mattermost alerts
+SEND_MATTERMOST_ALERTS = True
+
 # Enforce that phone and email alerts are off
 SEND_PHONE_ALERTS = False
 SEND_EMAIL_ALERTS = False
@@ -33,18 +37,17 @@ ALLOWED_HOSTS += ['testserver']
 # Home page stuff
 INSTANCE_TITLE = 'GraceDB Playground'
 INSTANCE_INFO = """
+<h3>Playground instance</h3>
 <p>
 This GraceDB instance is designed for users to develop and test their own
 applications. It mimics the production instance in all but the following ways:
 </p>
 <ul>
-<li>Phone and e-mail alerts are turned off</li>
-<li>Only LIGO logins are provided (no login via InCommon or Google)</li>
-<li>LVAlert messages are sent to lvalert-playground.cgca.uwm.edu</li>
+<li>Phone and e-mail alerts are turned off.</li>
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>LVAlert messages are sent to lvalert-playground.cgca.uwm.edu.</li>
 <li>Events and associated data will <b>not</b> be preserved indefinitely.
-A nightly cron job removes events older than 14 days.</li>
-<li><b>Note:</b> for O3 development, the above number has been updated to
-<b>112</b>.</li>
+A nightly cron job removes events older than 21 days.</li>
 </ul>
 """


--- a/config/settings/vm/production.py
+++ b/config/settings/vm/production.py
@@ -3,6 +3,8 @@
 # settings) and overrides or adds to them.
 from .base import *

+TIER = "production"
+
 DEBUG = False

 # LVAlert Overseer settings
@@ -17,6 +19,7 @@ LVALERT_OVERSEER_INSTANCES = [
 SEND_XMPP_ALERTS = True
 SEND_PHONE_ALERTS = True
 SEND_EMAIL_ALERTS = True
+SEND_MATTERMOST_ALERTS = True

 # Safety check on debug mode for production
 if (DEBUG == True):

--- a/config/urls.py
+++ b/config/urls.py
 from django.conf import settings
-from django.conf.urls import url, include
+
+from django.urls import re_path, include

 from django.contrib import admin
-from django.contrib.auth.views import logout
+from django.contrib.auth.views import LogoutView
 from django.views.generic import TemplateView

 # Import feeds
@@ -10,7 +11,9 @@ import core.views
 from events.feeds import EventFeed, feedview
 import events.reports
 import events.views
-from ligoauth.views import pre_login, post_login, shib_logout, manage_password
+from ligoauth.views import (
+    manage_password, ShibLoginView, ShibPostLoginView
+)
 import search.views

 # Django admin auto-discover
@@ -22,54 +25,54 @@ feeds = {


 urlpatterns = [
-    url(r'^$', events.views.index, name="home"),
-    url(r'^navbar_only$', TemplateView.as_view(
+    re_path(r'^$', events.views.index, name="home"),
+    re_path(r'^navbar_only$', TemplateView.as_view(
        template_name='navbar_only.html'), name="navbar-only"),
-    url(r'^SPInfo', TemplateView.as_view(template_name='gracedb/spinfo.html'),
+    re_path(r'^SPInfo', TemplateView.as_view(template_name='gracedb/spinfo.html'),
         name="spinfo"),
-    url(r'^SPPrivacy', TemplateView.as_view(
+    re_path(r'^SPPrivacy', TemplateView.as_view(
        template_name='gracedb/spprivacy.html'), name="spprivacy"),
-    url(r'^DiscoveryService', TemplateView.as_view(
+    re_path(r'^DiscoveryService', TemplateView.as_view(
        template_name='discovery.html'), name="discovery"),
-    url(r'^events/', include('events.urls')),
-    url(r'^superevents/', include('superevents.urls')),
-    url(r'^alerts/', include('alerts.urls')),
-    url(r'^feeds/(?P<url>.*)/$', EventFeed()),
-    url(r'^feeds/$', feedview, name="feeds"),
-
-    url(r'^performance/$', events.views.performance, name="performance"),
-    url(r'^reports/$', events.reports.histo, name="reports"),
-    url(r'^reports/cbc_report/(?P<format>(json|flex))?$',
-        events.reports.cbc_report, name="cbc_report"),
-    url(r'^latest/$', search.views.latest, name="latest"),
+    re_path(r'^events/', include('events.urls')),
+    re_path(r'^superevents/', include('superevents.urls')),
+    re_path(r'^alerts/', include('alerts.urls')),
+    re_path(r'^feeds/(?P<url>.*)/$', EventFeed()),
+    re_path(r'^feeds/$', feedview, name="feeds"),
+
+    re_path(r'^other/$', TemplateView.as_view(template_name='other.html'),
+        name='other'),
+    re_path(r'^performance/$', events.views.performance, name="performance"),
+    re_path(r'^reports/$', events.reports.reports_page_context, name="reports"),
+    re_path(r'^latest/$', search.views.latest, name="latest"),
    #(r'^reports/(?P<path>.+)$', 'django.views.static.serve',
    #        {'document_root': settings.LATENCY_REPORT_DEST_DIR}),
-    url(r'^search/$', search.views.search, name="mainsearch"),
+    re_path(r'^search/$', search.views.search, name="mainsearch"),

    # Authentication
-    url(r'^login/$', pre_login, name='login'),
-    url(r'^post-login/$', post_login, name='post-login'),
-    url(r'^logout/$', shib_logout, name='logout'),
+    re_path(r'^login/$', ShibLoginView.as_view(), name='login'),
+    re_path(r'^post-login/$', ShibPostLoginView.as_view(), name='post-login'),
+    re_path(r'^logout/$', LogoutView.as_view(), name='logout'),

    # Password management
-    url('^manage-password/$', manage_password, name='manage-password'),
+    re_path('^manage-password/$', manage_password, name='manage-password'),

    # API URLs
-    url(r'^api/', include('api.urls')),
+    re_path(r'^api/', include('api.urls')),
    # Legacy API URLs - must be maintained!
-    url(r'^apibasic/', include('api.urls', namespace='legacy_apibasic')),
-    url(r'^apiweb/', include('api.urls', namespace='legacy_apiweb')),
+    re_path(r'^apibasic/', include('api.urls', namespace='legacy_apibasic')),
+    re_path(r'^apiweb/', include('api.urls', namespace='legacy_apiweb')),

    # Heartbeat URL
-    url(r'^heartbeat/$', core.views.heartbeat, name='heartbeat'),
+    re_path(r'^heartbeat/$', core.views.heartbeat, name='heartbeat'),

    # Uncomment the admin/doc line below and add 'django.contrib.admindocs'
    # to INSTALLED_APPS to enable admin documentation:
    # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
-    url(r'^admin/', admin.site.urls),
+    re_path(r'^admin/', admin.site.urls),

    # Sessions
-    url(r'^', include('user_sessions.urls', 'user_sessions')),
+    re_path(r'^', include('user_sessions.urls', 'user_sessions')),

 ]

@@ -79,12 +82,12 @@ urlpatterns = [
 if ('silk' in settings.INSTALLED_APPS):
    # Add django-silk
    urlpatterns = [
-        url(r'^silk/', include('silk.urls', namespace='silk'))
+        re_path(r'^silk/', include('silk.urls', namespace='silk'))
    ] + urlpatterns

 # Add django-debug-toolbar
 if settings.DEBUG and 'debug_toolbar' in settings.INSTALLED_APPS:
    import debug_toolbar
    urlpatterns = [
-        url(r'^__debug__/', include(debug_toolbar.urls)),
+        re_path(r'^__debug__/', include(debug_toolbar.urls)),
    ] + urlpatterns
--- a/docker/apache-config
+++ b/docker/apache-config
@@ -23,6 +23,13 @@ ServerName ${DJANGO_PRIMARY_FQDN}
    Require all granted
  </Directory>

+  # Improve proxy behavior with gunicorn:
+  # https://serverfault.com/questions/206738/intermittent-error-when-using-mod-proxy-to-do-reverse-proxy-to-soap-service#comment1327184_209006
+  # https://github.com/benoitc/gunicorn/issues/207
+
+  SetEnv force-proxy-request-1.0 1
+  SetEnv proxy-nokeepalive 1
+
  ## Custom fragment
  # gUnicorn edits
  Alias /shibboleth-ds/idpselect_config.js /etc/shibboleth-ds/idpselect_config.js
@@ -40,7 +47,33 @@ ServerName ${DJANGO_PRIMARY_FQDN}
  ProxyPass "/static" "!"
  ProxyPass "/documentation" "!"
  ProxyPass "/admin_docs" "!"
-  ProxyPass "/" "http://localhost:8080/"
+  ProxyPass "/" "http://localhost:8080/" timeout=120
+  ProxyPassReverse "/" "http://localhost:8080/"
+
+  # This section is for apache2 timeout and keepalive tuning parameters. 
+  # https://ioflood.com/blog/2020/02/21/what-is-apache-keepalive-timeout-how-to-optimize-this-critical-setting/
+  
+  # KeepAlive will... keep a connection alive for subsequent requests. 
+  # Turn this on.
+  KeepAlive On
+
+  # The maximum number of requests served to a client before terminating the connection.
+  # This can be large, possibly safely unlimited. (0 = unlimited)
+  MaxKeepAliveRequests 0
+
+  # The number of seconds Apache will wait for a subsequent request before closing the 
+  # connection. Once a request has been received, the timeout value specified by the 
+  # Timeout directive applies. Setting KeepAliveTimeout to a high value may cause 
+  # performance problems in heavily loaded servers. The higher the timeout, the more 
+  # server processes will be kept occupied waiting on connections with idle clients
+  KeepAliveTimeout 5
+
+  # Amount of time the server will wait for certain events before failing a 
+  # request. The TimeOut directive defines the length of time Apache will wait for
+  # I/O (e.g., when reading data from the client, when writing data to the client, etc.) 
+  # Default: 300s. Try setting this lower, then do a test like a long query with the API
+  # and in the browser and see what happens. 
+  Timeout 60

  # Unset certain headers to help prevent spoofing
  RequestHeader unset REMOTE_USER
@@ -58,6 +91,9 @@ ServerName ${DJANGO_PRIMARY_FQDN}
  # Set X_FORWARDED_PROTO to https
  RequestHeader set X_FORWARDED_PROTO "https"

+  # Increase the max allowable header size:
+  LimitRequestFieldSize 16384
+
  # Set up mod_xsendfile for serving static event files as directed by Django
  XSendFile On
  XSendFilePath /app/db_data/
@@ -82,7 +118,7 @@ ServerName ${DJANGO_PRIMARY_FQDN}
    Require all granted
  </Directory>

-  Alias /robots.txt /home/gracedb/gracedb_project/static_root/robots.txt
+  Alias /robots.txt /app/gracedb_project/static_root/robots.txt

  <Location /Shibboleth.sso>
    SetHandler shib
@@ -119,7 +155,7 @@ ServerName ${DJANGO_PRIMARY_FQDN}
    AuthType Shibboleth
    ShibRequestSetting requireSession true
    ShibUseHeaders On
-    Require shib-user tanner.prestegard@LIGO.ORG alexander.pace@LIGO.ORG patrick.brady@LIGO.ORG thomas.downes@LIGO.ORG
+    Require shib-user duncan.meacher@ligo.org alexander.pace@ligo.org daniel.wysocki@ligo.org patrick.brady@ligo.org
  </Location>

 </VirtualHost>
--- a/docker/check_shibboleth_status
+++ b/docker/check_shibboleth_status
-#!/usr/bin/python
+#!/usr/bin/python3

 '''
 Pulls Shibboleth status.sso page, checks for:
@@ -8,8 +8,14 @@ Run ./check_shibboleth_status -h for help.
 '''

 # Imports
-import argparse, urllib2, sys
+import argparse
+import sys
 import xml.etree.ElementTree as ET
+try:
+    from urllib.request import urlopen
+    from urllib.error import URLError
+except ImportError:  # python < 3
+    from urllib2 import (urlopen, URLError)

 # Parameters - may need to be modified in the future
 # if Shibboleth status pages change or new metadata
@@ -48,12 +54,12 @@ metadata_feeds = args.feeds.split(",")
 # Get XML data from URL.
 host_url = host + "/" + urlpath
 try:
-    response = urllib2.urlopen(host_url, timeout=timeout)
-except urllib2.URLError:
-    print "Error opening Shibboleth status page (" + host_url + ")."
+    response = urlopen(host_url, timeout=timeout)
+except URLError:
+    print("Error opening Shibboleth status page (" + host_url + ").")
    sys.exit(2)
 except:
-    print "Unknown error opening Shibboleth status page (" + host_url + ")."
+    print("Unknown error opening Shibboleth status page (" + host_url + ").")
    sys.exit(3)

 # Convert from string to ElementTree
@@ -61,11 +67,11 @@ try:
    status_tree = ET.fromstring(response.read())
 except ET.ParseError:
    # Error parsing response.
-    print "Error parsing response from server - not in XML format."
+    print("Error parsing response from server - not in XML format.")
    sys.exit(2)
 except:
    # Error that is not ParseError.
-    print "Unknown error occurred when parsing response from server."
+    print("Unknown error occurred when parsing response from server.")
    sys.exit(3)
 response.close()

@@ -75,12 +81,12 @@ response.close()
 for tag in tags_to_check:
    status_tag = status_tree.find(tag)
    if (status_tag is None):
-        print "Error: tag \'" +  tag + "\' not found."
+        print("Error: tag \'" +  tag + "\' not found.")
        sys.exit(2)
    else:
        status_OK = status_tag.find('OK')
        if (status_OK is None):
-            print "Error: tag \'" + tag + "\' is not OK."
+            print("Error: tag \'" + tag + "\' is not OK.")
            sys.exit(2)

 # Check 2: make sure metadata feeds that we expect
@@ -90,12 +96,12 @@ srcs = [element.attrib['source'] for element in metaprov_tags]
 for feed in metadata_feeds:
    feed_found = [src.lower().find(feed) >= 0 for src in srcs]
    if (sum(feed_found) < 1):
-        print "MetadataProvider " + feed + " not found."
+        print("MetadataProvider " + feed + " not found.")
        sys.exit(2)
    elif (sum(feed_found) < 1):
-        print "MetadataProvider " + feed + "found in multiple elements."
+        print("MetadataProvider " + feed + "found in multiple elements.")
        sys.exit(2)

 # If we make it to this point, everything is OK.
-print "All MetadataProviders found. Status and SessionCache are OK."
+print("All MetadataProviders found. Status and SessionCache are OK.")
 sys.exit(0)
--- a/docker/cleanup
+++ b/docker/cleanup
 #!/bin/sh

-python /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap
-python /app/gracedb_project/manage.py remove_inactive_alerts
-python /app/gracedb_project/manage.py clearsessions
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap kagra
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap ligo
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap robots
+python3 /app/gracedb_project/manage.py update_catalog_managers_group
+python3 /app/gracedb_project/manage.py remove_inactive_alerts
+python3 /app/gracedb_project/manage.py clearsessions 2>&1 | grep -v segment
+python3 /app/gracedb_project/manage.py remove_old_mdc_public_perms
+PGPASSWORD=$DJANGO_DB_PASSWORD psql -h $DJANGO_DB_HOST -U $DJANGO_DB_USER -c "VACUUM VERBOSE ANALYZE;" $DJANGO_DB_NAME
--- a/docker/entrypoint
+++ b/docker/entrypoint
 #!/bin/bash

-export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python  -c 'import uuid; print(uuid.uuid4().hex)')
+export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python3  -c 'import uuid; print(uuid.uuid4().hex)')
+
+# Change the file permissions and ownership on /app/db_data:
+chown gracedb:www-data /app/db_data
+chmod 755 /app/db_data
+
+## PGA: 2019-10-15: use certs from secrets for Shibboleth SP
+SHIB_SP_CERT=/run/secrets/saml_certificate
+SHIB_SP_KEY=/run/secrets/saml_private_key
+if [[ -f $SHIB_SP_CERT && -f $SHIB_SP_KEY ]]
+then
+        echo "Using Shibboleth Cert from docker secrets over the image one"
+        cp -f $SHIB_SP_CERT /etc/shibboleth/sp-cert.pem
+        cp -f $SHIB_SP_KEY /etc/shibboleth/sp-key.pem
+        chown _shibd:_shibd /etc/shibboleth/sp-{cert,key}.pem
+        chmod 0600 /etc/shibboleth/sp-key.pem
+fi
+
+## PGA 2019-10-16: use secrets for sensitive environment variables
+LIST="aws_ses_access_key_id
+  aws_ses_secret_access_key
+  django_db_password
+  django_secret_key
+  django_twilio_account_sid
+  django_twilio_auth_token
+  lvalert_password
+  igwn_alert_password
+  gracedb_ldap_keytab
+  egad_url
+  egad_api_key
+  django_sentry_dsn"
+
+for SECRET in $LIST
+do
+        VARNAME=$( tr [:lower:] [:upper:] <<<$SECRET)
+        [  -f /run/secrets/$SECRET ] && export $VARNAME="$(< /run/secrets/$SECRET)"
+done
+
+# get x509 cert for ldap access from environment variable. 
+echo "${GRACEDB_LDAP_KEYTAB}" | base64 -d | install -m 0600 /dev/stdin keytab
+kinit ldap/gracedb.ligo.org@LIGO.ORG -k -t keytab
+
 exec "$@"
+
No results found