Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • alexander.pace/server
  • geoffrey.mo/gracedb-server
  • deep.chatterjee/gracedb-server
  • cody.messick/server
  • sushant.sharma-chaudhary/server
  • michael-coughlin/server
  • daniel.wysocki/gracedb-server
  • roberto.depietri/gracedb
  • philippe.grassia/gracedb
  • tri.nguyen/gracedb
  • jonah-kanner/gracedb
  • brandon.piotrzkowski/gracedb
  • joseph-areeda/gracedb
  • duncanmmacleod/gracedb
  • thomas.downes/gracedb
  • tanner.prestegard/gracedb
  • leo-singer/gracedb
  • computing/gracedb/server
18 results
Show changes
Commits on Source (1151)
1,051 additional commits have been omitted to prevent performance issues.
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 2307 additions and 245 deletions
.gitignore
View file @ 5797d777
......@@ -7,3 +7,6 @@ config/settings/local.py
docs/user_docs/build/*
docs/admin_docs/build/*
static_root/*
.pytest_cache
junit.xml
.coverage
.gitlab-ci.yml
View file @ 5797d777
......@@ -2,30 +2,205 @@
image: docker:latest
variables:
APT_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/apt"
DOCKER_DRIVER: overlay
DOCKER_BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
DOCKER_LATEST: $CI_REGISTRY_IMAGE:latest
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
stages:
- test
- branch
- latest
before_script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
include:
# Container scanning
- component: $CI_SERVER_FQDN/computing/gitlab/components/container-scanning/container-scanning@~latest
inputs:
job_name: branch_scan
# Software scanning
- component: $CI_SERVER_FQDN/computing/gitlab/components/sast/sast@~latest
inputs:
run_advanced_sast: true
- component: $CI_SERVER_FQDN/computing/gitlab/components/secret-detection/secret-detection@~latest
- component: $CI_SERVER_FQDN/computing/gitlab/components/python/dependency-scanning@~latest
# -- software scanning
# overwrite some settings for the scanning jobs
dependency_scanning:
stage: test
needs: []
variables:
DEBIAN_FRONTEND: "noninteractive"
before_script:
# install some underlying utilities using `apt` so that the dependency
# scanner can use pip to install everything else
- apt-get update -yqq
- apt-get install -yqq
libkrb5-dev
libldap-dev
libsasl2-dev
.sast-analyzer:
stage: test
needs: []
before_script: []
secret_detection:
stage: test
needs: []
before_script: []
# -- testing
.test: &test
image: igwn/base:bookworm
services:
- postgres:15.6
- memcached
variables:
AWS_SES_ACCESS_KEY_ID: "fake_aws_id"
AWS_SES_SECRET_ACCESS_KEY: "fake_aws_key"
DJANGO_ALERT_EMAIL_FROM: "fake_email"
DJANGO_DB_HOST: "postgres"
DJANGO_DB_PORT: "5432"
DJANGO_DB_NAME: "fake_name"
DJANGO_DB_USER: "runner"
DJANGO_DB_PASSWORD: ""
DJANGO_PRIMARY_FQDN: "fake_fqdn"
DJANGO_SECRET_KEY: "fake_key"
DJANGO_SETTINGS_MODULE: "config.settings.container.dev"
DJANGO_TWILIO_ACCOUNT_SID: "fake_sid"
DJANGO_TWILIO_AUTH_TOKEN: "fake_token"
DJANGO_DOCKER_MEMCACHED_ADDR: "memcached:11211"
EGAD_URL: "fake_url"
EGAD_API_KEY: "fake_key"
ENABLE_LVALERT_OVERSEER: "false"
ENABLE_IGWN_OVERSEER: "false"
LVALERT_OVERSEER_PORT: "2"
LVALERT_SERVER: "fake_server"
LVALERT_USER: "fake_user"
LVALERT_PASSWORD: "fake_password"
ENABLE_IGWN_OVERSEER: "false"
IGWN_ALERT_OVERSEER_PORT: "2"
IGWN_ALERT_SERVER: "fake_server"
IGWN_ALERT_USER: "fake_user"
IGWN_ALERT_PASSWORD: "fake_password"
POSTGRES_DB: "${DJANGO_DB_NAME}"
POSTGRES_USER: "${DJANGO_DB_USER}"
POSTGRES_PASSWORD: "${DJANGO_DB_PASSWORD}"
POSTGRES_HOST_AUTH_METHOD: trust
before_script:
# create apt cache directory
- mkdir -pv ${APT_CACHE_DIR}
# set python version
- PYTHON_VERSION="${CI_JOB_NAME##*:}"
- PYTHON_MAJOR="${PYTHON_VERSION:0:1}"
- PYTHON="python3"
# install build requirements
- apt-get -y install gnupg
- sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
- wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
- apt-get -yqq update
- apt-get -o dir::cache::archives="${APT_CACHE_DIR}" install -yqq
git
gnupg
libldap2-dev
libsasl2-dev
libssl-dev
libxml2-dev
krb5-user
libkrb5-dev
libsasl2-modules-gssapi-mit
swig
pkg-config
libpng-dev
libfreetype6-dev
libxslt-dev
${PYTHON}-pip
postgresql-15
postgresql-client-15
libpq-dev
# upgrade pip (requirement for lalsuite)
- ${PYTHON} -m pip install --upgrade pip --break-system-packages
# install everything else from pip
- ${PYTHON} -m pip install -r requirements.txt --break-system-packages
# create logs path required for tests
- mkdir -pv ../logs/
# list packages
- ${PYTHON} -m pip list installed
script:
- PYTHONPATH=${PYTHONPATH}:${PWD}/gracedb ${PYTHON} -m pytest --cov-report term-missing --cov ./gracedb --junitxml=${CI_PROJECT_DIR}/junit.xml
after_script:
- rm -fvr ${PIP_CACHE_DIR}/log
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
artifacts:
reports:
junit: junit.xml
cache:
key: "${CI_JOB_NAME}"
paths:
- .cache/pip
- .cache/apt
coverage: '/^TOTAL\s+.*\s+(\d+\.?\d*)%/'
tags:
- executor-docker
test:3.11:
<<: *test
# -- docker
branch_image:
stage: branch
script:
- docker build --pull -t $DOCKER_BRANCH .
- docker push $DOCKER_BRANCH
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
tags:
- executor-docker
branch_scan:
stage: branch
needs: [branch_image]
# default rules spawn a merge request pipeline, we don't want that
rules:
- if: $CI_COMMIT_BRANCH
variables:
GIT_STRATEGY: fetch
# image to scan
CS_IMAGE: "$DOCKER_BRANCH"
# image to compare to
CS_DEFAULT_BRANCH_IMAGE: "$CI_REGISTRY/computing/gitlab/server:latest"
# path to Dockerfile for remediation
CS_DOCKERFILE_PATH: "Dockerfile"
before_script: []
latest_image:
stage: latest
dependencies:
- branch_image
only:
refs:
- master
script:
- docker pull $DOCKER_BRANCH
- docker tag $DOCKER_BRANCH $DOCKER_LATEST
- docker push $DOCKER_LATEST
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
tags:
- executor-docker
.gitlab/issue_templates/Bug.md 0 → 100644
View file @ 5797d777
## Description of problem
<!--
Describe in detail what you are trying to do and what the result is.
Exact timestamps, error tracebacks, and screenshots (if applicable) are very helpful.
-->
## Expected behavior
<!-- What do you expect to happen instead? -->
## Steps to reproduce
<!-- Step-by-step procedure for reproducing the issue -->
## Context/environment
<!--
Describe the environment you are working in:
* If using the ligo-gracedb client package, which version?
* Your operating system
* Your browser (web interface issues only)
* If you are experiencing this problem while working on a LIGO or Virgo computing cluster, which cluster are you using?
-->
## Suggested solutions
<!-- Any ideas for how to resolve this problem? -->
.gitlab/issue_templates/Feature-request.md 0 → 100644
View file @ 5797d777
## Description of feature request
<!--
Describe your feature request!
Is it a web interface change? Some underlying feature? An API resource?
The more detail you can provide, the better.
-->
## Use cases
<!-- List some specific cases where this feature will be useful -->
## Benefits
<!-- Describe the benefits of adding this feature -->
## Drawbacks
<!--
Are there any drawbacks to adding this feature?
Can you think of any ways in which this will negatively affect the service for any set of users?
-->
## Suggested solutions
<!-- Do you have any ideas for how to implement this feature? -->
Dockerfile
View file @ 5797d777
FROM ligo/base:stretch
FROM debian:bookworm
LABEL name="LIGO GraceDB Django application" \
maintainer="tanner.prestegard@ligo.org" \
date="20181206"
maintainer="alexander.pace@ligo.org" \
date="20240306"
ARG SETTINGS_MODULE="config.settings.container.dev"
COPY docker/SWITCHaai-swdistrib.gpg /etc/apt/trusted.gpg.d
RUN echo 'deb http://pkg.switch.ch/switchaai/debian stretch main' > /etc/apt/sources.list.d/shibboleth.list
RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
COPY docker/backports.pref /etc/apt/preferences.d
RUN apt-get update && \
apt-get -y install gnupg curl
RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' > /etc/apt/sources.list.d/backports.list
RUN echo 'deb http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main' > /etc/apt/sources.list.d/pgdg.list
RUN echo 'deb [trusted=yes] https://hypatia.aei.mpg.de/lsc-amd64-bookworm ./' > /etc/apt/sources.list.d/lscsoft.list
RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
RUN apt-get update && \
apt-get --assume-yes upgrade && \
apt-get install --install-recommends --assume-yes \
apache2 \
emacs-nox \
gcc \
git \
krb5-user \
libkrb5-dev \
libapache2-mod-shib \
libapache2-mod-xsendfile \
libmariadbclient-dev \
libldap2-dev \
libldap-2.5-0 \
libsasl2-dev \
libsasl2-modules-gssapi-mit \
libxml2-dev \
pkg-config \
libpng-dev \
libpq-dev \
libfreetype6-dev \
libxslt-dev \
libsqlite3-dev \
mariadb-client \
nodejs \
ligo-ca-certs \
osg-ca-certs \
python2.7 \
python2.7-dev \
python-glue \
python-pip \
python-voeventlib \
php \
php8.2-pgsql \
php8.2-mbstring \
postgresql-client-15 \
python3 \
python3-dev \
python3-libxml2 \
python3-pip \
procps \
shibboleth \
supervisor \
redis \
shibboleth-sp-common \
shibboleth-sp-utils \
libssl-dev \
swig \
htop \
telnet \
vim && \
apt-get clean && \
npm install -g bower
curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
apt-get update && apt-get install --assume-yes yarn
# Install AWS X-ray daemon
RUN curl -O https://s3.us-east-2.amazonaws.com/aws-xray-assets.us-east-2/xray-daemon/aws-xray-daemon-3.x.deb
RUN dpkg -i aws-xray-daemon-3.x.deb
RUN rm aws-xray-daemon-3.x.deb
# Docker scripts:
COPY docker/entrypoint /usr/local/bin/entrypoint
COPY docker/cleanup /usr/local/bin/cleanup
# Supervisord configs:
COPY docker/supervisord.conf /etc/supervisor/supervisord.conf
COPY docker/supervisord-apache2.conf /etc/supervisor/conf.d/apache2.conf
COPY docker/supervisord-igwn-alert-overseer.conf /etc/supervisor/conf.d/igwn-overseer.conf
COPY docker/supervisord-shibd.conf /etc/supervisor/conf.d/shibd.conf
COPY docker/shibboleth-ds /etc/shibboleth-ds
COPY docker/supervisord-aws-xray.conf /etc/supervisor/conf.d/aws-xray.conf
COPY docker/supervisord-qcluster.conf /etc/supervisor/conf.d/qcluster.conf
# Apache configs:
COPY docker/apache-config /etc/apache2/sites-available/gracedb.conf
COPY docker/mpm_prefork.conf /etc/apache2/mods-enabled/mpm_prefork.conf
# Enable mpm_event module:
RUN rm /etc/apache2/mods-enabled/mpm_prefork.*
RUN rm /etc/apache2/mods-enabled/php8.2.*
RUN cp /etc/apache2/mods-available/mpm_event.* /etc/apache2/mods-enabled/
# Shibboleth configs and certs:
COPY docker/shibboleth-ds /etc/shibboleth-ds
COPY docker/login.ligo.org.cert.LIGOCA.pem /etc/shibboleth/login.ligo.org.cert.LIGOCA.pem
COPY docker/inc-md-cert.pem /etc/shibboleth/inc-md-cert.pem
COPY docker/check_shibboleth_status /usr/local/bin/check_shibboleth_status
......@@ -52,14 +102,17 @@ ADD . /app/gracedb_project
# install gracedb application itself
WORKDIR /app/gracedb_project
RUN bower install --allow-root
RUN pip install --upgrade setuptools wheel && \
pip install -r requirements.txt
RUN pip3 install --upgrade pip --break-system-packages
RUN pip3 install -r requirements.txt --break-system-packages
# install supervisor from pip
RUN pip3 install supervisor --break-system-packages
# Give pip-installed packages priority over distribution packages
ENV PYTHONPATH /usr/local/lib/python2.7/dist-packages:$PYTHONPATH
ENV PYTHONPATH /usr/local/lib/python3.11/dist-packages:$PYTHONPATH
ENV ENABLE_SHIBD false
ENV VIRTUAL_ENV dummy
ENV ENABLE_OVERSEER true
ENV VIRTUAL_ENV /dummy/
# Expose port and run Gunicorn
EXPOSE 8000
......@@ -75,20 +128,58 @@ RUN mkdir /app/logs /app/project_data
WORKDIR /app/gracedb_project
RUN DJANGO_SETTINGS_MODULE=${SETTINGS_MODULE} \
DJANGO_DB_NAME=fake_name \
DJANGO_DB_USER=fake_user \
DJANGO_DB_PASSWORD=fake_password \
DJANGO_SECRET_KEY=fake_key \
DJANGO_PRIMARY_FQDN=fake_fqdn \
DJANGO_ALERT_EMAIL_FROM=fake_email \
EGAD_URL=fake_url \
EGAD_API_KEY=fake_key \
LVALERT_USER=fake_user \
LVALERT_PASSWORD=fake_password \
LVALERT_SERVER=fake_server \
LVALERT_OVERSEER_PORT=2 \
python manage.py collectstatic --noinput
IGWN_ALERT_USER=fake_user \
IGWN_ALERT_PASSWORD=fake_password \
IGWN_ALERT_SERVER=fake_server \
IGWN_ALERT_OVERSEER_PORT=2 \
IGWN_ALERT_GROUP=fake_group \
DJANGO_TWILIO_ACCOUNT_SID=fake_sid \
DJANGO_TWILIO_AUTH_TOKEN=fake_token \
DJANGO_AWS_ELASTICACHE_ADDR=fake_address:11211 \
AWS_SES_ACCESS_KEY_ID=fake_aws_id \
AWS_SES_SECRET_ACCESS_KEY=fake_aws_key \
python3 manage.py collectstatic --noinput
RUN rm -rf /app/logs/* /app/project_data/*
RUN useradd -M -u 50001 -g www-data -s /bin/false gracedb
RUN chown gracedb:www-data /app/logs /app/project_data && \
chmod 0750 /app/logs /app/project_data
#RUN groupadd -r xray
#RUN useradd -M -u 50002 -g xray -s /bin/false xray
# set secure file/directory permissions. In particular, ADD command at
# beginning of recipe inherits umask of user running the build
RUN chmod 0755 /usr/local/bin/entrypoint && \
chmod 0755 /usr/local/bin/cleanup && \
chown gracedb:www-data /app/logs /app/project_data && \
chmod 0750 /app/logs /app/project_data && \
find /app/gracedb_project -type d -exec chmod 0755 {} + && \
find /app/gracedb_project -type f -exec chmod 0644 {} +
# create and set scitoken key cache directory
RUN mkdir /app/scitokens_cache && \
chown gracedb:www-data /app/scitokens_cache && \
chmod 0750 /app/scitokens_cache
ENV XDG_CACHE_HOME /app/scitokens_cache
# patch voeventparse for python3.10+:
RUN sed -i 's/collections.Iterable/collections.abc.Iterable/g' /usr/local/lib/python3.11/dist-packages/voeventparse/voevent.py
# Remove packages that expose security vulnerabilities and close out.
# Edit: zlib1g* can't be removed because of a PrePend error
RUN apt-get --assume-yes --purge autoremove wget libaom3 node-ip
RUN apt-get clean
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
ENTRYPOINT [ "/usr/local/bin/entrypoint" ]
CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
LICENSE 0 → 100644
View file @ 5797d777
This diff is collapsed.
bower.json
View file @ 5797d777
{
"name": "gracedb",
"dependencies": {
"dgrid": "0.4.0",
"dijit": "1.10.4",
"dojox": "1.10.4",
"jquery": "3.2.1",
"moment-timezone": "0.5.0",
"moment": "2.11.1"
"dojox": "1.10.4"
}
}
config/gunicorn_config.py
View file @ 5797d777
......@@ -6,6 +6,14 @@ from os.path import abspath, dirname, join
import sys
import multiprocessing
# Useful function for getting environment variables
def get_from_env(envvar, default_value=None, fail_if_not_found=True):
value = os.environ.get(envvar, default_value)
if (value == default_value and fail_if_not_found):
raise ImproperlyConfigured(
'Could not get environment variable {0}'.format(envvar))
return value
# Parameters
GUNICORN_PORT = 8080
LOG_DIR = abspath(join(dirname(__file__), "..", "..", "logs"))
......@@ -14,27 +22,143 @@ LOG_DIR = abspath(join(dirname(__file__), "..", "..", "logs"))
# Bind to localhost on specified port
bind = "127.0.0.1:{port}".format(port=GUNICORN_PORT)
# Number of workers = 2*CPU + 1 (recommendation from Gunicorn documentation)
workers = multiprocessing.cpu_count()*2 + 1
# Number of workers -----------------------------------------------------------
# 2*CPU + 1 (recommendation from Gunicorn documentation)
# bumped to 4*CPU + 1 after testing. Maybe increase this number in the cloud
# deployment?
workers = int(get_from_env('GUNICORN_WORKERS',
default_value=multiprocessing.cpu_count()*3 + 1,
fail_if_not_found=False))
# NOTE: it was found in extensive testing that threads > 1 are prone
# to connection lockups. Leave this at 1 for safety until there are
# fixes in gunicorn.
# Why not sync? The sync worker is prone to timeout for long requests,
# like big queries. But gthread sends a heartbeat back to the main worker
# to keep it alive. We could just set the timeout to a really large number
# which would keep the long requests stable, but if there is a stuck worker,
# then they would be subject to that really long timeout. It's a tradeoff.
# All this goes away with async workers, but as of 3.2, django's ORM does support
# async, and testing failed pretty catastrophically and unreliably.
threads = int(get_from_env('GUNICORN_THREADS',
default_value=1,
fail_if_not_found=False))
# Worker connections. Limit the number of connections between apache<-->gunicorn
worker_connections = workers * threads
# Worker type
worker_type = 'sync'
# Worker class ----------------------------------------------------------------
# sync by default, generally safe and low-resource:
# https://docs.gunicorn.org/en/stable/design.html#sync-workers
# Max requests settings - a worker restarts after handling this many
# requests. May be useful if we have memory leak problems.
worker_class = get_from_env('GUNICORN_WORKER_CLASS',
default_value='gthread',
fail_if_not_found=False)
# Timeout ---------------------------------------------------------------------
# If not specified, the timeout default is 30 seconds:
# https://gunicorn-docs.readthedocs.io/en/stable/settings.html#worker-processes
timeout = get_from_env('GUNICORN_TIMEOUT',
default_value=30,
fail_if_not_found=False)
graceful_timeout = timeout
# max_requests settings -------------------------------------------------------
# The maximum number of requests a worker will process before restarting.
# May be useful if we have memory leak problems.
# The jitter is drawn from a uniform distribution:
# randint(0, max_requests_jitter)
#max_requests = 0
#max_requests_jitter = 0
max_requests = get_from_env('GUNICORN_MAX_REQUESTS',
default_value=5000,
fail_if_not_found=False)
max_requests_jitter = get_from_env('GUNICORN_MAX_REQUESTS_JITTER',
default_value=250,
fail_if_not_found=False)
# keepalive -------------------------------------------------------------------
# The number of seconds to wait for requests on a Keep-Alive connection.
# Generally set in the 1-5 seconds range for servers with direct connection
# to the client (e.g. when you don’t have separate load balancer).
# When Gunicorn is deployed behind a load balancer, it often makes sense to set
# this to a higher value.
# NOTE: force gunicorn to close its connection to apache after each request.
# This has been the source of so many 502's. Basically in periods of high activity,
# gunicorn would hold on to open sockets with apache, and just deadlock itself:
# https://github.com/benoitc/gunicorn/issues/2917
keepalive = get_from_env('GUNICORN_KEEPALIVE',
default_value=0,
fail_if_not_found=False)
# preload_app -----------------------------------------------------------------
# Load application code before the worker processes are forked.
# By preloading an application you can save some RAM resources as well as speed
# up server boot times. Although, if you defer application loading to each
# worker process, you can reload your application code easily by restarting
# workers.
# If you aren't going to make use of on-the-fly reloading, consider preloading
# your application code to reduce its memory footprint. So, turn this on in
# production. This is default set to False for development, but
# **TURN THIS TO TRUE FOR AWS DEPLOYMENT **
preload_app = get_from_env('GUNICORN_PRELOAD_APP',
default_value=True,
fail_if_not_found=False)
# Logging ---------------------------------------------------------------------
# Access log
accesslog = join(LOG_DIR, "gunicorn_access.log")
access_log_format = '%(t)s %(h)s %(l)s %(u)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
access_log_format = ('GUNICORN | %(h)s %(l)s %(u)s %(t)s '
'"%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"')
# Error log
errorlog = join(LOG_DIR, "gunicorn_error.log")
loglevel = 'debug'
# debug logging doesn't provide actual information. And this will
# eliminate the "Connection closed." messages while still giving info
# about worker restarts.
loglevel = 'info'
capture_output = True
#forwarded_allow_ips = '127.0.0.1'
#proxy_allow_ips = '127.0.0.1'
# using /dev/shm/ instead of /tmp for the temporary worker directory. See:
# https://pythonspeed.com/articles/gunicorn-in-docker/
# “in AWS an EBS root instance volume may sometimes hang for half a minute
# and during this time Gunicorn workers may completely block.”
worker_tmp_dir='/dev/shm'
# Override logger class to modify error format
from gunicorn.glogging import Logger
class CustomLogger(Logger):
error_fmt = 'GUNICORN | ' + Logger.error_fmt
logger_class = CustomLogger
def post_fork(server, worker):
server.log.info("Worker spawned (pid: %s)", worker.pid)
def pre_fork(server, worker):
pass
def pre_exec(server):
server.log.info("Forked child, re-executing.")
def when_ready(server):
server.log.info("Server is ready. Spawning workers")
def worker_int(worker):
worker.log.info("worker received INT or QUIT signal")
def worker_abort(worker):
worker.log.info("worker received SIGABRT signal")
config/settings/base.py
View file @ 5797d777
from cloghandler import ConcurrentRotatingFileHandler
from concurrent_log_handler import ConcurrentRotatingFileHandler
from datetime import datetime, timedelta
import os, time, logging
import os, time, logging, multiprocessing
from os.path import abspath, dirname, join
import socket
from django.core.exceptions import ImproperlyConfigured
from aws_xray_sdk.core.exceptions.exceptions import SegmentNotFoundException
# Set up path to root of project
BASE_DIR = abspath(join(dirname(__file__), "..", ".."))
......@@ -12,17 +14,69 @@ PROJECT_ROOT = join(BASE_DIR, "gracedb")
# Other useful paths
PROJECT_DATA_DIR = join(BASE_DIR, "..", "project_data")
# Useful function for getting environment variables
def get_from_env(envvar, default_value=None, fail_if_not_found=True):
value = os.environ.get(envvar, default_value)
if (value == default_value and fail_if_not_found):
raise ImproperlyConfigured(
'Could not get environment variable {0}'.format(envvar))
return value
def parse_envvar_bool(x):
return x.lower() in ['t', 'true', '1']
# a sentry before_send function that filters aws SegmentNotFoundException's.
# these exceptions are harmless and occur when performing management tasks
# outside of the core gracedb app. but sentry picks it up and reports it as
# an error which is ANNOYING.
def before_send(event, hint):
if "exc_info" in hint:
exc_type, exc_value, tb = hint["exc_info"]
if isinstance(exc_value, (SegmentNotFoundException,)):
return None
return event
# Maintenance mode
MAINTENANCE_MODE = False
MAINTENANCE_MODE_MESSAGE = None
# Enable/Disable Information Banner:
INFO_BANNER_ENABLED = False
INFO_BANNER_MESSAGE = "TEST MESSAGE"
# Beta reports page:
BETA_REPORTS_LINK = False
# Version ---------------------------------------------------------------------
PROJECT_VERSION = '2.31.0'
# Unauthenticated access ------------------------------------------------------
# This variable controls whether unauthenticated access is allowed *ANYWHERE*
# on this service, except the home page, which is always public.
UNAUTHENTICATED_ACCESS = True
# This variable should eventually control whether unauthenticated access is
# allowed *ANYWHERE* on this service, except the home page, which is always
# public. For now, it just controls the API and the public alerts page.
# Update: make this updatable from the environment:
UNAUTHENTICATED_ACCESS = parse_envvar_bool(
get_from_env('ENABLE_UNAUTHENTICATED_ACCESS',
fail_if_not_found=False, default_value="true")
)
# Miscellaneous settings ------------------------------------------------------
# Debug mode is off by default
DEBUG = False
# When debug mode is enabled, use custom reporter
DEFAULT_EXCEPTION_REPORTER = 'core.utils.CustomExceptionReporter'
# Number of results to show on latest page
LATEST_RESULTS_NUMBER = 50
LATEST_RESULTS_NUMBER = 25
# Maximum number of log messages to display before throwing
# a warning. NOTE: There should be a better way of doing this,
# but just put in the hard cutoff for right now
# Set to cover this:
# https://gracedb.ligo.org/events/G184098
TOO_MANY_LOG_ENTRIES = int(get_from_env('DJANGO_TOO_MANY_LOG_ENTRIES',
fail_if_not_found=False, default_value=2000))
# Path to root URLconf
ROOT_URLCONF = '{module}.urls'.format(module=os.path.basename(CONFIG_ROOT))
......@@ -34,8 +88,9 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
# MANAGERS defines who gets broken link notifications when
# BrokenLinkEmailsMiddleware is enabled
ADMINS = [
("Tanner Prestegard", "tanner.prestegard@ligo.org"),
("Alexander Pace", "alexander.pace@ligo.org"),
("Duncan Meacher", "duncan.meacher@ligo.org"),
("Daniel Wysocki", "daniel.wysocki@ligo.org"),
]
MANAGERS = ADMINS
......@@ -54,9 +109,20 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
TWIML_BASE_URL = 'https://handler.twilio.com/twiml/'
# TwiML bin SIDs (for Twilio)
TWIML_BIN = {
'new': 'EH761b6a35102737e3d21830a484a98a08',
'label_added': 'EHb596a53b9c92a41950ce1a47335fd834',
'event': {
'new': 'EH761b6a35102737e3d21830a484a98a08',
'update': 'EH95d69491c166fbe8888a3b83b8aff4af',
'label_added': 'EHb596a53b9c92a41950ce1a47335fd834',
'label_removed': 'EH071c034f27f714bb7a832e85e6f82119',
},
'superevent': {
'new': 'EH5d4d61f5aee9f8687c5bc7d9d42acab9',
'update': 'EH35356707718e1b9a887c50359c3ab064',
'label_added': 'EH244c07ceeb152c6a374e4ffbd853e7a4',
'label_removed': 'EH9d796ce6a80e282a5c96e757e5c39406',
},
'test': 'EH6c0a168b0c6b011047afa1caeb49b241',
'verify': 'EHfaea274d4d87f6ff152ac39fea3a87d4',
}
# Use timezone-aware datetimes internally
......@@ -89,6 +155,24 @@ LOGOUT_REDIRECT_URL = 'home'
SEND_XMPP_ALERTS = False
SEND_PHONE_ALERTS = False
SEND_EMAIL_ALERTS = False
SEND_MATTERMOST_ALERTS = False
# igwn-alert group settings. the default development group is 'lvalert-dev'
# for the container deployments, the variable will be overwriten by the
# IGWN_ALERT_GROUP environment variable.
DEFAULT_IGWN_ALERT_GROUP = 'lvalert-dev'
# enable/disable sending alerts to topics that have the search tag
# for g/e-events. default to false, so only send to {group}_{pipeline}
SEND_TO_SEARCH_TOPICS = parse_envvar_bool(
get_from_env('IGWN_ALERT_SEARCH_TOPICS',
fail_if_not_found=False, default_value="false")
)
# overseer timeout:
OVERSEER_TIMEOUT = float(get_from_env('IGWN_ALERT_OVERSEER_TIMEOUT',
fail_if_not_found=False, default_value=0.1))
# Use LVAlert Overseer?
USE_LVALERT_OVERSEER = True
# For each LVAlert server, a separate instance of LVAlert Overseer
......@@ -97,16 +181,17 @@ USE_LVALERT_OVERSEER = True
# listen_port: port which that instance of overseer is listening on
LVALERT_OVERSEER_INSTANCES = [
{
"lvalert_server": "lvalert-test.cgca.uwm.edu",
"listen_port": 8001,
"lvalert_server": "kafka://kafka.scimma.org/",
"listen_port": 8002,
"igwn_alert_group": DEFAULT_IGWN_ALERT_GROUP,
},
]
# Access and authorization ----------------------------------------------------
# Some proper names related to authorization
LVC_GROUP = 'Communities:LSCVirgoLIGOGroupMembers'
LVEM_GROUP = 'gw-astronomy:LV-EM'
LVEM_OBSERVERS_GROUP = 'gw-astronomy:LV-EM:Observers'
LVC_GROUP = 'internal_users'
LVEM_GROUP = 'lvem_users'
LVEM_OBSERVERS_GROUP = 'lvem_observers'
PUBLIC_GROUP = 'public_users'
PRIORITY_USERS_GROUP = 'priority_users'
......@@ -120,21 +205,13 @@ ACCESS_MANAGERS_GROUP = 'access_managers'
EM_ADVOCATE_GROUP = 'em_advocates'
# Superevent managers
SUPEREVENT_MANAGERS_GROUP = 'superevent_managers'
# RRT group name:
RRT_MEMBERS_GROUP = 'rrt_members'
# Analysis groups
# Analysis group name for non-GW events
EXTERNAL_ANALYSIS_GROUP = 'External'
# Groups directly managed by GraceDB admins
ADMIN_MANAGED_GROUPS = [EM_ADVOCATE_GROUP, EXEC_GROUP, PUBLIC_GROUP,
SUPEREVENT_MANAGERS_GROUP, ACCESS_MANAGERS_GROUP]
# NOTE: soon, only superevent_managers will be fully managed by us. The other
# groups will be managed by the LIGO auth infrastructure for human users. But
# we will still have to still manage robot account membership in these groups:
# access_managers, executives, and em_advocates.
# NOTE: public_users is also managed by us and has only one member
# (guardian.AnonymousUser)
# Tag to apply to log messages to allow EM partners to view
EXTERNAL_ACCESS_TAGNAME = 'lvem'
PUBLIC_ACCESS_TAGNAME = 'public'
......@@ -176,17 +253,30 @@ COINC_PIPELINES = [
'spiir',
'MBTAOnline',
'pycbc',
'MBTA',
'PyGRB',
]
GRB_PIPELINES = [
'Fermi',
'Swift',
'INTEGRAL',
'AGILE',
'CHIME',
'SVOM',
]
# SkyAlert stuff - used for VOEvents (?) --------------------------------------
IVORN_PREFIX = "ivo://gwnet/LVC#"
SKYALERT_ROLE = "test"
SKYALERT_DESCRIPTION = "Report of a candidate gravitational wave event"
SKYALERT_SUBMITTERS = ['Patrick Brady', 'Brian Moe']
# List of pipelines that have been depreciated:
DEPRECIATED_PIPELINES = [
'X',
'Q',
'Omega',
]
UNAPPROVED_PIPELINES = []
# VOEvent stream --------------------------------------------------------------
VOEVENT_STREAM = 'gwnet/LVC'
# Stuff related to report/plot generation -------------------------------------
......@@ -205,7 +295,7 @@ REPORT_INFO_URL_PREFIX = "/report_info/"
REPORT_IFAR_IMAGE_DIR = PROJECT_DATA_DIR
# Stuff for the new rates plot
BINNED_COUNT_PIPELINES = ['gstlal', 'MBTAOnline', 'CWB', 'oLIB', 'spiir']
BINNED_COUNT_PIPELINES = ['gstlal', 'MBTAOnline', 'MBTA', 'CWB', 'oLIB', 'spiir']
BINNED_COUNT_FILE = join(PROJECT_DATA_DIR, "binned_counts.json")
# Defaults for RSS feed
......@@ -228,6 +318,7 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
},
# For API throttles
'throttles': {
'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
'LOCATION': 'api_throttle_cache', # Table name
......@@ -281,6 +372,11 @@ X509_INFOS_HEADER = 'HTTP_X_FORWARDED_TLS_CLIENT_CERT_INFOS'
# Path to CA store for X509 certificate verification
CAPATH = '/etc/grid-security/certificates'
# SciTokens claims settings
SCITOKEN_ISSUER = ['https://cilogon.org/igwn', 'https://test.cilogon.org/igwn', 'https://osdf.igwn.org/cit']
SCITOKEN_AUDIENCE = ["ANY"]
SCITOKEN_SCOPE = "gracedb.read"
# List of authentication backends to use when attempting to authenticate
# a user. Will be used in this order. Authentication for the API is
# handled by the REST_FRAMEWORK dictionary.
......@@ -292,6 +388,7 @@ AUTHENTICATION_BACKENDS = [
# List of middleware classes to use.
MIDDLEWARE = [
'core.middleware.maintenance.MaintenanceModeMiddleware',
'events.middleware.PerformanceMiddleware',
'core.middleware.accept.AcceptMiddleware',
'core.middleware.api.ClientVersionMiddleware',
......@@ -299,6 +396,7 @@ MIDDLEWARE = [
'django.middleware.common.CommonMiddleware',
'core.middleware.proxy.XForwardedForMiddleware',
'user_sessions.middleware.SessionMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'ligoauth.middleware.ShibbolethWebAuthMiddleware',
......@@ -315,29 +413,35 @@ SITE_ID=1
INSTALLED_APPS = [
'django.contrib.auth',
'django.contrib.admin',
'django_ses',
'django.contrib.contenttypes',
'user_sessions',
'django.contrib.sites',
'django.contrib.staticfiles',
'django.contrib.messages',
'alerts',
'annotations',
'api',
'core',
'events',
'ligoauth',
'search',
'superevents',
'userprofile',
'gwtc',
'rest_framework',
'guardian',
'django_twilio',
'django_extensions',
'django.contrib.sessions',
'computedfields',
'django_postgres_vacuum',
'django_q',
]
# Aliases for django-extensions shell_plus
SHELL_PLUS_MODEL_ALIASES = {
# Two 'Group' models - auth.Group and gracedb.Group
'auth': {'Group': 'AuthGroup'},
'auth': {'Group': 'DjangoGroup'},
# Superevents models which have the same name as
# models in the events app
'superevents': {
......@@ -365,12 +469,13 @@ REST_FRAMEWORK = {
'api.throttling.BurstAnonRateThrottle',
),
'DEFAULT_THROTTLE_RATES': {
'anon_burst': '3/second',
'event_creation': '1/second',
'annotation' : '10/second',
'anon_burst': '300/minute',
'event_creation': '50/second',
'annotation' : '50/second',
},
'DEFAULT_AUTHENTICATION_CLASSES': (
'api.backends.GraceDbAuthenticatedAuthentication',
'api.backends.GraceDbSciTokenAuthentication',
'api.backends.GraceDbX509Authentication',
'api.backends.GraceDbBasicAuthentication',
),
......@@ -386,9 +491,6 @@ if UNAUTHENTICATED_ACCESS is True:
REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'] = \
('rest_framework.permissions.IsAuthenticatedOrReadOnly',)
# Location of packages installed by bower
BOWER_DIR = join(BASE_DIR, "..", "bower_components")
# Location of static components, CSS, JS, etc.
STATIC_ROOT = join(BASE_DIR, "static_root")
STATIC_URL = "/static/"
......@@ -399,7 +501,6 @@ STATICFILES_FINDERS = [
]
STATICFILES_DIRS = [
join(PROJECT_ROOT, "static"),
BOWER_DIR,
]
# Added in order to perform data migrations on Django apps
......@@ -444,6 +545,8 @@ GUARDIAN_RENDER_403 = True
# See http://django-guardian.readthedocs.io/en/latest/userguide/custom-user-model.html
GUARDIAN_MONKEY_PATCH = False
# Lifetime of verification codes for contacts
VERIFICATION_CODE_LIFETIME = timedelta(hours=1)
# Basic auth passwords for LVEM scripted access expire after 365 days.
PASSWORD_EXPIRATION_TIME = timedelta(days=365)
......@@ -473,7 +576,7 @@ LOG_DIR = abspath(join(BASE_DIR, "..", "logs"))
# don't need it since we're using custom filters.
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format': '%(asctime)s | %(message)s',
......@@ -487,7 +590,14 @@ LOGGING = {
'format': '%(asctime)s.%(msecs)03d | %(name)s | %(levelname)s | ' \
+ '%(filename)s, line %(lineno)s | %(message)s',
'datefmt': LOG_DATEFMT,
}
},
'console': {
'format': ('DJANGO | %(asctime)s.%(msecs)03d | {host} | {ip} | '
'%(name)s | %(levelname)s | %(filename)s, line %(lineno)s | '
'%(message)s').format(host=INTERNAL_HOSTNAME,
ip=INTERNAL_IP_ADDRESS),
'datefmt': LOG_DATEFMT,
},
},
'handlers': {
'null': {
......@@ -520,7 +630,12 @@ LOGGING = {
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler'
}
},
'console': {
'class': 'logging.StreamHandler',
'formatter': 'console',
'level': 'DEBUG',
},
},
'loggers': {
'django': {
......@@ -568,15 +683,108 @@ LOGGING = {
'propagate': True,
'level': LOG_LEVEL,
},
'userprofile': {
'handlers': ['debug_file','error_file'],
'propagate': True,
'level': LOG_LEVEL,
},
'django.request': {
'django.request': {
'handlers': ['mail_admins'],
'level': 'ERROR',
'propagate': False,
},
},
}
# Turn off debug/error emails when in maintenance mode.
if MAINTENANCE_MODE:
LOGGING['loggers']['django.request']['handlers'].remove('mail_admins')
# Turn off logging emails of django requests:
# FIXME: figure out more reliable logging solution
LOGGING['loggers']['django.request']['handlers'].remove('mail_admins')
# Define some words for the instance stub:
ENABLED = {True: "enabled", False: "disabled"}
# Upgrading to django 3.2 produces warning: "Auto-created primary key used
# when not defining a primary key type, by default 'django.db.models.AutoField'.
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
# Define window for neighbouring s events of a given g event.
EVENT_SUPEREVENT_WINDOW_BEFORE = 100
EVENT_SUPEREVENT_WINDOW_AFTER = 100
# Define which observation periods to show on the public events page:
# TODO: Group O4b and O4a under O4, once implemented.
PUBLIC_PAGE_RUNS = ['O4', 'O4c', 'O4b', 'O4a', 'ER16', 'ER15', 'O3']
# Define how long to cache the public page:
PUBLIC_PAGE_CACHING = int(get_from_env('DJANGO_PUBLIC_PAGE_CACHING',
fail_if_not_found=False, default_value=300))
# Define the number of results per page on the public page:
PUBLIC_PAGE_RESULTS = int(get_from_env('DJANGO_PUBLIC_PAGE_RESULTS',
fail_if_not_found=False, default_value=15))
# Define DATA_UPLOAD_MAX_MEMORY_SIZE for larger uploads:
DATA_UPLOAD_MAX_MEMORY_SIZE = int(get_from_env('DJANGO_DATA_UPLOAD_MAX_MEMORY_SIZE',
fail_if_not_found=False, default_value=20*1024*1024))
# Choose whether to use to Julian or Civil definition of year
# when displaying far's in /year:
DISPLAY_CIVIL_YEAR_FAR = parse_envvar_bool(
get_from_env('DJANGO_DISPLAY_CIVIL_YEAR_FAR',
fail_if_not_found=False, default_value="false")
)
if DISPLAY_CIVIL_YEAR_FAR:
DAYS_PER_YEAR = 365.0
else:
DAYS_PER_YEAR = 365.25
# Put in some setting for the redis queue backend
ENABLE_REDIS_QUEUE = parse_envvar_bool(
get_from_env('DJANGO_ENABLE_REDIS_QUEUE',
fail_if_not_found=False, default_value="false")
)
REDIS_QUEUE_ADDRESS = get_from_env('DJANGO_REDIS_QUEUE_ADDRESS',
fail_if_not_found=False, default_value="127.0.0.1")
REDIS_QUEUE_PORT = int(get_from_env('DJANGO_REDIS_QUEUE_PORT',
fail_if_not_found=False, default_value="6379")
)
REDIS_QUEUE_DATABASE = int(get_from_env('DJANGO_REDIS_QUEUE_DATABASE',
fail_if_not_found=False, default_value=0)
)
REDIS_QUEUE_WORKERS = int(get_from_env('DJANGO_REDIS_QUEUE_WORKERS',
default_value=multiprocessing.cpu_count(),
fail_if_not_found=False))
REDIS_QUEUE_RETRY = int(get_from_env('DJANGO_REDIS_QUEUE_RETRY',
default_value=40,
fail_if_not_found=False))
REDIS_QUEUE_TIMEOUT = int(get_from_env('DJANGO_REDIS_QUEUE_TIMEOUT',
default_value=30,
fail_if_not_found=False))
REDIS_QUEUE_RECYCLE = int(get_from_env('DJANGO_REDIS_QUEUE_RECYCLE',
default_value=500,
fail_if_not_found=False))
ENABLE_REDIS_CLUSTERED = parse_envvar_bool(
get_from_env('DJANGO_ENABLE_REDIS_CLUSTERED',
fail_if_not_found=False, default_value="false")
)
# Define some defaults for the q-cluster parameters:
Q_CLUSTER_NAME = 'gracedb-async-queue'
Q_CLUSTER_LABEL = 'gracedb q cluster'
if not ENABLE_REDIS_CLUSTERED:
Q_CLUSTER_NAME+=f'-{INTERNAL_HOSTNAME}'
Q_CLUSTER_LABEL+=f', {INTERNAL_HOSTNAME}'
# Define MAX_DATATABLES_RESULTS to limit memory usage for web queries:
MAX_DATATABLES_RESULTS = int(get_from_env('DJANGO_MAX_DATATABLES_RESULTS',
fail_if_not_found=False, default_value=1000))
config/settings/container/base.py
View file @ 5797d777
# For running a containerized version of the service that gets secrets
# from environment variables. Builds on base.py settings.
import os
from django.core.exceptions import ImproperlyConfigured
from ..base import *
# Get required variables from environment variables ---------------------------
# Get database user from environment and check
db_user = os.environ.get('DJANGO_DB_USER', None)
if db_user is None:
raise ImproperlyConfigured('Could not get database user from envvars.')
# Get database password from environment and check
db_password = os.environ.get('DJANGO_DB_PASSWORD', None)
if db_password is None:
......@@ -25,64 +29,343 @@ if SECRET_KEY is None:
SERVER_FQDN = os.environ.get('DJANGO_PRIMARY_FQDN', None)
if SERVER_FQDN is None:
raise ImproperlyConfigured('Could not get FQDN from envvars.')
LIGO_FQDN = SERVER_FQDN
# Get LVAlert server
lvalert_server = os.environ.get('LVALERT_SERVER', None)
if lvalert_server is None:
raise ImproperlyConfigured('Could not get LVAlert server from envvars.')
# Get LVAlert Overseer listen port
lvalert_overseer_port = os.environ.get('LVALERT_OVERSEER_PORT', None)
if lvalert_overseer_port is None:
raise ImproperlyConfigured('Could not get LVAlert overseer port '
## EGAD (External GraceDB Alert Dispatcher) configuration
ENABLE_EGAD_EMAIL = parse_envvar_bool(
get_from_env('ENABLE_EGAD_EMAIL',
fail_if_not_found=False, default_value="false")
)
ENABLE_EGAD_KAFKA = parse_envvar_bool(
get_from_env('ENABLE_EGAD_KAFKA',
fail_if_not_found=False, default_value="false")
)
ENABLE_EGAD_MATTERMOST = parse_envvar_bool(
get_from_env('ENABLE_EGAD_MATTERMOST',
fail_if_not_found=False, default_value="false")
)
ENABLE_EGAD_PHONE = parse_envvar_bool(
get_from_env('ENABLE_EGAD_PHONE',
fail_if_not_found=False, default_value="false")
)
ENABLE_EGAD = (
ENABLE_EGAD_EMAIL or ENABLE_EGAD_KAFKA
or ENABLE_EGAD_MATTERMOST or ENABLE_EGAD_PHONE
)
EGAD_URL = get_from_env('EGAD_URL',
fail_if_not_found=ENABLE_EGAD, default_value=None)
EGAD_API_KEY = get_from_env('EGAD_API_KEY',
fail_if_not_found=ENABLE_EGAD, default_value=None)
# Turn LVAlert on/off from the environment. Adding this
# to turn lvalerts on/off from docker compose/update instead
# of having to rebuild containers. If the environment variable
# isn't set, then revert to the hardwired behavior:
xmpp_env_var = get_from_env('SEND_LVALERT_XMPP_ALERTS',
default_value=SEND_XMPP_ALERTS,
fail_if_not_found=False)
# Fix for other boolean values:
if (isinstance(xmpp_env_var, str) and
xmpp_env_var.lower() in ['true','t','1']):
SEND_XMPP_ALERTS=True
elif (isinstance(xmpp_env_var, str) and
xmpp_env_var.lower() in ['false','f','0']):
SEND_XMPP_ALERTS=False
else:
SEND_XMPP_ALERTS = True
# Get igwn_alert_overseer status:
igwn_alert_on = get_from_env(
'ENABLE_IGWN_OVERSEER',
default_value=False,
fail_if_not_found=False
)
if (isinstance(igwn_alert_on, str) and
igwn_alert_on.lower() in ['true', 't', '1']):
igwn_alert_overseer_on = True
else:
igwn_alert_overseer_on = False
# Get igwn-alert server
igwn_alert_server = os.environ.get('IGWN_ALERT_SERVER', None)
if igwn_alert_server is None:
raise ImproperlyConfigured('Could not get igwn-alert server from envvars.')
# Get igwn-alert Overseer listen port
igwn_alert_overseer_port = os.environ.get('IGWN_ALERT_OVERSEER_PORT', None)
if igwn_alert_overseer_port is None:
raise ImproperlyConfigured('Could not get igwn-alert overseer port '
'from envvars.')
# Get LVAlert username
lvalert_user = os.environ.get('LVALERT_USER', None)
if lvalert_user is None:
raise ImproperlyConfigured('Could not get LVAlert username from envvars.')
# Get igwn-alert group from envirnment:
igwn_alert_group = os.environ.get('IGWN_ALERT_GROUP', DEFAULT_IGWN_ALERT_GROUP)
# Get igwn-alert username
igwn_alert_user = os.environ.get('IGWN_ALERT_USER', None)
if igwn_alert_user is None:
raise ImproperlyConfigured('Could not get igwn-alert username from envvars.')
# Get LVAlert password
lvalert_password = os.environ.get('LVALERT_PASSWORD', None)
if lvalert_password is None:
raise ImproperlyConfigured('Could not get LVAlert password from envvars.')
# Get igwn-alert password
igwn_alert_password = os.environ.get('IGWN_ALERT_PASSWORD', None)
if igwn_alert_password is None:
raise ImproperlyConfigured('Could not get igwn-alert password from envvars.')
# Get Twilio account information from environment
# FIXME
TWILIO_ACCOUNT_SID = os.environ.get('DJANGO_TWILIO_ACCOUNT_SID', 'abcd')
TWILIO_AUTH_TOKEN = os.environ.get('DJANGO_TWILIO_AUTH_TOKEN', 'abcd')
TWILIO_ACCOUNT_SID = os.environ.get('DJANGO_TWILIO_ACCOUNT_SID', None)
if TWILIO_ACCOUNT_SID is None:
raise ImproperlyConfigured('Could not get Twilio acct SID from envvars.')
TWILIO_AUTH_TOKEN = os.environ.get('DJANGO_TWILIO_AUTH_TOKEN', None)
if TWILIO_AUTH_TOKEN is None:
raise ImproperlyConfigured('Could not get Twilio auth token from envvars.')
# Get maintenance mode settings from environment
maintenance_mode = get_from_env(
'DJANGO_MAINTENANCE_MODE_ACTIVE',
default_value=False,
fail_if_not_found=False
)
# DB "cool-down" factor for when a db conflict is detected. This
# factor scales a random number of seconds between zero and one.
DB_SLEEP_FACTOR = get_from_env(
'DJANGO_DB_SLEEP_FACTOR',
default_value=1.0,
fail_if_not_found=False
)
# Fix the factor (str to float)
try:
DB_SLEEP_FACTOR = float(DB_SLEEP_FACTOR)
except:
DB_SLEEP_FACTOR = 1.0
if (isinstance(maintenance_mode, str) and
maintenance_mode.lower() in ['true', 't', '1']):
MAINTENANCE_MODE = True
MAINTENANCE_MODE_MESSAGE = \
get_from_env('DJANGO_MAINTENANCE_MODE_MESSAGE', fail_if_not_found=False)
# Get info banner settings from environment
info_banner_enabled = get_from_env(
'DJANGO_INFO_BANNER_ENABLED',
default_value=False,
fail_if_not_found=False
)
# fix for other booleans:
if (isinstance(info_banner_enabled, str) and
info_banner_enabled.lower() in ['true','t','1']):
INFO_BANNER_ENABLED = True
INFO_BANNER_MESSAGE = \
get_from_env('DJANGO_INFO_BANNER_MESSAGE', fail_if_not_found=False)
# Get reports page boolean:
beta_reports_link = get_from_env(
'DJANGO_BETA_REPORTS_LINK',
default_value=False,
fail_if_not_found=False
)
# fix for other booleans:
if (isinstance(beta_reports_link, str) and
beta_reports_link.lower() in ['true','t','1']):
BETA_REPORTS_LINK = True
# Get email settings from environment
EMAIL_BACKEND = 'django_ses.SESBackend'
AWS_SES_ACCESS_KEY_ID = get_from_env('AWS_SES_ACCESS_KEY_ID')
AWS_SES_SECRET_ACCESS_KEY = get_from_env('AWS_SES_SECRET_ACCESS_KEY')
AWS_SES_REGION_NAME = get_from_env('AWS_SES_REGION_NAME',
default_value='us-west-2', fail_if_not_found=False)
AWS_SES_REGION_ENDPOINT = get_from_env('AWS_SES_REGION_ENDPOINT',
default_value='email.us-west-2.amazonaws.com', fail_if_not_found=False)
AWS_SES_AUTO_THROTTLE = 0.25
ALERT_EMAIL_FROM = get_from_env('DJANGO_ALERT_EMAIL_FROM')
# memcached settings. this variable should be set in the deployment to the
# same name as the service name in the docker deployment.
DOCKER_MEMCACHED_ADDR = get_from_env('DJANGO_DOCKER_MEMCACHED_ADDR',
default_value="memcached:11211",
fail_if_not_found=False)
DOCKER_MEMCACHED_SECONDS = get_from_env('DJANGO_DOCKER_MEMCACHED_SECONDS',
default_value="15",
fail_if_not_found=False)
try:
CACHE_MIDDLEWARE_SECONDS = int(DOCKER_MEMCACHED_SECONDS)
except:
CACHE_MIDDLEWARE_SECONDS = 15
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
'LOCATION': DOCKER_MEMCACHED_ADDR,
'OPTIONS': {
'ignore_exc': True,
}
},
# For API throttles
'throttles': {
'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
'LOCATION': 'api_throttle_cache', # Table name
},
}
if ENABLE_REDIS_QUEUE:
# For async alert follow-up:
CACHES.update({"async_followup": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": f"redis://{REDIS_QUEUE_ADDRESS}:{REDIS_QUEUE_PORT}/{REDIS_QUEUE_DATABASE}",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
}
}})
# Set queue backend for async django tasks:
# example django-redis connection
Q_CLUSTER = {
'name': Q_CLUSTER_NAME,
'label': Q_CLUSTER_LABEL,
'retry': REDIS_QUEUE_RETRY,
'timeout': REDIS_QUEUE_TIMEOUT,
'workers': REDIS_QUEUE_WORKERS,
'recycle': REDIS_QUEUE_RECYCLE,
'django_redis': 'async_followup'
}
MIDDLEWARE = [
'django.middleware.cache.UpdateCacheMiddleware',
'core.middleware.maintenance.MaintenanceModeMiddleware',
'events.middleware.PerformanceMiddleware',
'core.middleware.accept.AcceptMiddleware',
'core.middleware.api.ClientVersionMiddleware',
'core.middleware.api.CliExceptionMiddleware',
'django.middleware.common.CommonMiddleware',
'core.middleware.proxy.XForwardedForMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'user_sessions.middleware.SessionMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'ligoauth.middleware.ShibbolethWebAuthMiddleware',
'ligoauth.middleware.ControlRoomMiddleware',
'django.middleware.cache.FetchFromCacheMiddleware',
]
# Set up AWS X-ray patching if enabled
ENABLE_AWS_XRAY = (
get_from_env("ENABLE_AWS_XRAY",
default_value="false", fail_if_not_found=False).lower()
in ['true', 't', '1']
)
if ENABLE_AWS_XRAY:
# AWS X-ray middleware must be first in the list to measure timing
# accurately
MIDDLEWARE.insert(0, 'aws_xray_sdk.ext.django.middleware.XRayMiddleware')
# Include X-ray as an installed app in order to allow configuration beyond
# the default
INSTALLED_APPS.append('aws_xray_sdk.ext.django')
# Settings for AWS X-ray
XRAY_RECORDER = {
'AWS_XRAY_DAEMON_ADDRESS': '127.0.0.1:2000',
'AUTO_INSTRUMENT': True,
'AWS_XRAY_CONTEXT_MISSING': 'LOG_ERROR',
'PLUGINS': (),
'SAMPLING': True,
'SAMPLING_RULES': None,
'AWS_XRAY_TRACING_NAME': 'GraceDB',
'DYNAMIC_NAMING': None,
'STREAMING_THRESHOLD': None,
}
# Priority server settings ----------------------------------------------------
PRIORITY_SERVER = False
is_priority_server = get_from_env('DJANGO_PRIORITY_SERVER', None,
fail_if_not_found=False)
if (isinstance(is_priority_server, str) and
is_priority_server.lower() in ['true', 't']):
PRIORITY_SERVER = True
# If priority server, only allow priority users to the API
if PRIORITY_SERVER:
# Add custom permissions for the API
default_perms = list(REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'])
default_perms = ['api.permissions.IsPriorityUser'] + default_perms
REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'] = tuple(default_perms)
# Database settings -----------------------------------------------------------
# New postgresql database
# Configured for the CI pipeline:
# https://docs.gitlab.com/ee/ci/services/postgres.html
DATABASES = {
'default' : {
'NAME': db_name,
'ENGINE': 'django.db.backends.mysql',
'USER': os.environ.get('DJANGO_DB_USER', 'gracedb'),
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'USER': db_user,
'PASSWORD': db_password,
'HOST': os.environ.get('DJANGO_DB_HOST', ''),
'PORT': os.environ.get('DJANGO_DB_PORT', ''),
'OPTIONS': {
'init_command': 'SET storage_engine=MyISAM',
'CONN_MAX_AGE': 3600,
'TEST' : {
'NAME': 'gracedb_test_db',
},
}
},
}
# Main server "hostname" - a little hacky but OK
SERVER_HOSTNAME = SERVER_FQDN.split('.')[0]
# LVAlert Overseer settings - get from environment
LVALERT_OVERSEER_INSTANCES = [
{
"lvalert_server": lvalert_server,
"listen_port": int(lvalert_overseer_port),
"username": lvalert_user,
"password": lvalert_password,
},
]
# igwn_alert Overseer settings - get from environment
LVALERT_OVERSEER_INSTANCES = []
LVALERT_OVERSEER_INSTANCES.append(
{
"lvalert_server": igwn_alert_server,
"listen_port": int(igwn_alert_overseer_port),
"igwn_alert_group": igwn_alert_group,
"username": igwn_alert_user,
"password": igwn_alert_password,
}
)
# Pull in remaining (phone/email) alert variables from
# the environment. Default to false.
SEND_PHONE_ALERTS = parse_envvar_bool(get_from_env(
'SEND_PHONE_ALERTS',
default_value='False',
fail_if_not_found=False
))
SEND_EMAIL_ALERTS = parse_envvar_bool(get_from_env(
'SEND_EMAIL_ALERTS',
default_value='False',
fail_if_not_found=False
))
SEND_MATTERMOST_ALERTS = parse_envvar_bool(get_from_env(
'SEND_MATTERMOST_ALERTS',
default_value='False',
fail_if_not_found=False
))
INSTANCE_STUB = """
<li>Phone alerts (calls/SMS) are {0}</li>
<li>Email alerts are {1}</li>
<li><span class="text-monospace">igwn-alert</span> messages to <span class="text-monospace">{2}</span> are {3}</li>
"""
INSTANCE_LIST = INSTANCE_STUB.format(ENABLED[SEND_PHONE_ALERTS],
ENABLED[SEND_EMAIL_ALERTS],
LVALERT_OVERSEER_INSTANCES[0]['lvalert_server'],
ENABLED[SEND_XMPP_ALERTS])
# Use full client certificate to authenticate
REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] = (
'api.backends.GraceDbAuthenticatedAuthentication',
'api.backends.GraceDbSciTokenAuthentication',
'api.backends.GraceDbX509FullCertAuthentication',
'api.backends.GraceDbBasicAuthentication',
)
......@@ -94,15 +377,25 @@ if hosts_from_env is not None:
ALLOWED_HOSTS += [SERVER_FQDN]
# Email settings - dependent on server hostname and FQDN ----------------------
EMAIL_HOST = 'localhost'
SERVER_EMAIL = 'GraceDB <gracedb@{fqdn}>'.format(fqdn=SERVER_FQDN)
ALERT_EMAIL_FROM = SERVER_EMAIL
SERVER_EMAIL = ALERT_EMAIL_FROM
ALERT_EMAIL_TO = []
ALERT_EMAIL_BCC = []
ALERT_TEST_EMAIL_FROM = SERVER_EMAIL
ALERT_TEST_EMAIL_FROM = ALERT_EMAIL_FROM
ALERT_TEST_EMAIL_TO = []
# EMBB email settings
EMBB_MAIL_ADDRESS = 'embb@{fqdn}.ligo.org'.format(fqdn=SERVER_FQDN)
EMBB_SMTP_SERVER = 'localhost'
EMBB_MAIL_ADMINS = [admin[1] for admin in ADMINS]
EMBB_IGNORE_ADDRESSES = ['Mailer-Daemon@{fqdn}'.format(fqdn=SERVER_FQDN)]
# Set up logging to stdout only
for key in LOGGING['loggers']:
LOGGING['loggers'][key]['handlers'] = ['console']
LOGGING['loggers']['django.request']['handlers'].append('mail_admins')
# Turn off debug/error emails when in maintenance mode.
if MAINTENANCE_MODE:
LOGGING['loggers']['django.request']['handlers'].remove('mail_admins')
# Set SciToken accepted audience to server FQDN
SCITOKEN_AUDIENCE = ["https://" + SERVER_FQDN, "https://" + LIGO_FQDN]
config/settings/container/dev.py
View file @ 5797d777
# Settings for a test/dev GraceDB instance running in a container
#import socket
from .base import *
CONFIG_NAME = "TEST"
TIER = "dev"
CONFIG_NAME = "DEV"
# Debug settings
DEBUG = True
......@@ -29,6 +29,11 @@ INSTALLED_APPS += [
# Add testserver to ALLOWED_HOSTS
ALLOWED_HOSTS += ['testserver']
# Enforce that phone and email alerts are off XXX: Set by deployment variables!
#SEND_PHONE_ALERTS = False
#SEND_EMAIL_ALERTS = False
#SEND_MATTERMOST_ALERTS = True
# Settings for django-silk profiler
SILKY_AUTHENTICATION = True
SILKY_AUTHORISATION = True
......@@ -47,7 +52,45 @@ INTERNAL_IPS = [
INTERNAL_IP_ADDRESS,
]
# Adjust ADMINS for dev instances
ADMINS = [
("Tanner Prestegard", "tanner.prestegard@ligo.org"),
]
# Set up Sentry for error logging
sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
if sentry_dsn is not None:
USE_SENTRY = True
# Set up Sentry
import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration
sentry_sdk.init(
environment='dev',
dsn=sentry_dsn,
integrations=[DjangoIntegration()],
before_send=before_send,
)
# Turn off default admin error emails
LOGGING['loggers']['django.request']['handlers'] = []
# Home page stuff
INSTANCE_TITLE = 'GraceDB Development VM'
# Add sub-bullet with igwn-alert group:
group_sub_bullet = """<ul>
<li> Messages are sent to group: <span class="text-monospace"> {0} </span></li>
</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
INSTANCE_TITLE = 'GraceDB Development Server'
INSTANCE_INFO = """
<h5>Development Instance</h5>
<hr>
<p>
This GraceDB instance is designed for GraceDB maintainers to develop and
test in the AWS cloud architecture. There is <b>no guarantee</b> that the
behavior of this instance will mimic the production system at any time.
Events and associated data may change or be removed at any time.
</p>
<ul>
{}
<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
</ul>
""".format(INSTANCE_LIST)
config/settings/container/playground.py 0 → 100644
View file @ 5797d777
# Settings for a playground GraceDB instance (for user testing) running
# in a container on AWS. These settings inherent from base.py)
# and overrides or adds to them.
from .base import *
TIER = "playground"
CONFIG_NAME = "USER TESTING"
# Debug settings
DEBUG = False
# Override EMBB email address
# TP (8 Aug 2017): not sure why?
EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
# Enforce that phone and email alerts are off XXX: Set by deployment variables!
#SEND_PHONE_ALERTS = False
#SEND_EMAIL_ALERTS = False
# Enable Mattermost alerts
SEND_MATTERMOST_ALERTS = True
# Add testserver to ALLOWED_HOSTS
ALLOWED_HOSTS += ['testserver']
# Set up Sentry for error logging
sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
if sentry_dsn is not None:
USE_SENTRY = True
# Set up Sentry
import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration
sentry_sdk.init(
environment='playground',
dsn=sentry_dsn,
integrations=[DjangoIntegration()],
before_send=before_send,
)
# Turn off default admin error emails
LOGGING['loggers']['django.request']['handlers'] = []
# Home page stuff
INSTANCE_TITLE = 'GraceDB Playground'
# Add sub-bullet with igwn-alert group:
group_sub_bullet = """<ul>
<li> Messages are sent to group: <span class="text-monospace"> {0} </span></li>
</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
INSTANCE_INFO = """
<h5>Playground instance</h5>
<hr>
<p>
This GraceDB instance is designed for users to develop and test their own
applications. It mimics the production instance in all but the following ways:
</p>
<ul>
{}
<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
<li>Events and associated data will <b>not</b> be preserved indefinitely.
A nightly cron job removes events older than 21 days.</li>
</ul>
""".format(INSTANCE_LIST)
# Safety check on debug mode for playground
if (DEBUG == True):
raise RuntimeError("Turn off debug mode for playground")
config/settings/container/production.py
View file @ 5797d777
# Settings for a production GraceDB instance running in a container
import copy
from django.core.exceptions import ImproperlyConfigured
from .base import *
TIER = "production"
DEBUG = False
# Turn on alerts
SEND_XMPP_ALERTS = True
SEND_PHONE_ALERTS = True
SEND_EMAIL_ALERTS = True
# Priority server?
PRIORITY_SERVER = False
is_priority_server = os.environ.get('DJANGO_PRIORITY_SERVER', None)
if (isinstance(is_priority_server, str) and
is_priority_server.lower() in ['true', 't']):
PRIORITY_SERVER = True
# If priority server, do some things
if PRIORITY_SERVER:
# Add custom permissions for API
default_perms = list(REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'])
default_perms = ['api.permissions.IsPriorityUser'] + default_perms
REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'] = tuple(default_perms)
# Don't do anything to databases. Priority servers use the master
# for both read and write operations
else:
# If not a priority server, we use the read-only replica database
# for reads and master for writes.
# Get information from environment variables
replica_db_name = os.environ.get('DJANGO_REPLICA_DB_NAME', None)
if replica_db_name is None:
raise ImproperlyConfigured('Could not get replica database name from '
'envvars.')
replica_db_password = os.environ.get('DJANGO_REPLICA_DB_PASSWORD', None)
if replica_db_password is None:
raise ImproperlyConfigured('Could not get replica database password '
'from envvars.')
# Set up dict and add to DATABASES setting
read_replica = {
'NAME': replica_db_name,
'ENGINE': 'django.db.backends.mysql',
'USER': os.environ.get('DJANGO_REPLICA_DB_USER', 'gracedb'),
'PASSWORD': replica_db_password,
'HOST': os.environ.get('DJANGO_REPLICA_DB_HOST', ''),
'PORT': os.environ.get('DJANGO_REPLICA_DB_PORT', ''),
'OPTIONS': {
'init_command': 'SET storage_engine=MyISAM',
},
}
DATABASES['read_replica'] = read_replica
# Set up database router
DATABASE_ROUTERS = ['core.db.routers.NonPriorityRouter',]
# Turn on alerts: XXX: Set by deployment variables!
#SEND_PHONE_ALERTS = True
#SEND_EMAIL_ALERTS = True
#SEND_MATTERMOST_ALERTS = True
# TP, March 2019: for now, it looks infeasible to use multiple databases
# since there are many operations which normal LVC users can do that
# do a write and then a read very soon after. And we can't rely on
# the read replica being updated quickly enough for that to work.
# So there are several workflows that need to be redone in order for
# this to be possible, but it's not obvious that they even can be
# reworked properly. I.e. this is a much bigger project than expected
# so we're going to have to revisit it at some point. We'll leave the
# config here for now.
# if not PRIORITY_SERVER:
# # If not a priority server, we use the read-only replica database
# # for reads and master for writes.
# # The username, password, and database name are all replicated
# # from the production database
#
# # Set up dict and add to DATABASES setting
# read_replica = {
# 'NAME': DATABASES['default']['NAME'],
# 'ENGINE': 'django.db.backends.mysql',
# 'USER': DATABASES['default']['USER'],
# 'PASSWORD': DATABASES['default']['PASSWORD'],
# 'HOST': os.environ.get('DJANGO_REPLICA_DB_HOST', ''),
# 'PORT': os.environ.get('DJANGO_REPLICA_DB_PORT', ''),
# 'OPTIONS': {
# 'init_command': 'SET storage_engine=MyISAM',
# },
# }
# DATABASES['read_replica'] = read_replica
#
# # Set up database router
# DATABASE_ROUTERS = ['core.db.routers.NonPriorityRouter',]
# Set up Sentry for error logging
sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
if sentry_dsn is not None:
USE_SENTRY = True
# Set up Sentry
import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration
sentry_sdk.init(
environment='production',
dsn=sentry_dsn,
integrations=[DjangoIntegration()],
before_send=before_send,
)
# Turn off default admin error emails
LOGGING['loggers']['django.request']['handlers'] = []
# Home page stuff
INSTANCE_TITLE = 'GraceDB'
# Add sub-bullet with igwn-alert group:
group_sub_bullet = """<ul>
<li> Messages are sent to group: <span class="text-monospace"> {0} </span></li>
</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
INSTANCE_INFO = """
<h5>GraceDB Notifications</h5>
<hr>
<p>
GraceDB notifies registered users of Gravitational-Wave candidate detections
in real-time during LIGO/Virgo/KAGRA observation periods. Current notifications
mechanisms are:
</p>
<ul>
{}
</ul>
""".format(INSTANCE_LIST)
# Safety check on debug mode for production
if (DEBUG == True):
raise RuntimeError("Turn off debug mode for production")
# Hardcode pipelines not approved for production:
UNAPPROVED_PIPELINES += ['aframe', 'GWAK']
config/settings/container/test.py 0 → 100644
View file @ 5797d777
# Settings for a test/dev GraceDB instance running in a container
from .base import *
TIER = "test"
CONFIG_NAME = "TEST"
# Debug settings
DEBUG = True
# Override EMBB email address
# TP (8 Aug 2017): not sure why?
EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
# Add middleware
debug_middleware = 'debug_toolbar.middleware.DebugToolbarMiddleware'
MIDDLEWARE += [
debug_middleware,
#'silk.middleware.SilkyMiddleware',
#'core.middleware.profiling.ProfileMiddleware',
#'core.middleware.admin.AdminsOnlyMiddleware',
]
# Add to installed apps
INSTALLED_APPS += [
'debug_toolbar',
#'silk'
]
# Add testserver to ALLOWED_HOSTS
ALLOWED_HOSTS += ['testserver']
# Settings for django-silk profiler
SILKY_AUTHENTICATION = True
SILKY_AUTHORISATION = True
if 'silk' in INSTALLED_APPS:
# Needed to prevent RequestDataTooBig for files > 2.5 MB
# when silk is being used. This setting is typically used to
# prevent DOS attacks, so should not be changed in production.
DATA_UPLOAD_MAX_MEMORY_SIZE = 20*(1024**2)
# Tuple of IPs which are marked as internal, useful for debugging.
# Tanner (5 Dec. 2017): DON'T CHANGE THIS! Django Debug Toolbar exposes
# some headers which we want to keep hidden. So to be safe, we only allow
# it to be used through this server. You need to configure a SOCKS proxy
# on your local machine to use DJDT (see admin docs).
INTERNAL_IPS = [
INTERNAL_IP_ADDRESS,
]
# Enforce that phone and email alerts are off XXX: Set by deployment variables!
#SEND_PHONE_ALERTS = False
#SEND_EMAIL_ALERTS = False
#SEND_MATTERMOST_ALERTS = True
# Set up Sentry for error logging
sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
if sentry_dsn is not None:
USE_SENTRY = True
# Set up Sentry
import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration
sentry_sdk.init(
environment='test',
dsn=sentry_dsn,
integrations=[DjangoIntegration()],
before_send=before_send,
)
# Turn off default admin error emails
LOGGING['loggers']['django.request']['handlers'] = []
# Home page stuff
INSTANCE_TITLE = 'GraceDB Testing Server'
# Add sub-bullet with igwn-alert group:
group_sub_bullet = """<ul>
<li> Messages are sent to group: <span class="text-monospace"> {0} </span></li>
</ul>""".format(LVALERT_OVERSEER_INSTANCES[0]['igwn_alert_group'])
INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
INSTANCE_INFO = """
<h5>Testing Instance</h5>
<hr>
<p>
This GraceDB instance is designed for Quality Assurance (QA) testing and
validation for GraceDB and electromagnetic follow-up (EMFollow) developers.
Software should meet QA milestones on the test instance before being moved
to Playground or Production. Note, on this GraceDB instance:
</p>
<ul>
{}
<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
</ul>
""".format(INSTANCE_LIST)
config/settings/vm/base.py
View file @ 5797d777
......@@ -10,20 +10,20 @@ import socket
DATABASES = {
'default' : {
'NAME': 'gracedb',
'ENGINE': 'django.db.backends.mysql',
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'USER': 'gracedb',
'PASSWORD': DB_PASSWORD,
'OPTIONS': {
'init_command': 'SET storage_engine=MyISAM',
},
}
'HOST':'127.0.0.1',
'PORT':'5432',
'CONN_MAX_AGE': 3600,
},
}
# Set up allowed hosts
SERVER_FQDN = socket.getfqdn()
SERVER_HOSTNAME = INTERNAL_HOSTNAME
ALLOWED_HOSTS += [SERVER_FQDN, '{hostname}.ligo.org'.format(
hostname=SERVER_HOSTNAME)]
LIGO_FQDN = '{hostname}.ligo.org'.format(hostname=SERVER_HOSTNAME)
ALLOWED_HOSTS += [SERVER_FQDN, LIGO_FQDN]
# Email settings - dependent on server hostname and FQDN ----------------------
EMAIL_HOST = 'localhost'
......@@ -38,3 +38,148 @@ EMBB_MAIL_ADDRESS = 'embb@{fqdn}.ligo.org'.format(fqdn=SERVER_FQDN)
EMBB_SMTP_SERVER = 'localhost'
EMBB_MAIL_ADMINS = [admin[1] for admin in ADMINS]
EMBB_IGNORE_ADDRESSES = ['Mailer-Daemon@{fqdn}'.format(fqdn=SERVER_FQDN)]
# Load modified caching middleware:
# https://docs.djangoproject.com/en/2.2/ref/middleware/#middleware-ordering
MIDDLEWARE = [
'django.middleware.cache.UpdateCacheMiddleware',
'django.middleware.gzip.GZipMiddleware',
'events.middleware.PerformanceMiddleware',
'core.middleware.accept.AcceptMiddleware',
'core.middleware.api.ClientVersionMiddleware',
'core.middleware.api.CliExceptionMiddleware',
'core.middleware.proxy.XForwardedForMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'user_sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'core.middleware.maintenance.MaintenanceModeMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'ligoauth.middleware.ShibbolethWebAuthMiddleware',
'ligoauth.middleware.ControlRoomMiddleware',
'django.middleware.cache.FetchFromCacheMiddleware',
]
# Set caches:
CACHE_MIDDLEWARE_SECONDS = 5
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
'LOCATION': 'localhost:11211',
'TIMEOUT': 60,
'KEY_PREFIX': 'NULL',
'OPTIONS': {
'ignore_exc': True,
}
},
# For API throttles
'throttles': {
'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
'LOCATION': 'api_throttle_cache', # Table name
},
}
# FIXME: hardwire this for now in the VMs for testing
ENABLE_REDIS_QUEUE = True
if ENABLE_REDIS_QUEUE:
# For async alert follow-up:
CACHES.update({"async_followup": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": f"redis://{REDIS_QUEUE_ADDRESS}:{REDIS_QUEUE_PORT}/{REDIS_QUEUE_DATABASE}",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
}
}})
# Set queue backend for async django tasks:
# example django-redis connection
Q_CLUSTER = {
'name': Q_CLUSTER_NAME,
'label': Q_CLUSTER_LABEL,
'retry': REDIS_QUEUE_RETRY,
'timeout': REDIS_QUEUE_TIMEOUT,
'workers': REDIS_QUEUE_WORKERS,
'recycle': REDIS_QUEUE_RECYCLE,
'django_redis': 'async_followup'
}
# DB "cool-down" factor for when a db conflict is detected. This
# factor scales a random number of seconds between zero and one.
DB_SLEEP_FACTOR = get_from_env(
'DJANGO_DB_SLEEP_FACTOR',
default_value=1.0,
fail_if_not_found=False
)
# Fix the factor (str to float)
try:
DB_SLEEP_FACTOR = float(DB_SLEEP_FACTOR)
except:
DB_SLEEP_FACTOR = 1.0
BETA_REPORTS_LINK = True
## EGAD (External GraceDB Alert Dispatcher) configuration
ENABLE_EGAD_EMAIL = parse_envvar_bool(
get_from_env('ENABLE_EGAD_EMAIL',
fail_if_not_found=False, default_value="false")
)
ENABLE_EGAD_KAFKA = parse_envvar_bool(
get_from_env('ENABLE_EGAD_KAFKA',
fail_if_not_found=False, default_value="false")
)
ENABLE_EGAD_MATTERMOST = parse_envvar_bool(
get_from_env('ENABLE_EGAD_MATTERMOST',
fail_if_not_found=False, default_value="false")
)
ENABLE_EGAD_PHONE = parse_envvar_bool(
get_from_env('ENABLE_EGAD_PHONE',
fail_if_not_found=False, default_value="false")
)
ENABLE_EGAD = (
ENABLE_EGAD_EMAIL or ENABLE_EGAD_KAFKA
or ENABLE_EGAD_MATTERMOST or ENABLE_EGAD_PHONE
)
# Pull in remaining (phone/email) alert variables from
# the environment. Default to false.
SEND_PHONE_ALERTS = parse_envvar_bool(get_from_env(
'SEND_PHONE_ALERTS',
default_value='False',
fail_if_not_found=False
))
SEND_EMAIL_ALERTS = parse_envvar_bool(get_from_env(
'SEND_EMAIL_ALERTS',
default_value='False',
fail_if_not_found=False
))
SEND_MATTERMOST_ALERTS = parse_envvar_bool(get_from_env(
'SEND_MATTERMOST_ALERTS',
default_value='False',
fail_if_not_found=False
))
INSTANCE_STUB = """
<li>Phone alerts (calls/SMS) are {0}</li>
<li>Email alerts are {1}</li>
<li><span class="text-monospace">igwn-alert</span> messages to <span class="text-monospace">{2}</span> are {3}</li>
"""
INSTANCE_LIST = INSTANCE_STUB.format(ENABLED[SEND_PHONE_ALERTS],
ENABLED[SEND_EMAIL_ALERTS],
LVALERT_OVERSEER_INSTANCES[0]['lvalert_server'],
ENABLED[SEND_XMPP_ALERTS])
if (len(LVALERT_OVERSEER_INSTANCES) == 2):
IGWN_STUB = '<li><span class="text-monospace">igwn-alert</span> messages to <span class="text-monospace">{0}</span> are {1}</li>'
IGWN_LIST = IGWN_STUB.format(LVALERT_OVERSEER_INSTANCES[1]['lvalert_server'],
ENABLED[SEND_XMPP_ALERTS])
INSTANCE_LIST = INSTANCE_LIST + IGWN_LIST
# Set SciToken accepted audience to server FQDN
SCITOKEN_AUDIENCE = ["https://" + SERVER_FQDN, "https://" + LIGO_FQDN]
config/settings/vm/dev.py
View file @ 5797d777
......@@ -4,10 +4,13 @@
import socket
from .base import *
CONFIG_NAME = "TEST"
TIER = "dev"
CONFIG_NAME = "DEV"
# Debug settings
DEBUG = True
SEND_XMPP_ALERTS=True
SEND_MATTERMOST_ALERTS=True
# Override EMBB email address
# TP (8 Aug 2017): not sure why?
......@@ -25,7 +28,6 @@ MIDDLEWARE += [
# Add to installed apps
INSTALLED_APPS += [
'debug_toolbar',
#'silk'
]
# Add testserver to ALLOWED_HOSTS
......@@ -48,8 +50,33 @@ if 'silk' in INSTALLED_APPS:
INTERNAL_IPS = [
INTERNAL_IP_ADDRESS,
]
INSTANCE_TITLE = 'GraceDB Development VM'
# Adjust ADMINS for dev instances
ADMINS = [
("Tanner Prestegard", "tanner.prestegard@ligo.org"),
]
# Add sub-bullet with igwn-alert group:
if (len(LVALERT_OVERSEER_INSTANCES) == 2):
igwn_alert_group = os.environ.get('IGWN_ALERT_GROUP', 'lvalert-dev')
group_sub_bullet = """<ul>
<li> Messages are sent to group: <span class="text-monospace"> {0} </span></li>
</ul>""".format(igwn_alert_group)
INSTANCE_LIST = INSTANCE_LIST + group_sub_bullet
INSTANCE_INFO = """
<h5>Development Instance</h5>
<hr>
<p>
This GraceDB instance is designed for GraceDB maintainers to develop and
test in the AWS cloud architecture. There is <b>no guarantee</b> that the
behavior of this instance will mimic the production system at any time.
Events and associated data may change or be removed at any time.
</p>
<ul>
{}
<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
</ul>
""".format(INSTANCE_LIST)
# Turn off public page caching for development and testing:
PUBLIC_PAGE_CACHING = 0
# Hardcode pipelines not approved for production (for vm testing)
# UNAPPROVED_PIPELINES += ['aframe', 'GWAK']
config/settings/vm/playground.py
View file @ 5797d777
......@@ -3,6 +3,7 @@
# from base.py settings) and overrides or adds to them.
from .base import *
TIER = "playground"
CONFIG_NAME = "USER TESTING"
# Debug settings
......@@ -15,6 +16,9 @@ EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
# Turn on XMPP alerts
SEND_XMPP_ALERTS = True
# Turn on Mattermost alerts
SEND_MATTERMOST_ALERTS = True
# Enforce that phone and email alerts are off
SEND_PHONE_ALERTS = False
SEND_EMAIL_ALERTS = False
......@@ -33,18 +37,17 @@ ALLOWED_HOSTS += ['testserver']
# Home page stuff
INSTANCE_TITLE = 'GraceDB Playground'
INSTANCE_INFO = """
<h3>Playground instance</h3>
<p>
This GraceDB instance is designed for users to develop and test their own
applications. It mimics the production instance in all but the following ways:
</p>
<ul>
<li>Phone and e-mail alerts are turned off</li>
<li>Only LIGO logins are provided (no login via InCommon or Google)</li>
<li>LVAlert messages are sent to lvalert-playground.cgca.uwm.edu</li>
<li>Phone and e-mail alerts are turned off.</li>
<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
<li>LVAlert messages are sent to lvalert-playground.cgca.uwm.edu.</li>
<li>Events and associated data will <b>not</b> be preserved indefinitely.
A nightly cron job removes events older than 14 days.</li>
<li><b>Note:</b> for O3 development, the above number has been updated to
<b>112</b>.</li>
A nightly cron job removes events older than 21 days.</li>
</ul>
"""
......
config/settings/vm/production.py
View file @ 5797d777
......@@ -3,6 +3,8 @@
# settings) and overrides or adds to them.
from .base import *
TIER = "production"
DEBUG = False
# LVAlert Overseer settings
......@@ -17,6 +19,7 @@ LVALERT_OVERSEER_INSTANCES = [
SEND_XMPP_ALERTS = True
SEND_PHONE_ALERTS = True
SEND_EMAIL_ALERTS = True
SEND_MATTERMOST_ALERTS = True
# Safety check on debug mode for production
if (DEBUG == True):
......
config/urls.py
View file @ 5797d777
from django.conf import settings
from django.conf.urls import url, include
from django.urls import re_path, include
from django.contrib import admin
from django.contrib.auth.views import logout
from django.contrib.auth.views import LogoutView
from django.views.generic import TemplateView
# Import feeds
......@@ -10,7 +11,9 @@ import core.views
from events.feeds import EventFeed, feedview
import events.reports
import events.views
from ligoauth.views import pre_login, post_login, shib_logout
from ligoauth.views import (
manage_password, ShibLoginView, ShibPostLoginView
)
import search.views
# Django admin auto-discover
......@@ -22,51 +25,54 @@ feeds = {
urlpatterns = [
url(r'^$', events.views.index, name="home"),
url(r'^navbar_only$', TemplateView.as_view(
re_path(r'^$', events.views.index, name="home"),
re_path(r'^navbar_only$', TemplateView.as_view(
template_name='navbar_only.html'), name="navbar-only"),
url(r'^SPInfo', TemplateView.as_view(template_name='gracedb/spinfo.html'),
re_path(r'^SPInfo', TemplateView.as_view(template_name='gracedb/spinfo.html'),
name="spinfo"),
url(r'^SPPrivacy', TemplateView.as_view(
re_path(r'^SPPrivacy', TemplateView.as_view(
template_name='gracedb/spprivacy.html'), name="spprivacy"),
url(r'^DiscoveryService', TemplateView.as_view(
re_path(r'^DiscoveryService', TemplateView.as_view(
template_name='discovery.html'), name="discovery"),
url(r'^events/', include('events.urls')),
url(r'^superevents/', include('superevents.urls')),
url(r'^options/', include('userprofile.urls')),
url(r'^feeds/(?P<url>.*)/$', EventFeed()),
url(r'^feeds/$', feedview, name="feeds"),
url(r'^performance/$', events.views.performance, name="performance"),
url(r'^reports/$', events.reports.histo, name="reports"),
url(r'^reports/cbc_report/(?P<format>(json|flex))?$',
events.reports.cbc_report, name="cbc_report"),
url(r'^latest/$', search.views.latest, name="latest"),
re_path(r'^events/', include('events.urls')),
re_path(r'^superevents/', include('superevents.urls')),
re_path(r'^alerts/', include('alerts.urls')),
re_path(r'^feeds/(?P<url>.*)/$', EventFeed()),
re_path(r'^feeds/$', feedview, name="feeds"),
re_path(r'^other/$', TemplateView.as_view(template_name='other.html'),
name='other'),
re_path(r'^performance/$', events.views.performance, name="performance"),
re_path(r'^reports/$', events.reports.reports_page_context, name="reports"),
re_path(r'^latest/$', search.views.latest, name="latest"),
#(r'^reports/(?P<path>.+)$', 'django.views.static.serve',
# {'document_root': settings.LATENCY_REPORT_DEST_DIR}),
url(r'^search/$', search.views.search, name="mainsearch"),
re_path(r'^search/$', search.views.search, name="mainsearch"),
# Authentication
url(r'^login/$', pre_login, name='login'),
url(r'^post-login/$', post_login, name='post-login'),
url(r'^logout/$', shib_logout, name='logout'),
re_path(r'^login/$', ShibLoginView.as_view(), name='login'),
re_path(r'^post-login/$', ShibPostLoginView.as_view(), name='post-login'),
re_path(r'^logout/$', LogoutView.as_view(), name='logout'),
# Password management
re_path('^manage-password/$', manage_password, name='manage-password'),
# API URLs
url(r'^api/', include('api.urls')),
re_path(r'^api/', include('api.urls')),
# Legacy API URLs - must be maintained!
url(r'^apibasic/', include('api.urls', namespace='legacy_apibasic')),
url(r'^apiweb/', include('api.urls', namespace='legacy_apiweb')),
re_path(r'^apibasic/', include('api.urls', namespace='legacy_apibasic')),
re_path(r'^apiweb/', include('api.urls', namespace='legacy_apiweb')),
# Heartbeat URL
url(r'^heartbeat/$', core.views.heartbeat, name='heartbeat'),
re_path(r'^heartbeat/$', core.views.heartbeat, name='heartbeat'),
# Uncomment the admin/doc line below and add 'django.contrib.admindocs'
# to INSTALLED_APPS to enable admin documentation:
# (r'^admin/doc/', include('django.contrib.admindocs.urls')),
url(r'^admin/', admin.site.urls),
re_path(r'^admin/', admin.site.urls),
# sessions
#url(r'', include('user_sessions.urls', 'user_sessions')),
# Sessions
re_path(r'^', include('user_sessions.urls', 'user_sessions')),
]
......@@ -76,12 +82,12 @@ urlpatterns = [
if ('silk' in settings.INSTALLED_APPS):
# Add django-silk
urlpatterns = [
url(r'^silk/', include('silk.urls', namespace='silk'))
re_path(r'^silk/', include('silk.urls', namespace='silk'))
] + urlpatterns
# Add django-debug-toolbar
if settings.DEBUG and 'debug_toolbar' in settings.INSTALLED_APPS:
import debug_toolbar
urlpatterns = [
url(r'^__debug__/', include(debug_toolbar.urls)),
re_path(r'^__debug__/', include(debug_toolbar.urls)),
] + urlpatterns
docker/apache-config
View file @ 5797d777
......@@ -9,6 +9,9 @@ ServerName ${DJANGO_PRIMARY_FQDN}
ServerAdmin cgca-admins@uwm.edu
## Log format
LogFormat "APACHE | %a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""
## Vhost docroot
DocumentRoot "/var/www/html"
......@@ -20,6 +23,13 @@ ServerName ${DJANGO_PRIMARY_FQDN}
Require all granted
</Directory>
# Improve proxy behavior with gunicorn:
# https://serverfault.com/questions/206738/intermittent-error-when-using-mod-proxy-to-do-reverse-proxy-to-soap-service#comment1327184_209006
# https://github.com/benoitc/gunicorn/issues/207
SetEnv force-proxy-request-1.0 1
SetEnv proxy-nokeepalive 1
## Custom fragment
# gUnicorn edits
Alias /shibboleth-ds/idpselect_config.js /etc/shibboleth-ds/idpselect_config.js
......@@ -37,7 +47,33 @@ ServerName ${DJANGO_PRIMARY_FQDN}
ProxyPass "/static" "!"
ProxyPass "/documentation" "!"
ProxyPass "/admin_docs" "!"
ProxyPass "/" "http://localhost:8080/"
ProxyPass "/" "http://localhost:8080/" timeout=120
ProxyPassReverse "/" "http://localhost:8080/"
# This section is for apache2 timeout and keepalive tuning parameters.
# https://ioflood.com/blog/2020/02/21/what-is-apache-keepalive-timeout-how-to-optimize-this-critical-setting/
# KeepAlive will... keep a connection alive for subsequent requests.
# Turn this on.
KeepAlive On
# The maximum number of requests served to a client before terminating the connection.
# This can be large, possibly safely unlimited. (0 = unlimited)
MaxKeepAliveRequests 0
# The number of seconds Apache will wait for a subsequent request before closing the
# connection. Once a request has been received, the timeout value specified by the
# Timeout directive applies. Setting KeepAliveTimeout to a high value may cause
# performance problems in heavily loaded servers. The higher the timeout, the more
# server processes will be kept occupied waiting on connections with idle clients
KeepAliveTimeout 5
# Amount of time the server will wait for certain events before failing a
# request. The TimeOut directive defines the length of time Apache will wait for
# I/O (e.g., when reading data from the client, when writing data to the client, etc.)
# Default: 300s. Try setting this lower, then do a test like a long query with the API
# and in the browser and see what happens.
Timeout 60
# Unset certain headers to help prevent spoofing
RequestHeader unset REMOTE_USER
......@@ -55,6 +91,9 @@ ServerName ${DJANGO_PRIMARY_FQDN}
# Set X_FORWARDED_PROTO to https
RequestHeader set X_FORWARDED_PROTO "https"
# Increase the max allowable header size:
LimitRequestFieldSize 16384
# Set up mod_xsendfile for serving static event files as directed by Django
XSendFile On
XSendFilePath /app/db_data/
......@@ -79,7 +118,7 @@ ServerName ${DJANGO_PRIMARY_FQDN}
Require all granted
</Directory>
Alias /robots.txt /home/gracedb/gracedb_project/static_root/robots.txt
Alias /robots.txt /app/gracedb_project/static_root/robots.txt
<Location /Shibboleth.sso>
SetHandler shib
......@@ -116,7 +155,7 @@ ServerName ${DJANGO_PRIMARY_FQDN}
AuthType Shibboleth
ShibRequestSetting requireSession true
ShibUseHeaders On
Require shib-user tanner.prestegard@LIGO.ORG alexander.pace@LIGO.ORG patrick.brady@LIGO.ORG thomas.downes@LIGO.ORG
Require shib-user duncan.meacher@ligo.org alexander.pace@ligo.org daniel.wysocki@ligo.org patrick.brady@ligo.org
</Location>
</VirtualHost>