diff --git a/config/settings/base.py b/config/settings/base.py
index bde2e44cb0c783e07d4c91a91b3bc099d4579f64..33149a3386bddd516e9c1e6f2c0f2b8979769ab1 100644
--- a/config/settings/base.py
+++ b/config/settings/base.py
@@ -26,7 +26,7 @@ MAINTENANCE_MODE = False
MAINTENANCE_MODE_MESSAGE = None
# Version ---------------------------------------------------------------------
-PROJECT_VERSION = '2.8.1-2'
+PROJECT_VERSION = '2.8.3'
# Unauthenticated access ------------------------------------------------------
# This variable should eventually control whether unauthenticated access is
@@ -198,6 +198,8 @@ COINC_PIPELINES = [
GRB_PIPELINES = [
'Fermi',
'Swift',
+ 'INTEGRAL',
+ 'AGILE',
]
# VOEvent stream --------------------------------------------------------------
@@ -386,8 +388,8 @@ REST_FRAMEWORK = {
),
'DEFAULT_THROTTLE_RATES': {
'anon_burst': '300/minute',
- 'event_creation': '10/second',
- 'annotation' : '10/second',
+ 'event_creation': '25/second',
+ 'annotation' : '25/second',
},
'DEFAULT_AUTHENTICATION_CLASSES': (
'api.backends.GraceDbAuthenticatedAuthentication',
diff --git a/config/settings/container/base.py b/config/settings/container/base.py
index c679bb80cfe8439565dc5d9699c8d7ec4b8da916..97c75a62c263e164781d67e6f9498a3608370647 100644
--- a/config/settings/container/base.py
+++ b/config/settings/container/base.py
@@ -118,6 +118,7 @@ MIDDLEWARE = [
#}
+
# Priority server settings ----------------------------------------------------
PRIORITY_SERVER = False
is_priority_server = get_from_env('DJANGO_PRIORITY_SERVER', None,
@@ -152,6 +153,64 @@ DATABASES = {
}
}
+
+# Adding a fun conditional to control Amazon AWS Elasticache settings.
+# Here's the logic:
+# 1) check for the existence of the DJANGO_AWS_ELASTICACHE_ADDR address
+# variable. If it's present, then load the CACHE settings and MIDDLEWARE
+# settings required for AWS elasticache'ing
+#
+# 2) Check for the existence of DJANGO_AWS_ELASTICACHE_TIMEOUT variable. This
+# will control the cache timeout. If it's not set, default to 30s.
+#
+# 3) If not, default to old cache settings, which is effectively no-caches.
+
+try:
+ AWS_ELASTICACHE_ADDR = get_from_env('DJANGO_AWS_ELASTICACHE_ADDR')
+
+ # I *think* if the variable isn't set, then that should raise an exception
+ # and then it should skip the rest:
+
+ try:
+ AWS_ELASTICACHE_TIMEOUT = get_from_env('DJANGO_AWS_ELASTICACHE_TIMEOUT')
+ except:
+ AWS_ELASTICACHE_TIMEOUT = 30
+
+ # Load modified caching middleware:
+ MIDDLEWARE = [
+ 'core.middleware.maintenance.MaintenanceModeMiddleware',
+ 'events.middleware.PerformanceMiddleware',
+ 'core.middleware.accept.AcceptMiddleware',
+ 'core.middleware.api.ClientVersionMiddleware',
+ 'core.middleware.api.CliExceptionMiddleware',
+ 'django.middleware.cache.UpdateCacheMiddleware',
+ 'django.middleware.common.CommonMiddleware',
+ 'django.middleware.cache.FetchFromCacheMiddleware',
+ 'core.middleware.proxy.XForwardedForMiddleware',
+ 'user_sessions.middleware.SessionMiddleware',
+ 'django.contrib.messages.middleware.MessageMiddleware',
+ 'django.contrib.auth.middleware.AuthenticationMiddleware',
+ 'ligoauth.middleware.ShibbolethWebAuthMiddleware',
+ 'ligoauth.middleware.ControlRoomMiddleware',
+ ]
+
+ # Set caches:
+ CACHES = {
+ 'default': {
+ 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+ 'LOCATION': AWS_ELASTICACHE_ADDR,
+ 'TIMEOUT': AWS_ELASTICACHE_TIMEOUT,
+ 'KEY_PREFIX': 'NULL',
+ },
+ # For API throttles
+ 'throttles': {
+ 'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
+ 'LOCATION': 'api_throttle_cache', # Table name
+ },
+ }
+except:
+ pass
+
# Main server "hostname" - a little hacky but OK
SERVER_HOSTNAME = SERVER_FQDN.split('.')[0]
diff --git a/config/settings/container/dev.py b/config/settings/container/dev.py
index 79154c6fce1cbe6d862c4e9853fca2f63c63c098..b3ce05c527af22d7433b5277eaeceb3231d81e95 100644
--- a/config/settings/container/dev.py
+++ b/config/settings/container/dev.py
@@ -89,19 +89,5 @@ Events and associated data may change or be removed at any time.
</ul>
"""
-# AWS Elasticache settings:
-AWS_ELASTICACHE_ADDR = get_from_env('DJANGO_AWS_ELASTICACHE_ADDR')
-
-CACHES = {
- 'default': {
- 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
- 'LOCATION': AWS_ELASTICACHE_ADDR,
- 'TIMEOUT': 30,
- 'KEY_PREFIX': "1",
- },
- # For API throttles
- 'throttles': {
- 'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
- 'LOCATION': 'api_throttle_cache', # Table name
- },
-}
+if AWS_ELASTICACHE_ADDR:
+ CACHES['default']['KEY_PREFIX'] = '1'
diff --git a/config/settings/container/playground.py b/config/settings/container/playground.py
index 3f2714036d8b39c2c51e48c30f69c870f9bffdd1..6ff537127263032d87ca78251a7a803cd9820266 100644
--- a/config/settings/container/playground.py
+++ b/config/settings/container/playground.py
@@ -43,19 +43,6 @@ A nightly cron job removes events older than 21 days.</li>
if (DEBUG == True):
raise RuntimeError("Turn off debug mode for playground")
-# AWS Elasticache settings:
-AWS_ELASTICACHE_ADDR = get_from_env('DJANGO_AWS_ELASTICACHE_ADDR')
-
-CACHES = {
- 'default': {
- 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
- 'LOCATION': AWS_ELASTICACHE_ADDR,
- 'TIMEOUT': 30,
- 'KEY_PREFIX': '3',
- },
- # For API throttles
- 'throttles': {
- 'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
- 'LOCATION': 'api_throttle_cache', # Table name
- },
-}
+# Set elasticache prefix if the correct variables are set.
+if AWS_ELASTICACHE_ADDR:
+ CACHES['default']['KEY_PREFIX'] = '3'
diff --git a/config/settings/container/production.py b/config/settings/container/production.py
index 1e1b50e060078a4961d0542088fc0ebf7ca41502..01892e2ca5167b3f0ca3ec000a2533b32500d7fe 100644
--- a/config/settings/container/production.py
+++ b/config/settings/container/production.py
@@ -61,3 +61,8 @@ if sentry_dsn is not None:
# Safety check on debug mode for production
if (DEBUG == True):
raise RuntimeError("Turn off debug mode for production")
+
+
+# Set elasticache prefix if the correct variables are set.
+if AWS_ELASTICACHE_ADDR:
+ CACHES['default']['KEY_PREFIX'] = '4'
diff --git a/config/settings/container/test.py b/config/settings/container/test.py
index f2731b0b25130065f8887dd51b220e7431294b68..a03510bf7b4ae06cb1a480b557bdaf4cd9be675a 100644
--- a/config/settings/container/test.py
+++ b/config/settings/container/test.py
@@ -89,19 +89,5 @@ to Playground or Production. Note, on this GraceDB instance:
</ul>
"""
-# AWS Elasticache settings:
-AWS_ELASTICACHE_ADDR = get_from_env('DJANGO_AWS_ELASTICACHE_ADDR')
-
-CACHES = {
- 'default': {
- 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
- 'LOCATION': AWS_ELASTICACHE_ADDR,
- 'TIMEOUT': 30,
- 'KEY_PREFIX': '2',
- },
- # For API throttles
- 'throttles': {
- 'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
- 'LOCATION': 'api_throttle_cache', # Table name
- },
-}
+if AWS_ELASTICACHE_ADDR:
+ CACHES['default']['KEY_PREFIX'] = '2'
diff --git a/docs/user_docs/source/models.rst b/docs/user_docs/source/models.rst
index d3f279a43545a8760e34879f44f62275004c7ed3..9ca6e5852a317dd6e2131ca2f63bea90280cbe8f 100644
--- a/docs/user_docs/source/models.rst
+++ b/docs/user_docs/source/models.rst
@@ -12,7 +12,7 @@ The different types of events in GraceDB are distinguished by the following para
- ``Group``: the working group responsible for finding the candidate
- values: ``CBC``, ``Burst``, ``External``, ``Test``
- ``Pipeline``: the data analysis software tool used make the detection
- - values: ``MBTAOnline``, ``CWB``, ``CWB2G``, ``gstlal``, ``spiir``, ``HardwareInjection``, ``Fermi``, ``Swift``, ``SNEWS``, ``oLIB``
+ - values: ``MBTAOnline``, ``CWB``, ``CWB2G``, ``gstlal``, ``spiir``, ``HardwareInjection``, ``Fermi``, ``Swift``, ``INTEGRAL``, ``AGILE``, ``SNEWS``, ``oLIB``
- ``Search``: the search activity which led to the detection
- values: ``AllSky``, ``AllSkyLong``, ``LowMass``, ``HighMass``, ``GRB``, ``Supernova``, ``MDC``, ``BBH``
diff --git a/docs/user_docs/source/rest.rst b/docs/user_docs/source/rest.rst
index 3f4157fecb2a8e75c4698acee1a3b6d071b8cc5f..e2328887b61b453f941d57de2d01761504864880 100644
--- a/docs/user_docs/source/rest.rst
+++ b/docs/user_docs/source/rest.rst
@@ -43,7 +43,7 @@ Using the API programmatically
==============================
At present, the GraceDB API is not well-documented, and it may be challenging to write a custom client for accessing it.
The LSC maintains a Python package, ligo-gracedb, for simplifying access to the API.
-This package's `documentation <https://gw.readthedocs.io/ligo-gracedb>`__ is comprehensive and includes basic installation, configuration, and usage.
+This package's `documentation <https://ligo-gracedb.readthedocs.io/en/latest/>`__ is comprehensive and includes basic installation, configuration, and usage.
For non-Python users, ligo-gracedb also comes with a command-line client.
diff --git a/gracedb/alerts/xmpp.py b/gracedb/alerts/xmpp.py
index 09445b45690ee1d802422f57edbc984f5c5fec57..1e5521c0b2b2089959f162c7efcf172c1a8008b6 100644
--- a/gracedb/alerts/xmpp.py
+++ b/gracedb/alerts/xmpp.py
@@ -8,6 +8,7 @@ import sys
from django.core.mail import EmailMessage
from django.conf import settings
+from xml.sax.saxutils import escape
from core.time_utils import gpsToUtc
from events.permission_utils import is_external
@@ -92,6 +93,9 @@ def issue_xmpp_alerts(event_or_superevent, alert_type, serialized_object,
# Dump to JSON format:
# simplejson.dumps is needed to properly handle Decimal fields
msg = simplejson.dumps(lva_data)
+
+ # Try 'escaping' the message:
+ msg = escape(msg)
# Log message for debugging
logger.info("issue_xmpp_alerts: sending message {msg} for {uid}" \
diff --git a/gracedb/events/migrations/0045_add_early_warning_search.py b/gracedb/events/migrations/0045_add_early_warning_search.py
new file mode 100644
index 0000000000000000000000000000000000000000..87360f41c53d0f266f3d36e948521cc655d6a053
--- /dev/null
+++ b/gracedb/events/migrations/0045_add_early_warning_search.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2017-10-25 19:16
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# Creates initial search instances
+
+# Search names and descriptions
+SEARCH_TYPES = [
+ {'name': 'EarlyWarning', 'description': 'GstLAL Early Warning Search'},
+]
+
+def add_searches(apps, schema_editor):
+ Search = apps.get_model('events', 'Search')
+
+ # Create searches
+ for search_dict in SEARCH_TYPES:
+ search, created = Search.objects.get_or_create(name=search_dict['name'])
+ if 'description' in search_dict:
+ search.description = search_dict['description']
+ search.save()
+
+def remove_searches(apps, schema_editor):
+ Search = apps.get_model('events', 'Search')
+
+ # Delete searches
+ Search.objects.filter(name__in=[s['name'] for s in SEARCH_TYPES]).delete()
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('events', '0044_add_skymap_labels'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_searches, remove_searches),
+ ]
diff --git a/gracedb/events/migrations/0046_add_integral_agile_pipelines.py b/gracedb/events/migrations/0046_add_integral_agile_pipelines.py
new file mode 100644
index 0000000000000000000000000000000000000000..ca043f920cfb4e5335d9af4fa2e44ce534bcb6bc
--- /dev/null
+++ b/gracedb/events/migrations/0046_add_integral_agile_pipelines.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2017-10-25 19:16
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# Creates initial search pipeline instances
+
+# List of search pipeline names
+NEW_GRB_PIPELINES = [
+ 'INTEGRAL',
+ 'AGILE',
+]
+
+def add_pipelines(apps, schema_editor):
+ Pipeline = apps.get_model('events', 'Pipeline')
+
+ # Create pipelines
+ for pipeline_name in NEW_GRB_PIPELINES:
+ pipeline, created = Pipeline.objects.get_or_create(name=pipeline_name)
+
+def remove_pipelines(apps, schema_editor):
+ Pipeline = apps.get_model('events', 'Pipeline')
+
+ # Delete pipelines
+ Pipeline.objects.filter(name__in=SEARCH_PIPELINES).delete()
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('events', '0045_add_early_warning_search'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_pipelines, remove_pipelines),
+ ]
diff --git a/gracedb/events/migrations/0047_specify_integralagile_pipeline_types.py b/gracedb/events/migrations/0047_specify_integralagile_pipeline_types.py
new file mode 100644
index 0000000000000000000000000000000000000000..743e0a2b657e3c1f7d2bf036db9ad982400905bd
--- /dev/null
+++ b/gracedb/events/migrations/0047_specify_integralagile_pipeline_types.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-07-10 19:34
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+from events.models import Pipeline as pipeline_model
+
+PIPELINES = [
+ ('INTEGRAL', pipeline_model.PIPELINE_TYPE_EXTERNAL),
+ ('AGILE', pipeline_model.PIPELINE_TYPE_EXTERNAL),
+]
+
+DEFAULT_PIPELINE_TYPE = pipeline_model.PIPELINE_TYPE_OTHER
+
+def update_pipeline_types(apps, schema_editor):
+ Pipeline = apps.get_model('events', 'Pipeline')
+
+ for p_tuple in PIPELINES:
+ p = Pipeline.objects.get(name=p_tuple[0])
+ p.pipeline_type = p_tuple[1]
+ p.save(update_fields=['pipeline_type'])
+
+
+def revert_pipeline_types(apps, schema_editor):
+ Pipeline = apps.get_model('events', 'Pipeline')
+
+ for p_tuple in PIPELINES:
+ p = Pipeline.objects.get(name=p_tuple[0])
+ p.pipeline_type = DEFAULT_PIPELINE_TYPE
+ p.save(update_fields=['pipeline_type'])
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('events', '0046_add_integral_agile_pipelines'),
+ ]
+
+ operations = [
+ migrations.RunPython(update_pipeline_types, revert_pipeline_types),
+ ]
diff --git a/gracedb/events/translator.py b/gracedb/events/translator.py
index 7aa442be7e56c46f0e4f720fbf8c5a8fdebe8ac9..dfaba25be5979de0c8ce5bf380be20e98af17bd0 100644
--- a/gracedb/events/translator.py
+++ b/gracedb/events/translator.py
@@ -333,7 +333,7 @@ def handle_uploaded_data(event, datafilename,
comment=comment)
log.save()
- elif pipeline in ['Swift', 'Fermi', 'SNEWS']:
+ elif pipeline in ['Swift', 'Fermi', 'SNEWS', 'INTEGRAL','AGILE']:
# Get the event time from the VOEvent file
error = None
populateGrbEventFromVOEventFile(datafilename, event)
diff --git a/gracedb/events/view_logic.py b/gracedb/events/view_logic.py
index 5520b39d39f81678df6e1c98f442c9a3dad2c2c1..32855feca4a5fa35b9f22158275123b13c5d5336 100644
--- a/gracedb/events/view_logic.py
+++ b/gracedb/events/view_logic.py
@@ -55,7 +55,7 @@ def _createEventFromForm(request, form):
# Create Event
if pipeline.name in ['gstlal', 'spiir', 'MBTAOnline', 'pycbc',]:
event = CoincInspiralEvent()
- elif pipeline.name in ['Fermi', 'Swift', 'SNEWS']:
+ elif pipeline.name in ['Fermi', 'Swift', 'SNEWS','INTEGRAL','AGILE']:
event = GrbEvent()
elif pipeline.name in ['CWB', 'CWB2G']:
event = MultiBurstEvent()
diff --git a/gracedb/migrations/guardian/0009_populate_integralagile_uploaders.py b/gracedb/migrations/guardian/0009_populate_integralagile_uploaders.py
new file mode 100644
index 0000000000000000000000000000000000000000..9cc0841fa6b615fd962c19695665cf2f93f2c9a8
--- /dev/null
+++ b/gracedb/migrations/guardian/0009_populate_integralagile_uploaders.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2017-11-01 16:19
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# Creates UserObjectPermission objects which allow specific users
+# to add events for pipelines. Based on current production database
+# content (27 October 2017)
+
+# List of pipeline names and lists of usernames who should
+# be allowed to add events for them
+PP_LIST = [
+ {
+ 'pipeline': 'INTEGRAL',
+ 'usernames': [
+ 'brandon.piotrzkowski@LIGO.org',
+ 'alexander.urban@LIGO.ORG',
+ 'grb.exttrig',
+ 'emfollow',
+ ]
+ },
+ {
+ 'pipeline': 'AGILE',
+ 'usernames': [
+ 'brandon.piotrzkowski@LIGO.org',
+ 'alexander.urban@LIGO.ORG',
+ 'grb.exttrig',
+ 'emfollow',
+ ]
+ },
+]
+
+def add_permissions(apps, schema_editor):
+ User = apps.get_model('auth', 'User')
+ Permission = apps.get_model('auth', 'Permission')
+ UserObjectPermission = apps.get_model('guardian', 'UserObjectPermission')
+ Pipeline = apps.get_model('events', 'Pipeline')
+ ContentType = apps.get_model('contenttypes', 'ContentType')
+
+ perm = Permission.objects.get(codename='populate_pipeline')
+ ctype = ContentType.objects.get_for_model(Pipeline)
+ for pp_dict in PP_LIST:
+ pipeline = Pipeline.objects.get(name=pp_dict['pipeline'])
+
+ # Loop over users
+ for username in pp_dict['usernames']:
+
+ # Robot users should have been already created by ligoauth 0003,
+ # but we have to create human user accounts here
+ user, _ = User.objects.get_or_create(username=username)
+
+ # Create UserObjectPermission
+ uop, uop_created = UserObjectPermission.objects.get_or_create(
+ user=user, permission=perm, content_type=ctype,
+ object_pk=pipeline.id)
+
+def remove_permissions(apps, schema_editor):
+ pass
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('guardian', '0008_add_edoardo_milotti_to_cwb'),
+ ('events', '0046_add_integral_agile_pipelines'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_permissions, remove_permissions),
+ ]
diff --git a/gracedb/migrations/guardian/0010_authorize_raven_users_to_populate_new_pipelines.py b/gracedb/migrations/guardian/0010_authorize_raven_users_to_populate_new_pipelines.py
new file mode 100644
index 0000000000000000000000000000000000000000..726e2b571aa280ffc40bccb42e483e98ed9aefdd
--- /dev/null
+++ b/gracedb/migrations/guardian/0010_authorize_raven_users_to_populate_new_pipelines.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-05-08 16:27
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+GROUP_NAME = 'raven_users'
+PIPELINES = [
+ 'INTEGRAL',
+ 'AGILE',
+]
+
+
+def add_permissions(apps, schema_editor):
+ Group = apps.get_model('auth', 'Group')
+ Permission = apps.get_model('auth', 'Permission')
+ GroupObjectPermission = apps.get_model('guardian', 'GroupObjectPermission')
+ Pipeline = apps.get_model('events', 'Pipeline')
+ ContentType = apps.get_model('contenttypes', 'ContentType')
+
+ # Get group
+ group = Group.objects.get(name=GROUP_NAME)
+
+ perm = Permission.objects.get(codename='populate_pipeline')
+ ctype = ContentType.objects.get_for_model(Pipeline)
+ for pipeline in PIPELINES:
+ pipeline = Pipeline.objects.get(name=pipeline)
+
+ # Create GroupObjectPermission
+ gop = GroupObjectPermission.objects.create(group=group,
+ permission=perm, content_type=ctype, object_pk=pipeline.id)
+
+
+def remove_permissions(apps, schema_editor):
+ Group = apps.get_model('auth', 'Group')
+ Permission = apps.get_model('auth', 'Permission')
+ GroupObjectPermission = apps.get_model('guardian', 'GroupObjectPermission')
+ Pipeline = apps.get_model('events', 'Pipeline')
+ ContentType = apps.get_model('contenttypes', 'ContentType')
+
+ # Get group
+ group = Group.objects.get(name=GROUP_NAME)
+
+ perm = Permission.objects.get(codename='populate_pipeline')
+ ctype = ContentType.objects.get_for_model(Pipeline)
+ for pipeline in PIPELINES:
+ pipeline = Pipeline.objects.get(name=pipeline)
+
+ # Get GroupObjectPermission and delete
+ gop = GroupObjectPermission.objects.get(group=group,
+ permission=perm, content_type=ctype, object_pk=pipeline.id)
+ gop.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('guardian', '0009_populate_integralagile_uploaders'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_permissions, remove_permissions),
+ ]