diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9e999545b020ff98f79858993d6ea6bf87b7944..5a20caf47b9698349a20ce69cffbc7bb4342e111 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,7 @@ before_script: DJANGO_SETTINGS_MODULE: "config.settings.container.dev" DJANGO_TWILIO_ACCOUNT_SID: "fake_sid" DJANGO_TWILIO_AUTH_TOKEN: "fake_token" + DJANGO_AWS_ELASTICACHE_ADDR: "fake_server:80" LVALERT_OVERSEER_PORT: "2" LVALERT_SERVER: "fake_server" LVALERT_USER: "fake_user" diff --git a/Dockerfile b/Dockerfile index b0934dcb27a18cb325e64736b09b62aaac34ed1b..f740b09880da7d7991ad2ec2db87c64dbb2e2b0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ligo/base:stretch +FROM igwn/base:stretch LABEL name="LIGO GraceDB Django application" \ maintainer="tanner.prestegard@ligo.org" \ date="20190920" @@ -34,6 +34,7 @@ RUN apt-get update && \ libssl-dev \ swig \ htop \ + telnet \ vim && \ apt-get clean && \ npm install -g bower @@ -96,6 +97,7 @@ RUN DJANGO_SETTINGS_MODULE=${SETTINGS_MODULE} \ LVALERT_OVERSEER_PORT=2 \ DJANGO_TWILIO_ACCOUNT_SID=fake_sid \ DJANGO_TWILIO_AUTH_TOKEN=fake_token \ + DJANGO_AWS_ELASTICACHE_ADDR=fake_address:11211 \ AWS_SES_ACCESS_KEY_ID=fake_aws_id \ AWS_SES_SECRET_ACCESS_KEY=fake_aws_key \ python3 manage.py collectstatic --noinput diff --git a/config/settings/base.py b/config/settings/base.py index 948286ecf91a4987ad7c79b4ff3a52b3a2bffa25..8b1c752619451423f95b6a87bbf8959daf2439e4 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -26,7 +26,7 @@ MAINTENANCE_MODE = False MAINTENANCE_MODE_MESSAGE = None # Version --------------------------------------------------------------------- -PROJECT_VERSION = '2.8.2' +PROJECT_VERSION = '2.8.3' # Unauthenticated access ------------------------------------------------------ # This variable should eventually control whether unauthenticated access is @@ -198,6 +198,8 @@ COINC_PIPELINES = [ GRB_PIPELINES = [ 'Fermi', 'Swift', + 'INTEGRAL', + 'AGILE', ] # VOEvent stream -------------------------------------------------------------- @@ -386,8 +388,8 @@ REST_FRAMEWORK = { ), 'DEFAULT_THROTTLE_RATES': { 'anon_burst': '300/minute', - 'event_creation': '10/second', - 'annotation' : '10/second', + 'event_creation': '25/second', + 'annotation' : '25/second', }, 'DEFAULT_AUTHENTICATION_CLASSES': ( 'api.backends.GraceDbAuthenticatedAuthentication', diff --git a/config/settings/container/base.py b/config/settings/container/base.py index 7914e3b89d52c4e4bd922114e7b068d965ee3f36..9bb7f8513281cbbe99e3f73f2b3be74c207ec96d 100644 --- a/config/settings/container/base.py +++ b/config/settings/container/base.py @@ -85,6 +85,7 @@ AWS_SES_AUTO_THROTTLE = 0.25 ALERT_EMAIL_FROM = get_from_env('DJANGO_ALERT_EMAIL_FROM') + # Priority server settings ---------------------------------------------------- PRIORITY_SERVER = False is_priority_server = get_from_env('DJANGO_PRIORITY_SERVER', None, @@ -119,6 +120,64 @@ DATABASES = { } } + +# Adding a fun conditional to control Amazon AWS Elasticache settings. +# Here's the logic: +# 1) check for the existence of the DJANGO_AWS_ELASTICACHE_ADDR address +# variable. If it's present, then load the CACHE settings and MIDDLEWARE +# settings required for AWS elasticache'ing +# +# 2) Check for the existence of DJANGO_AWS_ELASTICACHE_TIMEOUT variable. This +# will control the cache timeout. If it's not set, default to 30s. +# +# 3) If not, default to old cache settings, which is effectively no-caches. + +try: + AWS_ELASTICACHE_ADDR = get_from_env('DJANGO_AWS_ELASTICACHE_ADDR') + + # I *think* if the variable isn't set, then that should raise an exception + # and then it should skip the rest: + + try: + AWS_ELASTICACHE_TIMEOUT = get_from_env('DJANGO_AWS_ELASTICACHE_TIMEOUT') + except: + AWS_ELASTICACHE_TIMEOUT = 30 + + # Load modified caching middleware: + MIDDLEWARE = [ + 'core.middleware.maintenance.MaintenanceModeMiddleware', + 'events.middleware.PerformanceMiddleware', + 'core.middleware.accept.AcceptMiddleware', + 'core.middleware.api.ClientVersionMiddleware', + 'core.middleware.api.CliExceptionMiddleware', + 'django.middleware.cache.UpdateCacheMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.cache.FetchFromCacheMiddleware', + 'core.middleware.proxy.XForwardedForMiddleware', + 'user_sessions.middleware.SessionMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'ligoauth.middleware.ShibbolethWebAuthMiddleware', + 'ligoauth.middleware.ControlRoomMiddleware', + ] + + # Set caches: + CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'LOCATION': AWS_ELASTICACHE_ADDR, + 'TIMEOUT': AWS_ELASTICACHE_TIMEOUT, + 'KEY_PREFIX': 'NULL', + }, + # For API throttles + 'throttles': { + 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', + 'LOCATION': 'api_throttle_cache', # Table name + }, + } +except: + pass + # Main server "hostname" - a little hacky but OK SERVER_HOSTNAME = SERVER_FQDN.split('.')[0] diff --git a/config/settings/container/dev.py b/config/settings/container/dev.py index e705a2932f912c2f15ad4c3b6613e096379c200d..bd18c48417888940a2ac84de4de7dfcd2cc749ab 100644 --- a/config/settings/container/dev.py +++ b/config/settings/container/dev.py @@ -87,3 +87,6 @@ Events and associated data may change or be removed at any time. <li>LVAlert messages are sent to lvalert-dev.cgca.uwm.edu.</li> </ul> """ + +if AWS_ELASTICACHE_ADDR: + CACHES['default']['KEY_PREFIX'] = '1' diff --git a/config/settings/container/playground.py b/config/settings/container/playground.py index 354f25b08c79e46cc96532669ee3f03c131f7a13..39b147e06a905aaea59d12803888bb214da90933 100644 --- a/config/settings/container/playground.py +++ b/config/settings/container/playground.py @@ -42,3 +42,7 @@ A nightly cron job removes events older than 21 days.</li> # Safety check on debug mode for playground if (DEBUG == True): raise RuntimeError("Turn off debug mode for playground") + +# Set elasticache prefix if the correct variables are set. +if AWS_ELASTICACHE_ADDR: + CACHES['default']['KEY_PREFIX'] = '3' diff --git a/config/settings/container/production.py b/config/settings/container/production.py index 16fdfa04e7f1015d8e34e60a8c120c7c1ffb552f..01892e2ca5167b3f0ca3ec000a2533b32500d7fe 100644 --- a/config/settings/container/production.py +++ b/config/settings/container/production.py @@ -2,6 +2,7 @@ from .base import * DEBUG = False +CONFIG_NAME="PROD" # Turn on alerts SEND_XMPP_ALERTS = True @@ -60,3 +61,8 @@ if sentry_dsn is not None: # Safety check on debug mode for production if (DEBUG == True): raise RuntimeError("Turn off debug mode for production") + + +# Set elasticache prefix if the correct variables are set. +if AWS_ELASTICACHE_ADDR: + CACHES['default']['KEY_PREFIX'] = '4' diff --git a/config/settings/container/test.py b/config/settings/container/test.py index 0bdbe4838d984467a3b4330a555e0255e82cf87d..934b8ebd4515256232262af8c88b083a12410f4d 100644 --- a/config/settings/container/test.py +++ b/config/settings/container/test.py @@ -87,3 +87,6 @@ to Playground or Production. Note, on this GraceDB instance: <li>LVAlert messages are sent to lvalert-test.cgca.uwm.edu.</li> </ul> """ + +if AWS_ELASTICACHE_ADDR: + CACHES['default']['KEY_PREFIX'] = '2' diff --git a/docs/user_docs/source/models.rst b/docs/user_docs/source/models.rst index d3f279a43545a8760e34879f44f62275004c7ed3..9ca6e5852a317dd6e2131ca2f63bea90280cbe8f 100644 --- a/docs/user_docs/source/models.rst +++ b/docs/user_docs/source/models.rst @@ -12,7 +12,7 @@ The different types of events in GraceDB are distinguished by the following para - ``Group``: the working group responsible for finding the candidate - values: ``CBC``, ``Burst``, ``External``, ``Test`` - ``Pipeline``: the data analysis software tool used make the detection - - values: ``MBTAOnline``, ``CWB``, ``CWB2G``, ``gstlal``, ``spiir``, ``HardwareInjection``, ``Fermi``, ``Swift``, ``SNEWS``, ``oLIB`` + - values: ``MBTAOnline``, ``CWB``, ``CWB2G``, ``gstlal``, ``spiir``, ``HardwareInjection``, ``Fermi``, ``Swift``, ``INTEGRAL``, ``AGILE``, ``SNEWS``, ``oLIB`` - ``Search``: the search activity which led to the detection - values: ``AllSky``, ``AllSkyLong``, ``LowMass``, ``HighMass``, ``GRB``, ``Supernova``, ``MDC``, ``BBH`` diff --git a/docs/user_docs/source/rest.rst b/docs/user_docs/source/rest.rst index 3f4157fecb2a8e75c4698acee1a3b6d071b8cc5f..e2328887b61b453f941d57de2d01761504864880 100644 --- a/docs/user_docs/source/rest.rst +++ b/docs/user_docs/source/rest.rst @@ -43,7 +43,7 @@ Using the API programmatically ============================== At present, the GraceDB API is not well-documented, and it may be challenging to write a custom client for accessing it. The LSC maintains a Python package, ligo-gracedb, for simplifying access to the API. -This package's `documentation <https://gw.readthedocs.io/ligo-gracedb>`__ is comprehensive and includes basic installation, configuration, and usage. +This package's `documentation <https://ligo-gracedb.readthedocs.io/en/latest/>`__ is comprehensive and includes basic installation, configuration, and usage. For non-Python users, ligo-gracedb also comes with a command-line client. diff --git a/gracedb/alerts/xmpp.py b/gracedb/alerts/xmpp.py index 09445b45690ee1d802422f57edbc984f5c5fec57..1e5521c0b2b2089959f162c7efcf172c1a8008b6 100644 --- a/gracedb/alerts/xmpp.py +++ b/gracedb/alerts/xmpp.py @@ -8,6 +8,7 @@ import sys from django.core.mail import EmailMessage from django.conf import settings +from xml.sax.saxutils import escape from core.time_utils import gpsToUtc from events.permission_utils import is_external @@ -92,6 +93,9 @@ def issue_xmpp_alerts(event_or_superevent, alert_type, serialized_object, # Dump to JSON format: # simplejson.dumps is needed to properly handle Decimal fields msg = simplejson.dumps(lva_data) + + # Try 'escaping' the message: + msg = escape(msg) # Log message for debugging logger.info("issue_xmpp_alerts: sending message {msg} for {uid}" \ diff --git a/gracedb/api/urls.py b/gracedb/api/urls.py index 14c81ff91cb5a61a7a94c2b251b5a1ce3db55113..d445b6b75613b28b3c17e816c0952ec2c5ca627e 100644 --- a/gracedb/api/urls.py +++ b/gracedb/api/urls.py @@ -5,7 +5,6 @@ from .v2 import urls as v2_urls app_name = 'api' - urlpatterns = [ url(r'^', include((v1_urls, 'default'))), url(r'^v1/', include((v1_urls, 'v1'))), diff --git a/gracedb/api/v1/events/urls.py b/gracedb/api/v1/events/urls.py index 4bb80dd44aeeeccef7c65809b13c346be89ff569..da6a750264153e7ce245e4eb7ae5fae3a7985061 100644 --- a/gracedb/api/v1/events/urls.py +++ b/gracedb/api/v1/events/urls.py @@ -1,4 +1,6 @@ from django.conf.urls import url, include +# Turn off api caching: +from django.views.decorators.cache import never_cache from .views import * @@ -6,76 +8,76 @@ from .views import * urlpatterns = [ # Event Resources # events/[{graceid}[/{version}]] - url(r'^$', EventList.as_view(), name='event-list'), - url(r'^(?P<graceid>[GEHMT]\d+)$', EventDetail.as_view(), + url(r'^$', never_cache(EventList.as_view()), name='event-list'), + url(r'^(?P<graceid>[GEHMT]\d+)$', never_cache(EventDetail.as_view()), name='event-detail'), url(r'^(?P<graceid>[GEHMT]\d+)/update-grbevent/$', - GrbEventPatchView.as_view(), name='update-grbevent'), + never_cache(GrbEventPatchView.as_view()), name='update-grbevent'), # Event Log Resources # events/{graceid}/logs/[{logid}] - url(r'^(?P<graceid>[GEHMT]\d+)/log/$', EventLogList.as_view(), + url(r'^(?P<graceid>[GEHMT]\d+)/log/$', never_cache(EventLogList.as_view()), name='eventlog-list'), url(r'^(?P<graceid>[GEHMT]\d+)/log/(?P<n>\d+)$', - EventLogDetail.as_view(), name='eventlog-detail'), + never_cache(EventLogDetail.as_view()), name='eventlog-detail'), # VOEvent Resources # events/{graceid}/voevent/[{serial_number}] - url(r'^(?P<graceid>[GEHMT]\d+)/voevent/$', VOEventList.as_view(), + url(r'^(?P<graceid>[GEHMT]\d+)/voevent/$', never_cache(VOEventList.as_view()), name='voevent-list'), url(r'^(?P<graceid>[GEHMT]\d+)/voevent/(?P<n>\d+)$', - VOEventDetail.as_view(), name='voevent-detail'), + never_cache(VOEventDetail.as_view()), name='voevent-detail'), # EMBB Resources # events/{graceid}/logs/[{logid}] - url(r'^(?P<graceid>[GEHMT]\d+)/embb/$', EMBBEventLogList.as_view(), + url(r'^(?P<graceid>[GEHMT]\d+)/embb/$', never_cache(EMBBEventLogList.as_view()), name='embbeventlog-list'), url(r'^(?P<graceid>[GEHMT]\d+)/embb/(?P<n>\d+)$', - EMBBEventLogDetail.as_view(), name='embbeventlog-detail'), + never_cache(EMBBEventLogDetail.as_view()), name='embbeventlog-detail'), url(r'^(?P<graceid>[GEHMT]\d+)/emobservation/$', - EMObservationList.as_view(), name='emobservation-list'), + never_cache(EMObservationList.as_view()), name='emobservation-list'), url(r'^(?P<graceid>[GEHMT]\d+)/emobservation/(?P<n>\d+)$', - EMObservationDetail.as_view(), name='emobservation-detail'), + never_cache(EMObservationDetail.as_view()), name='emobservation-detail'), # url(r'(?P<graceid>[GEHMT]\d+)/emobservation/(?P<n>\d+)/emfootprint/$', # EMFootprintList.as_view(), name='emfootprint-list'), # url(r'(?P<graceid>[GEHMT]\d+)/emobservation/(?P<n>\d+)/emfootprint/(?P<m>\d+)$', # EMFootprintDetail.as_view(), name='emfootprint-detail'), # Tag Resources - url(r'^(?P<graceid>[GEHMT]\d+)/tag/$', EventTagList.as_view(), + url(r'^(?P<graceid>[GEHMT]\d+)/tag/$', never_cache(EventTagList.as_view()), name='eventtag-list'), url(r'^(?P<graceid>[GEHMT]\d+)/tag/(?P<tagname>.+)$', - EventTagDetail.as_view(), name='eventtag-detail'), + never_cache(EventTagDetail.as_view()), name='eventtag-detail'), url(r'^(?P<graceid>[GEHMT]\d+)/log/(?P<n>\d+)/tag/$', - EventLogTagList.as_view(), name='eventlogtag-list'), + never_cache(EventLogTagList.as_view()), name='eventlogtag-list'), url(r'^(?P<graceid>[GEHMT]\d+)/log/(?P<n>\d+)/tag/(?P<tagname>.+)$', - EventLogTagDetail.as_view(), name='eventlogtag-detail'), + never_cache(EventLogTagDetail.as_view()), name='eventlogtag-detail'), # Permission Resources url(r'^(?P<graceid>[GEHMT]\d+)/perms/$', - EventPermissionList.as_view(), name='eventpermission-list'), + never_cache(EventPermissionList.as_view()), name='eventpermission-list'), url(r'^(?P<graceid>[GEHMT]\d+)/perms/(?P<group_name>.+)/$', - GroupEventPermissionList.as_view(), name='groupeventpermission-list'), + never_cache(GroupEventPermissionList.as_view()), name='groupeventpermission-list'), url(r'^(?P<graceid>[GEHMT]\d+)/perms/(?P<group_name>.+)/(?P<perm_shortname>\w+)$', - GroupEventPermissionDetail.as_view(), name='groupeventpermission-detail'), + never_cache(GroupEventPermissionDetail.as_view()), name='groupeventpermission-detail'), # Event File Resources # events/{graceid}/files/[{filename}[/{version}]] url(r'^(?P<graceid>\w[\d]+)/files/(?P<filename>.+)?$', - Files.as_view(), name="files"), + never_cache(Files.as_view()), name="files"), # Event Labels # events/{graceid}/labels/[{label}] url(r'^(?P<graceid>\w[\d]+)/labels/(?P<label>.+)?$', - EventLabel.as_view(), name="labels"), + never_cache(EventLabel.as_view()), name="labels"), # Event Neighbors # events/{graceid}/neighbors/[?delta=(N|(N,N))] - url(r'^(?P<graceid>\w[\d]+)/neighbors/$', EventNeighbors.as_view(), + url(r'^(?P<graceid>\w[\d]+)/neighbors/$', never_cache(EventNeighbors.as_view()), name="neighbors"), # Operator Signoff Resources url(r'^(?P<graceid>[GEHMT]\d+)/signoff/$', - OperatorSignoffList.as_view(), name='signoff-list'), + never_cache(OperatorSignoffList.as_view()), name='signoff-list'), ] diff --git a/gracedb/api/v1/superevents/urls.py b/gracedb/api/v1/superevents/urls.py index 0ba80993f289d478080f114d4acec322ea2ba9b4..6f9fc6aa11ff59465eae981fc7228414297cd28e 100644 --- a/gracedb/api/v1/superevents/urls.py +++ b/gracedb/api/v1/superevents/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import url, include +from django.views.decorators.cache import never_cache from .views import * from .settings import SUPEREVENT_LOOKUP_REGEX @@ -13,94 +14,94 @@ SUPEREVENT_DETAIL_ROOT = '(?P<{lookup_url_kwarg}>{regex})'.format( # These are included under a superevent's id URL prefix (see below) suburlpatterns = [ # Superevent detail and update - url(r'^$', SupereventViewSet.as_view({'get': 'retrieve', - 'patch': 'partial_update'}), name='superevent-detail'), + url(r'^$', never_cache(SupereventViewSet.as_view({'get': 'retrieve', + 'patch': 'partial_update'})), name='superevent-detail'), # Superevent GW confirmation - url(r'^confirm-as-gw/$', SupereventViewSet.as_view( - {'post': 'confirm_as_gw'}), name='superevent-confirm-as-gw'), + url(r'^confirm-as-gw/$', never_cache(SupereventViewSet.as_view( + {'post': 'confirm_as_gw'})), name='superevent-confirm-as-gw'), # Event list and creation (addition to superevent) - url(r'^events/$', SupereventEventViewSet.as_view({'get': 'list', - 'post': 'create'}), name='superevent-event-list'), + url(r'^events/$', never_cache(SupereventEventViewSet.as_view({'get': 'list', + 'post': 'create'})), name='superevent-event-list'), # Event detail and delete (remove from superevent) url(r'^events/(?P<{lookup_url_kwarg}>[GEHMT]\d+)/$'.format( lookup_url_kwarg=SupereventEventViewSet.lookup_url_kwarg), - SupereventEventViewSet.as_view({'get': 'retrieve', - 'delete': 'destroy'}), name='superevent-event-detail'), + never_cache(SupereventEventViewSet.as_view({'get': 'retrieve', + 'delete': 'destroy'})), name='superevent-event-detail'), # Labelling list and creation - url(r'^labels/$', SupereventLabelViewSet.as_view({'get': 'list', - 'post': 'create'}), name='superevent-label-list'), + url(r'^labels/$', never_cache(SupereventLabelViewSet.as_view({'get': 'list', + 'post': 'create'})), name='superevent-label-list'), # Labelling detail and deletion url(r'^labels/(?P<{lookup_url_kwarg}>.+)/$'.format(lookup_url_kwarg= SupereventLabelViewSet.lookup_url_kwarg), - SupereventLabelViewSet.as_view({'get': 'retrieve', - 'delete': 'destroy'}), name='superevent-label-detail'), + never_cache(SupereventLabelViewSet.as_view({'get': 'retrieve', + 'delete': 'destroy'})), name='superevent-label-detail'), # Log list and creation - url(r'^logs/$', SupereventLogViewSet.as_view({'get': 'list', - 'post': 'create'}), name='superevent-log-list'), + url(r'^logs/$', never_cache(SupereventLogViewSet.as_view({'get': 'list', + 'post': 'create'})), name='superevent-log-list'), # Log detail url(r'^logs/(?P<{lookup_url_kwarg}>\d+)/$'.format(lookup_url_kwarg= - SupereventLogViewSet.lookup_url_kwarg), SupereventLogViewSet.as_view({ - 'get': 'retrieve'}), name='superevent-log-detail'), + SupereventLogViewSet.lookup_url_kwarg), never_cache(SupereventLogViewSet.as_view({ + 'get': 'retrieve'})), name='superevent-log-detail'), # Tag list (for log) and creation (addition of tag to log) url(r'^logs/(?P<{lookup_url_kwarg}>\d+)/tags/$'.format( lookup_url_kwarg=SupereventLogViewSet.lookup_url_kwarg), - SupereventLogTagViewSet.as_view({'get': 'list', 'post': 'create'}), + never_cache(SupereventLogTagViewSet.as_view({'get': 'list', 'post': 'create'})), name='superevent-log-tag-list'), # Tag detail and deletion (removal of tag from log) url(r'^logs/(?P<{log_lookup}>\d+)/tags/(?P<{tag_lookup}>.+)/$'.format( log_lookup=SupereventLogViewSet.lookup_url_kwarg, tag_lookup= SupereventLogTagViewSet.lookup_url_kwarg), - SupereventLogTagViewSet.as_view({'get': 'retrieve', - 'delete': 'destroy'}), name='superevent-log-tag-detail'), + never_cache(SupereventLogTagViewSet.as_view({'get': 'retrieve', + 'delete': 'destroy'})), name='superevent-log-tag-detail'), # File list - url(r'^files/$', SupereventFileViewSet.as_view({'get': 'list',}), + url(r'^files/$', never_cache(SupereventFileViewSet.as_view({'get': 'list',})), name='superevent-file-list'), # File detail (download) url(r'^files/(?P<{lookup_url_kwarg}>.+)$'.format(lookup_url_kwarg= - SupereventFileViewSet.lookup_url_kwarg), SupereventFileViewSet.as_view( - {'get': 'retrieve'}), name='superevent-file-detail'), + SupereventFileViewSet.lookup_url_kwarg), never_cache(SupereventFileViewSet.as_view( + {'get': 'retrieve'})), name='superevent-file-detail'), # Note: no option for POST since file uploads should be handled # by writing a log message # VOEvent list and creation - url(r'^voevents/$', SupereventVOEventViewSet.as_view({'get': 'list', - 'post': 'create'}), name='superevent-voevent-list'), + url(r'^voevents/$', never_cache(SupereventVOEventViewSet.as_view({'get': 'list', + 'post': 'create'})), name='superevent-voevent-list'), # VOEvent detail url(r'^voevents/(?P<{lookup_url_kwarg}>\d+)/$'.format(lookup_url_kwarg= SupereventVOEventViewSet.lookup_url_kwarg), - SupereventVOEventViewSet.as_view({'get': 'retrieve'}), + never_cache(SupereventVOEventViewSet.as_view({'get': 'retrieve'})), name='superevent-voevent-detail'), # EMObservation list and creation - url(r'^emobservations/$', SupereventEMObservationViewSet.as_view( - {'get': 'list', 'post': 'create'}), + url(r'^emobservations/$', never_cache(SupereventEMObservationViewSet.as_view( + {'get': 'list', 'post': 'create'})), name='superevent-emobservation-list'), # EMObservation detail url(r'^emobservations/(?P<{lookup_url_kwarg}>\d+)/$'.format( lookup_url_kwarg=SupereventEMObservationViewSet.lookup_url_kwarg), - SupereventEMObservationViewSet.as_view({'get': 'retrieve'}), + never_cache(SupereventEMObservationViewSet.as_view({'get': 'retrieve'})), name='superevent-emobservation-detail'), # Signoff list and creation - url(r'signoffs/$', SupereventSignoffViewSet.as_view( - {'get': 'list', 'post': 'create'}), name='superevent-signoff-list'), + url(r'signoffs/$', never_cache(SupereventSignoffViewSet.as_view( + {'get': 'list', 'post': 'create'})), name='superevent-signoff-list'), # Signoff detail url(r'signoffs/(?P<{lookup_url_kwarg}>.+)/$'.format(lookup_url_kwarg= SupereventSignoffViewSet.lookup_url_kwarg), - SupereventSignoffViewSet.as_view({'get': 'retrieve', - 'patch': 'partial_update', 'delete': 'destroy'}), + never_cache(SupereventSignoffViewSet.as_view({'get': 'retrieve', + 'patch': 'partial_update', 'delete': 'destroy'})), name='superevent-signoff-detail'), # Permissions list and creation - url(r'permissions/$', SupereventGroupObjectPermissionViewSet.as_view( - {'get': 'list'}), name='superevent-permission-list'), + url(r'permissions/$', never_cache(SupereventGroupObjectPermissionViewSet.as_view( + {'get': 'list'})), name='superevent-permission-list'), # Permissions modification (expose/hide superevent). url(r'^permissions/modify/$', - SupereventGroupObjectPermissionViewSet.as_view({'post': 'modify'}), + never_cache(SupereventGroupObjectPermissionViewSet.as_view({'post': 'modify'})), name='superevent-permission-modify'), ] @@ -108,7 +109,7 @@ suburlpatterns = [ urlpatterns = [ # Superevent list and creation - url(r'^$', SupereventViewSet.as_view({'get': 'list', 'post': 'create'}), + url(r'^$', never_cache(SupereventViewSet.as_view({'get': 'list', 'post': 'create'})), name='superevent-list'), # All sub-URLs for a single superevent diff --git a/gracedb/api/v1/urls.py b/gracedb/api/v1/urls.py index b4d4597b844f95ea997cc093f83411256419d147..75abc039591e395775b0286682a6fd116bd2ed4d 100644 --- a/gracedb/api/v1/urls.py +++ b/gracedb/api/v1/urls.py @@ -8,20 +8,23 @@ from .main.views import GracedbRoot, PerformanceInfo, TagList, UserInfoView, \ from .events import urls as event_urls from .superevents import urls as superevent_urls +# Turn off api caching: +from django.views.decorators.cache import never_cache + urlpatterns = [ # Root level API resources ------------------------------------------------ # API root - url(r'^$', GracedbRoot.as_view(), name="root"), + url(r'^$', never_cache(GracedbRoot.as_view()), name="root"), # User information - url(r'^user-info/', UserInfoView.as_view(), name='user-info'), + url(r'^user-info/', never_cache(UserInfoView.as_view()), name='user-info'), # Tags - url(r'^tag/', TagList.as_view(), name='tag-list'), + url(r'^tag/', never_cache(TagList.as_view()), name='tag-list'), # Performance stats - url(r'^performance/', PerformanceInfo.as_view(), name='performance-info'), + url(r'^performance/', never_cache(PerformanceInfo.as_view()), name='performance-info'), # Certificate debugging #url(r'^cert-debug/', CertDebug.as_view(), name='cert-debug'), diff --git a/gracedb/events/migrations/0045_add_early_warning_search.py b/gracedb/events/migrations/0045_add_early_warning_search.py new file mode 100644 index 0000000000000000000000000000000000000000..87360f41c53d0f266f3d36e948521cc655d6a053 --- /dev/null +++ b/gracedb/events/migrations/0045_add_early_warning_search.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2017-10-25 19:16 +from __future__ import unicode_literals + +from django.db import migrations + +# Creates initial search instances + +# Search names and descriptions +SEARCH_TYPES = [ + {'name': 'EarlyWarning', 'description': 'GstLAL Early Warning Search'}, +] + +def add_searches(apps, schema_editor): + Search = apps.get_model('events', 'Search') + + # Create searches + for search_dict in SEARCH_TYPES: + search, created = Search.objects.get_or_create(name=search_dict['name']) + if 'description' in search_dict: + search.description = search_dict['description'] + search.save() + +def remove_searches(apps, schema_editor): + Search = apps.get_model('events', 'Search') + + # Delete searches + Search.objects.filter(name__in=[s['name'] for s in SEARCH_TYPES]).delete() + +class Migration(migrations.Migration): + + dependencies = [ + ('events', '0044_add_skymap_labels'), + ] + + operations = [ + migrations.RunPython(add_searches, remove_searches), + ] diff --git a/gracedb/events/migrations/0046_add_integral_agile_pipelines.py b/gracedb/events/migrations/0046_add_integral_agile_pipelines.py new file mode 100644 index 0000000000000000000000000000000000000000..ca043f920cfb4e5335d9af4fa2e44ce534bcb6bc --- /dev/null +++ b/gracedb/events/migrations/0046_add_integral_agile_pipelines.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2017-10-25 19:16 +from __future__ import unicode_literals + +from django.db import migrations + +# Creates initial search pipeline instances + +# List of search pipeline names +NEW_GRB_PIPELINES = [ + 'INTEGRAL', + 'AGILE', +] + +def add_pipelines(apps, schema_editor): + Pipeline = apps.get_model('events', 'Pipeline') + + # Create pipelines + for pipeline_name in NEW_GRB_PIPELINES: + pipeline, created = Pipeline.objects.get_or_create(name=pipeline_name) + +def remove_pipelines(apps, schema_editor): + Pipeline = apps.get_model('events', 'Pipeline') + + # Delete pipelines + Pipeline.objects.filter(name__in=SEARCH_PIPELINES).delete() + +class Migration(migrations.Migration): + + dependencies = [ + ('events', '0045_add_early_warning_search'), + ] + + operations = [ + migrations.RunPython(add_pipelines, remove_pipelines), + ] diff --git a/gracedb/events/migrations/0047_specify_integralagile_pipeline_types.py b/gracedb/events/migrations/0047_specify_integralagile_pipeline_types.py new file mode 100644 index 0000000000000000000000000000000000000000..743e0a2b657e3c1f7d2bf036db9ad982400905bd --- /dev/null +++ b/gracedb/events/migrations/0047_specify_integralagile_pipeline_types.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-07-10 19:34 +from __future__ import unicode_literals + +from django.db import migrations + +from events.models import Pipeline as pipeline_model + +PIPELINES = [ + ('INTEGRAL', pipeline_model.PIPELINE_TYPE_EXTERNAL), + ('AGILE', pipeline_model.PIPELINE_TYPE_EXTERNAL), +] + +DEFAULT_PIPELINE_TYPE = pipeline_model.PIPELINE_TYPE_OTHER + +def update_pipeline_types(apps, schema_editor): + Pipeline = apps.get_model('events', 'Pipeline') + + for p_tuple in PIPELINES: + p = Pipeline.objects.get(name=p_tuple[0]) + p.pipeline_type = p_tuple[1] + p.save(update_fields=['pipeline_type']) + + +def revert_pipeline_types(apps, schema_editor): + Pipeline = apps.get_model('events', 'Pipeline') + + for p_tuple in PIPELINES: + p = Pipeline.objects.get(name=p_tuple[0]) + p.pipeline_type = DEFAULT_PIPELINE_TYPE + p.save(update_fields=['pipeline_type']) + + +class Migration(migrations.Migration): + + dependencies = [ + ('events', '0046_add_integral_agile_pipelines'), + ] + + operations = [ + migrations.RunPython(update_pipeline_types, revert_pipeline_types), + ] diff --git a/gracedb/events/translator.py b/gracedb/events/translator.py index 521ddecfc4c460c95a9ea07d5ad36e4488469791..9bce5fd7ec449f50518cba282c01495202dd6ce9 100644 --- a/gracedb/events/translator.py +++ b/gracedb/events/translator.py @@ -333,7 +333,7 @@ def handle_uploaded_data(event, datafilename, comment=comment) log.save() - elif pipeline in ['Swift', 'Fermi', 'SNEWS']: + elif pipeline in ['Swift', 'Fermi', 'SNEWS', 'INTEGRAL','AGILE']: # Get the event time from the VOEvent file error = None populateGrbEventFromVOEventFile(datafilename, event) diff --git a/gracedb/events/view_logic.py b/gracedb/events/view_logic.py index 3c8ab99a09c0effc5bfb9c2b96f721267c716efc..fa4dd2d3c0994fa3cd1ab23f5337523f094bd123 100644 --- a/gracedb/events/view_logic.py +++ b/gracedb/events/view_logic.py @@ -55,7 +55,7 @@ def _createEventFromForm(request, form): # Create Event if pipeline.name in ['gstlal', 'spiir', 'MBTAOnline', 'pycbc',]: event = CoincInspiralEvent() - elif pipeline.name in ['Fermi', 'Swift', 'SNEWS']: + elif pipeline.name in ['Fermi', 'Swift', 'SNEWS','INTEGRAL','AGILE']: event = GrbEvent() elif pipeline.name in ['CWB', 'CWB2G']: event = MultiBurstEvent() diff --git a/gracedb/migrations/guardian/0009_populate_integralagile_uploaders.py b/gracedb/migrations/guardian/0009_populate_integralagile_uploaders.py new file mode 100644 index 0000000000000000000000000000000000000000..9cc0841fa6b615fd962c19695665cf2f93f2c9a8 --- /dev/null +++ b/gracedb/migrations/guardian/0009_populate_integralagile_uploaders.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2017-11-01 16:19 +from __future__ import unicode_literals + +from django.db import migrations + +# Creates UserObjectPermission objects which allow specific users +# to add events for pipelines. Based on current production database +# content (27 October 2017) + +# List of pipeline names and lists of usernames who should +# be allowed to add events for them +PP_LIST = [ + { + 'pipeline': 'INTEGRAL', + 'usernames': [ + 'brandon.piotrzkowski@LIGO.org', + 'alexander.urban@LIGO.ORG', + 'grb.exttrig', + 'emfollow', + ] + }, + { + 'pipeline': 'AGILE', + 'usernames': [ + 'brandon.piotrzkowski@LIGO.org', + 'alexander.urban@LIGO.ORG', + 'grb.exttrig', + 'emfollow', + ] + }, +] + +def add_permissions(apps, schema_editor): + User = apps.get_model('auth', 'User') + Permission = apps.get_model('auth', 'Permission') + UserObjectPermission = apps.get_model('guardian', 'UserObjectPermission') + Pipeline = apps.get_model('events', 'Pipeline') + ContentType = apps.get_model('contenttypes', 'ContentType') + + perm = Permission.objects.get(codename='populate_pipeline') + ctype = ContentType.objects.get_for_model(Pipeline) + for pp_dict in PP_LIST: + pipeline = Pipeline.objects.get(name=pp_dict['pipeline']) + + # Loop over users + for username in pp_dict['usernames']: + + # Robot users should have been already created by ligoauth 0003, + # but we have to create human user accounts here + user, _ = User.objects.get_or_create(username=username) + + # Create UserObjectPermission + uop, uop_created = UserObjectPermission.objects.get_or_create( + user=user, permission=perm, content_type=ctype, + object_pk=pipeline.id) + +def remove_permissions(apps, schema_editor): + pass + +class Migration(migrations.Migration): + + dependencies = [ + ('guardian', '0008_add_edoardo_milotti_to_cwb'), + ('events', '0046_add_integral_agile_pipelines'), + ] + + operations = [ + migrations.RunPython(add_permissions, remove_permissions), + ] diff --git a/gracedb/migrations/guardian/0010_authorize_raven_users_to_populate_new_pipelines.py b/gracedb/migrations/guardian/0010_authorize_raven_users_to_populate_new_pipelines.py new file mode 100644 index 0000000000000000000000000000000000000000..726e2b571aa280ffc40bccb42e483e98ed9aefdd --- /dev/null +++ b/gracedb/migrations/guardian/0010_authorize_raven_users_to_populate_new_pipelines.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2019-05-08 16:27 +from __future__ import unicode_literals + +from django.db import migrations + +GROUP_NAME = 'raven_users' +PIPELINES = [ + 'INTEGRAL', + 'AGILE', +] + + +def add_permissions(apps, schema_editor): + Group = apps.get_model('auth', 'Group') + Permission = apps.get_model('auth', 'Permission') + GroupObjectPermission = apps.get_model('guardian', 'GroupObjectPermission') + Pipeline = apps.get_model('events', 'Pipeline') + ContentType = apps.get_model('contenttypes', 'ContentType') + + # Get group + group = Group.objects.get(name=GROUP_NAME) + + perm = Permission.objects.get(codename='populate_pipeline') + ctype = ContentType.objects.get_for_model(Pipeline) + for pipeline in PIPELINES: + pipeline = Pipeline.objects.get(name=pipeline) + + # Create GroupObjectPermission + gop = GroupObjectPermission.objects.create(group=group, + permission=perm, content_type=ctype, object_pk=pipeline.id) + + +def remove_permissions(apps, schema_editor): + Group = apps.get_model('auth', 'Group') + Permission = apps.get_model('auth', 'Permission') + GroupObjectPermission = apps.get_model('guardian', 'GroupObjectPermission') + Pipeline = apps.get_model('events', 'Pipeline') + ContentType = apps.get_model('contenttypes', 'ContentType') + + # Get group + group = Group.objects.get(name=GROUP_NAME) + + perm = Permission.objects.get(codename='populate_pipeline') + ctype = ContentType.objects.get_for_model(Pipeline) + for pipeline in PIPELINES: + pipeline = Pipeline.objects.get(name=pipeline) + + # Get GroupObjectPermission and delete + gop = GroupObjectPermission.objects.get(group=group, + permission=perm, content_type=ctype, object_pk=pipeline.id) + gop.delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('guardian', '0009_populate_integralagile_uploaders'), + ] + + operations = [ + migrations.RunPython(add_permissions, remove_permissions), + ]