diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 46fe1652c85916d218b5aa6913a462b5f5eab18f..b9e999545b020ff98f79858993d6ea6bf87b7944 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -46,7 +46,7 @@ before_script:
# set python version
- PYTHON_VERSION="${CI_JOB_NAME##*:}"
- PYTHON_MAJOR="${PYTHON_VERSION:0:1}"
- - if [[ "${PYTHON_MAJOR}" -eq 2 ]]; then PYTHON="python"; else PYTHON="python3"; fi
+ - PYTHON="python3"
# install build requirements
- apt-get -yqq update
- apt-get -o dir::cache::archives="${APT_CACHE_DIR}" install -yqq
@@ -58,8 +58,6 @@ before_script:
libxml2-dev
swig
${PYTHON}-pip
- # install voeventlib for python2
- - if [[ "${PYTHON_MAJOR}" -eq 2 ]]; then apt-get -o dir::cache::archives="${APT_CACHE_DIR}" install -yqq python-voeventlib; fi
# install everything else from pip
- ${PYTHON} -m pip install -r requirements.txt
# create logs path required for tests
@@ -79,13 +77,9 @@ before_script:
- .cache/pip
- .cache/apt
-test:2.7:
+test:3.5:
<<: *test
-test:3.7:
- <<: *test
- allow_failure: true
-
branch_image:
stage: branch
script:
diff --git a/Dockerfile b/Dockerfile
index d349e3782bf6c9aaae046a881c03fc16ec6e0872..5ef4cf53d21e66e79360a10407a17be6f0271763 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
FROM ligo/base:stretch
LABEL name="LIGO GraceDB Django application" \
maintainer="tanner.prestegard@ligo.org" \
- date="20190430"
+ date="20190920"
ARG SETTINGS_MODULE="config.settings.container.dev"
COPY docker/SWITCHaai-swdistrib.gpg /etc/apt/trusted.gpg.d
@@ -24,11 +24,10 @@ RUN apt-get update && \
mariadb-client \
nodejs \
osg-ca-certs \
- python2.7 \
- python2.7-dev \
- python-libxml2 \
- python-pip \
- python-voeventlib \
+ python3.5 \
+ python3.5-dev \
+ python3-libxml2 \
+ python3-pip \
procps \
shibboleth \
supervisor \
@@ -61,11 +60,12 @@ ADD . /app/gracedb_project
# install gracedb application itself
WORKDIR /app/gracedb_project
RUN bower install --allow-root
-RUN pip install --upgrade setuptools wheel && \
- pip install -r requirements.txt
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools wheel && \
+ pip3 install -r requirements.txt
# Give pip-installed packages priority over distribution packages
-ENV PYTHONPATH /usr/local/lib/python2.7/dist-packages:$PYTHONPATH
+ENV PYTHONPATH /usr/local/lib/python3.5/dist-packages:$PYTHONPATH
ENV ENABLE_SHIBD false
ENV ENABLE_OVERSEER true
ENV VIRTUAL_ENV dummy
@@ -97,7 +97,7 @@ RUN DJANGO_SETTINGS_MODULE=${SETTINGS_MODULE} \
DJANGO_TWILIO_AUTH_TOKEN=fake_token \
AWS_SES_ACCESS_KEY_ID=fake_aws_id \
AWS_SES_SECRET_ACCESS_KEY=fake_aws_key \
- python manage.py collectstatic --noinput
+ python3 manage.py collectstatic --noinput
RUN rm -rf /app/logs/* /app/project_data/*
diff --git a/config/settings/base.py b/config/settings/base.py
index 855096930674372ad38d734dbb65e4f2dfcd73bf..4f26af00f91548e74f1756365ae823c1e6d3800a 100644
--- a/config/settings/base.py
+++ b/config/settings/base.py
@@ -26,7 +26,7 @@ MAINTENANCE_MODE = False
MAINTENANCE_MODE_MESSAGE = None
# Version ---------------------------------------------------------------------
-PROJECT_VERSION = '2.7.1'
+PROJECT_VERSION = '2.8.0'
# Unauthenticated access ------------------------------------------------------
# This variable should eventually control whether unauthenticated access is
@@ -51,7 +51,6 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
# MANAGERS defines who gets broken link notifications when
# BrokenLinkEmailsMiddleware is enabled
ADMINS = [
- ("Tanner Prestegard", "tanner.prestegard@ligo.org"),
("Alexander Pace", "alexander.pace@ligo.org"),
("Duncan Meacher", "duncan.meacher@ligo.org"),
]
@@ -201,11 +200,9 @@ GRB_PIPELINES = [
'Swift',
]
-# SkyAlert stuff - used for VOEvents (?) --------------------------------------
-IVORN_PREFIX = "ivo://gwnet/LVC#"
-SKYALERT_ROLE = "test"
-SKYALERT_DESCRIPTION = "Report of a candidate gravitational wave event"
-SKYALERT_SUBMITTERS = ['Patrick Brady', 'Brian Moe']
+# VOEvent stream --------------------------------------------------------------
+VOEVENT_STREAM = 'gwnet/LVC'
+
# Stuff related to report/plot generation -------------------------------------
@@ -343,6 +340,7 @@ INSTALLED_APPS = [
'django.contrib.staticfiles',
'django.contrib.messages',
'alerts',
+ 'annotations',
'api',
'core',
'events',
diff --git a/config/settings/container/dev.py b/config/settings/container/dev.py
index 65c467a37346cf34bfe04b50ec00ee01d54e78ee..e705a2932f912c2f15ad4c3b6613e096379c200d 100644
--- a/config/settings/container/dev.py
+++ b/config/settings/container/dev.py
@@ -1,7 +1,7 @@
# Settings for a test/dev GraceDB instance running in a container
from .base import *
-CONFIG_NAME = "TEST"
+CONFIG_NAME = "DEV"
# Debug settings
DEBUG = True
@@ -28,6 +28,14 @@ INSTALLED_APPS += [
# Add testserver to ALLOWED_HOSTS
ALLOWED_HOSTS += ['testserver']
+# Turn on XMPP alerts
+SEND_XMPP_ALERTS = True
+
+# Enforce that phone and email alerts are off
+SEND_PHONE_ALERTS = False
+SEND_EMAIL_ALERTS = False
+
+
# Settings for django-silk profiler
SILKY_AUTHENTICATION = True
SILKY_AUTHORISATION = True
@@ -62,3 +70,20 @@ if sentry_dsn is not None:
# Turn off default admin error emails
LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Development Server'
+INSTANCE_INFO = """
+<h3>Development Instance</h3>
+<p>
+This GraceDB instance is designed for GraceDB maintainers to develop and
+test in the AWS cloud architecture. There is <b>no guarantee</b> that the
+behavior of this instance will mimic the production system at any time.
+Events and associated data may change or be removed at any time.
+</p>
+<ul>
+<li>Phone and e-mail alerts are turned off.</li>
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>LVAlert messages are sent to lvalert-dev.cgca.uwm.edu.</li>
+</ul>
+"""
diff --git a/config/settings/container/playground.py b/config/settings/container/playground.py
new file mode 100644
index 0000000000000000000000000000000000000000..354f25b08c79e46cc96532669ee3f03c131f7a13
--- /dev/null
+++ b/config/settings/container/playground.py
@@ -0,0 +1,44 @@
+# Settings for a playground GraceDB instance (for user testing) running
+# in a container on AWS. These settings inherent from base.py)
+# and overrides or adds to them.
+from .base import *
+
+CONFIG_NAME = "USER TESTING"
+
+# Debug settings
+DEBUG = False
+
+# Override EMBB email address
+# TP (8 Aug 2017): not sure why?
+EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
+
+# Turn on XMPP alerts
+SEND_XMPP_ALERTS = True
+
+# Enforce that phone and email alerts are off
+SEND_PHONE_ALERTS = False
+SEND_EMAIL_ALERTS = False
+
+# Add testserver to ALLOWED_HOSTS
+ALLOWED_HOSTS += ['testserver']
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Playground'
+INSTANCE_INFO = """
+<h3>Playground instance</h3>
+<p>
+This GraceDB instance is designed for users to develop and test their own
+applications. It mimics the production instance in all but the following ways:
+</p>
+<ul>
+<li>Phone and e-mail alerts are turned off.</li>
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>LVAlert messages are sent to lvalert-playground.cgca.uwm.edu.</li>
+<li>Events and associated data will <b>not</b> be preserved indefinitely.
+A nightly cron job removes events older than 21 days.</li>
+</ul>
+"""
+
+# Safety check on debug mode for playground
+if (DEBUG == True):
+ raise RuntimeError("Turn off debug mode for playground")
diff --git a/config/settings/container/test.py b/config/settings/container/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..0bdbe4838d984467a3b4330a555e0255e82cf87d
--- /dev/null
+++ b/config/settings/container/test.py
@@ -0,0 +1,89 @@
+# Settings for a test/dev GraceDB instance running in a container
+from .base import *
+
+CONFIG_NAME = "TEST"
+
+# Debug settings
+DEBUG = True
+
+# Override EMBB email address
+# TP (8 Aug 2017): not sure why?
+EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
+
+# Add middleware
+debug_middleware = 'debug_toolbar.middleware.DebugToolbarMiddleware'
+MIDDLEWARE += [
+ debug_middleware,
+ #'silk.middleware.SilkyMiddleware',
+ #'core.middleware.profiling.ProfileMiddleware',
+ #'core.middleware.admin.AdminsOnlyMiddleware',
+]
+
+# Add to installed apps
+INSTALLED_APPS += [
+ 'debug_toolbar',
+ #'silk'
+]
+
+# Add testserver to ALLOWED_HOSTS
+ALLOWED_HOSTS += ['testserver']
+
+# Settings for django-silk profiler
+SILKY_AUTHENTICATION = True
+SILKY_AUTHORISATION = True
+if 'silk' in INSTALLED_APPS:
+ # Needed to prevent RequestDataTooBig for files > 2.5 MB
+ # when silk is being used. This setting is typically used to
+ # prevent DOS attacks, so should not be changed in production.
+ DATA_UPLOAD_MAX_MEMORY_SIZE = 20*(1024**2)
+
+# Tuple of IPs which are marked as internal, useful for debugging.
+# Tanner (5 Dec. 2017): DON'T CHANGE THIS! Django Debug Toolbar exposes
+# some headers which we want to keep hidden. So to be safe, we only allow
+# it to be used through this server. You need to configure a SOCKS proxy
+# on your local machine to use DJDT (see admin docs).
+INTERNAL_IPS = [
+ INTERNAL_IP_ADDRESS,
+]
+
+# Turn on XMPP alerts
+SEND_XMPP_ALERTS = True
+
+# Enforce that phone and email alerts are off
+SEND_PHONE_ALERTS = False
+SEND_EMAIL_ALERTS = False
+
+
+# Set up Sentry for error logging
+sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
+if sentry_dsn is not None:
+ USE_SENTRY = True
+
+ # Set up Sentry
+ import sentry_sdk
+ from sentry_sdk.integrations.django import DjangoIntegration
+ sentry_sdk.init(
+ environment='test',
+ dsn=sentry_dsn,
+ integrations=[DjangoIntegration()]
+ )
+
+ # Turn off default admin error emails
+ LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Testing Server'
+INSTANCE_INFO = """
+<h3>Testing Instance</h3>
+<p>
+This GraceDB instance is designed for Quality Assurance (QA) testing and
+validation for GraceDB and electromagnetic follow-up (EMFollow) developers.
+Software should meet QA milestones on the test instance before being moved
+to Playground or Production. Note, on this GraceDB instance:
+</p>
+<ul>
+<li>Phone and e-mail alerts are turned off.</li>
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>LVAlert messages are sent to lvalert-test.cgca.uwm.edu.</li>
+</ul>
+"""
diff --git a/docker/check_shibboleth_status b/docker/check_shibboleth_status
index b0ba42bde1545adffa8122058da57eed867f5316..71cfb0c0f0f3528be22dd1d4148796e4634b3e04 100644
--- a/docker/check_shibboleth_status
+++ b/docker/check_shibboleth_status
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
'''
Pulls Shibboleth status.sso page, checks for:
@@ -8,8 +8,14 @@ Run ./check_shibboleth_status -h for help.
'''
# Imports
-import argparse, urllib2, sys
+import argparse
+import sys
import xml.etree.ElementTree as ET
+try:
+ from urllib.request import urlopen
+ from urllib.error import URLError
+except ImportError: # python < 3
+ from urllib2 import (urlopen, URLError)
# Parameters - may need to be modified in the future
# if Shibboleth status pages change or new metadata
@@ -48,12 +54,12 @@ metadata_feeds = args.feeds.split(",")
# Get XML data from URL.
host_url = host + "/" + urlpath
try:
- response = urllib2.urlopen(host_url, timeout=timeout)
-except urllib2.URLError:
- print "Error opening Shibboleth status page (" + host_url + ")."
+ response = urlopen(host_url, timeout=timeout)
+except URLError:
+ print("Error opening Shibboleth status page (" + host_url + ").")
sys.exit(2)
except:
- print "Unknown error opening Shibboleth status page (" + host_url + ")."
+ print("Unknown error opening Shibboleth status page (" + host_url + ").")
sys.exit(3)
# Convert from string to ElementTree
@@ -61,11 +67,11 @@ try:
status_tree = ET.fromstring(response.read())
except ET.ParseError:
# Error parsing response.
- print "Error parsing response from server - not in XML format."
+ print("Error parsing response from server - not in XML format.")
sys.exit(2)
except:
# Error that is not ParseError.
- print "Unknown error occurred when parsing response from server."
+ print("Unknown error occurred when parsing response from server.")
sys.exit(3)
response.close()
@@ -75,12 +81,12 @@ response.close()
for tag in tags_to_check:
status_tag = status_tree.find(tag)
if (status_tag is None):
- print "Error: tag \'" + tag + "\' not found."
+ print("Error: tag \'" + tag + "\' not found.")
sys.exit(2)
else:
status_OK = status_tag.find('OK')
if (status_OK is None):
- print "Error: tag \'" + tag + "\' is not OK."
+ print("Error: tag \'" + tag + "\' is not OK.")
sys.exit(2)
# Check 2: make sure metadata feeds that we expect
@@ -90,12 +96,12 @@ srcs = [element.attrib['source'] for element in metaprov_tags]
for feed in metadata_feeds:
feed_found = [src.lower().find(feed) >= 0 for src in srcs]
if (sum(feed_found) < 1):
- print "MetadataProvider " + feed + " not found."
+ print("MetadataProvider " + feed + " not found.")
sys.exit(2)
elif (sum(feed_found) < 1):
- print "MetadataProvider " + feed + "found in multiple elements."
+ print("MetadataProvider " + feed + "found in multiple elements.")
sys.exit(2)
# If we make it to this point, everything is OK.
-print "All MetadataProviders found. Status and SessionCache are OK."
+print("All MetadataProviders found. Status and SessionCache are OK.")
sys.exit(0)
diff --git a/docker/cleanup b/docker/cleanup
index 5ccf6dc81ef6f894e315f6cc29fa92ca215ed645..13e6994f8522faafff7002f01724f538672166ed 100644
--- a/docker/cleanup
+++ b/docker/cleanup
@@ -1,5 +1,5 @@
#!/bin/sh
-python /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap people
-python /app/gracedb_project/manage.py remove_inactive_alerts
-python /app/gracedb_project/manage.py clearsessions
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap people
+python3 /app/gracedb_project/manage.py remove_inactive_alerts
+python3 /app/gracedb_project/manage.py clearsessions
diff --git a/docker/entrypoint b/docker/entrypoint
index 9d8be7681826d0029095ddf2cd16ce16fdf1b769..5e442b7f4a1ad28202827a616e529fb9e0f71bdb 100644
--- a/docker/entrypoint
+++ b/docker/entrypoint
@@ -1,4 +1,37 @@
#!/bin/bash
-export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python -c 'import uuid; print(uuid.uuid4().hex)')
+export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python3 -c 'import uuid; print(uuid.uuid4().hex)')
+
+# Change the file permissions and ownership on /app/db_data:
+chown gracedb:www-data /app/db_data
+chmod 755 /app/db_data
+
+## PGA: 2019-10-15: use certs from secrets for Shibboleth SP
+SHIB_SP_CERT=/run/secrets/saml_certificate
+SHIB_SP_KEY=/run/secrets/saml_private_key
+if [[ -f $SHIB_SP_CERT && -f $SHIB_SP_KEY ]]
+then
+ echo "Using Shibboleth Cert from docker secrets over the image one"
+ cp -f $SHIB_SP_CERT /etc/shibboleth/sp-cert.pem
+ cp -f $SHIB_SP_KEY /etc/shibboleth/sp-key.pem
+ chown _shibd:_shibd /etc/shibboleth/sp-{cert,key}.pem
+ chmod 0600 /etc/shibboleth/sp-key.pem
+fi
+
+## PGA 2019-10-16: use secrets for sensitive environment variables
+LIST="aws_ses_access_key_id
+ aws_ses_secret_access_key
+ django_db_password
+ django_secret_key
+ django_twilio_account_sid
+ django_twilio_auth_token
+ lvalert_password"
+
+for SECRET in $LIST
+do
+ VARNAME=$( tr [:lower:] [:upper:] <<<$SECRET)
+ [ -f /run/secrets/$SECRET ] && export $VARNAME="$(< /run/secrets/$SECRET)"
+done
+
exec "$@"
+
diff --git a/docs/admin_docs/source/miscellaneous.rst b/docs/admin_docs/source/miscellaneous.rst
index 35be9e290d163080b45282578ef287d0706528de..4aa34efab9d3dd12b5a0c261f07c59da7fbf3a38 100644
--- a/docs/admin_docs/source/miscellaneous.rst
+++ b/docs/admin_docs/source/miscellaneous.rst
@@ -87,12 +87,15 @@ but you still have to go through the same sequence of steps that you would
for a true developement task. I recommend the workflow described in :ref:`new_server_feature`.
In this particular case, the only necessary code change is to edit the
-file ``gracedb/events/buildVOEvent.py`` and add something like::
+file ``gracedb/annotations/voevent_utils.py`` and add something like::
- w.add_Param(Param(name="MyParam",
- dataType="float",
+ p_new = vp.Param(
+ "MyParam",
value=getMyParamForEvent(event),
- Description=["My lovely new parameter"]))
+ dataType="float"
+ )
+ p_new.Description = "My lovely new parameter"
+ v.What.append(p_new)
working by analogy with the other parameters present. I only wanted to give
this example here, because it seems likely that such a task will be considered
@@ -108,7 +111,7 @@ A good starting point is to search the GraceDB server code for "L1" to see where
Specifics (assume X1 is the IFO code):
1. Add X1OPS, X1OK, X1NO labels, update ``gracedb/templates/gracedb/event_detail_script.js`` with description, and update ``gracedb/templates/search/query_help_frag.html``
-2. Add to instruments in ``gracedb/events/buildVOEvent.py``
+2. Add to instruments in ``gracedb/annotations/voevent_utils.py``
3. Update ifoList in ``gracedb/events/query.py``
4. Add entry to ``CONTROL_ROOM_IPS`` in ``gracedb/config/settings/base.py``
5. Add signoff option for X1 in ``gracedb/templates/gracedb/event_detail.html``
diff --git a/docs/admin_docs/source/public_gracedb.rst b/docs/admin_docs/source/public_gracedb.rst
index 5b563dec99ba1533f3f9597a55e651a8d822406a..9fdcd4bffcdbfac73c0efa6d931a0b4127055823 100644
--- a/docs/admin_docs/source/public_gracedb.rst
+++ b/docs/admin_docs/source/public_gracedb.rst
@@ -114,7 +114,7 @@ example, here is how to grant ``view`` permissions to ``public``::
group_name = 'public'
perm_shortname = 'view'
- url = g.service_url + urllib.quote('events/%s/%s/%s' % (graceid, group_name, perm_codename))
+ url = g.service_url + urllib.parse.quote('events/%s/%s/%s' % (graceid, group_name, perm_codename))
r = g.put(url)
Templates
diff --git a/docs/admin_docs/source/user_permissions.rst b/docs/admin_docs/source/user_permissions.rst
index 781969a56f252a34196c6510867ca26030cf6f74..e5ddbd924dcfcdbea9d750dbf2e1a98da463a609 100644
--- a/docs/admin_docs/source/user_permissions.rst
+++ b/docs/admin_docs/source/user_permissions.rst
@@ -62,7 +62,7 @@ in real life (from inside the Django shell)::
>>> u = User.objects.get(username='albert.einstein@LIGO.ORG')
>>> if p in u.user_permissions.all():
- ...: print "Albert can add events!"
+ ...: print("Albert can add events!")
The Django ``User`` class has a convenience function ``has_perm`` to
make this easier::
@@ -72,7 +72,7 @@ make this easier::
>>> u = User.objects.get(username='albert.einstein@LIGO.ORG')
>>> if u.has_perm('events.add_event'):
- ...: print "Albert can add events!"
+ ...: print("Albert can add events!")
Again, notice that the ``has_perm`` function needs the codename to be scoped by
the app to which the model belongs. Both are required to fully specify the model.
@@ -124,7 +124,7 @@ event data. Thus, we have added a custom ``view`` permission for the event model
>>> perms = Permission.objects.filter(codename__startswith='view')
>>> for p in perms:
- ...: print p.codename
+ ...: print(p.codename)
...:
view_coincinspiralevent
view_event
@@ -314,7 +314,7 @@ can be done by adding the permission by hand::
>>> u = User.objects.get(username='albert.einstein@LIGO.ORG')
>>> u.user_permissions.add(p):
- ...: print "Albert can add events!"
+ ...: print("Albert can add events!")
Granting permission to populate a pipeline
------------------------------------------
diff --git a/docs/user_docs/source/auth.rst b/docs/user_docs/source/auth.rst
index 216f05f6fb085e64d749eb521fd3e9f69863eeb7..21d751974de7b451e5ec43c2644b4fe135693c40 100644
--- a/docs/user_docs/source/auth.rst
+++ b/docs/user_docs/source/auth.rst
@@ -75,9 +75,9 @@ Shibbolized client as follows::
try:
r = client.ping()
- except HTTPError, e:
- print e.message
+ except HTTPError as e:
+ print(e.message)
- print "Response code: %d" % r.status
- print "Response content: %s" % r.json()
+ print("Response code: %d" % r.status)
+ print("Response content: %s" % r.json())
diff --git a/docs/user_docs/source/labels.rst b/docs/user_docs/source/labels.rst
index 9d6fb257006109659244cf79e05ad7cee4024b54..3fa3ea690218a4bd5e4219e50cef15af04ec73b8 100644
--- a/docs/user_docs/source/labels.rst
+++ b/docs/user_docs/source/labels.rst
@@ -29,9 +29,9 @@ Here is a table showing the currently available labels and their meanings.
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
| EM_COINC | Signifies that a coincidence was found between gravitational-wave candidates and External triggers. |
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
-| EM_READY | Has been processed by GDB Processor. Skymaps have been produced. |
+| EM_READY | Indicates data products associated with a Superevent's preferred event are complete |
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
-| EM_Selected | GraceID automatically chosen as the most promising candidate out of a set of entries thought to correspond to the same physical event. |
+| EM_Selected | Indicates Superevent has been selected to be sent out as a public alert and freezes the preferred event from updates |
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
| EM_SENT | Has been sent to MOU partners. |
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
diff --git a/docs/user_docs/source/lvem.rst b/docs/user_docs/source/lvem.rst
index 488f6e7c6b258c907ae7ca9c40421e90b128380c..5234a94e8adfbe610cc64d4a8cf8f4b49dcb86ce 100644
--- a/docs/user_docs/source/lvem.rst
+++ b/docs/user_docs/source/lvem.rst
@@ -84,11 +84,11 @@ For example, you can use the GraceDB Python client::
try:
r = client.ping()
- except HTTPError, e:
- print e.message
-
- print "Response code: %d" % r.status
- print "Response content: %s" % r.json()
+ except HTTPError as e:
+ print(e.message)
+
+ print("Response code: %d" % r.status)
+ print("Response content: %s" % r.json())
If you're not comfortable using Python for scripted access to GraceDB, it is
also possible to use ``curl`` to directly make requests to the server with the
@@ -153,7 +153,7 @@ observation record consisting of three separate footprints::
decList, decWidthList, startTimeList, durationList, comment)
if r.status == 201: # 201 means 'Created'
- print 'Success!'
+ print('Success!')
Note that the start times are always assumed to be in UTC. For users not
familiar with Python, there are several other options available for uploading
diff --git a/docs/user_docs/source/responding_to_lvalert.rst b/docs/user_docs/source/responding_to_lvalert.rst
index a2a7db67c8dac5d0ac5a5339896c20d46cd5bc2c..764060924461a7ebb831e5c8305ea4c8944d5dfd 100644
--- a/docs/user_docs/source/responding_to_lvalert.rst
+++ b/docs/user_docs/source/responding_to_lvalert.rst
@@ -329,7 +329,7 @@ Create a Python executable ``iReact.py`` and fill it with the following::
import sys
alert = json.loads(sys.stdin.read())
- print 'uid : ' + alert['uid']
+ print('uid : ' + alert['uid'])
Don't forget to give this executable permissions with::
@@ -377,7 +377,7 @@ Open ``iReact.py`` and modify it so it reads::
from ligo.gracedb.rest import GraceDb
alert = json.loads(sys.stdin.read())
- print 'uid : ' + alert['uid']
+ print('uid : ' + alert['uid'])
gdb = GraceDb() ### instantiate a GraceDB object which connects to the default server
@@ -419,15 +419,14 @@ more example for what ``iReact.py`` might look like::
FarThr = float(sys.argv[1])
alert = json.loads(sys.stdin.read())
- print 'uid : '+alert['uid']
+ print('uid : '+alert['uid'])
gdb = GraceDb() ### instantiate a GraceDB object which connects to the default server
if alert['alert_type'] == 'new': ### the event was just created and this is the first announcment
if alert['far'] < FarThr:
- file_obj = open("iReact.txt", "w")
- print >> file_obj, "wow! this was a rare event! It had FAR = %.3e < %.3e, which was my threshold"%(alert['far'], FarThr)
- file_obj.close()
+ with open("iReact.txt", "w") as file_obj:
+ print("wow! this was a rare event! It had FAR = %.3e < %.3e, which was my threshold"%(alert['far'], FarThr), file=file_obj)
gdb.writeLog( alert['uid'], message="user.name heard an alert about this new event!", filename="iReact.txt", tagname=["data_quality"] )
Try to figure out exactly what this version does. If you can
diff --git a/docs/user_docs/source/rest.rst b/docs/user_docs/source/rest.rst
index e5a5790873d4cdba5dc1417341a06b83b0038ba3..3f4157fecb2a8e75c4698acee1a3b6d071b8cc5f 100644
--- a/docs/user_docs/source/rest.rst
+++ b/docs/user_docs/source/rest.rst
@@ -78,10 +78,10 @@ If you have reason to believe that your request may be throttled, you can wrap i
try:
r = gracedb.writeLog(graceid, "Hello, this is a log message.")
success = True
- except HTTPError, e:
+ except HTTPError as e:
try:
rdict = json.loads(e.message)
- if 'retry-after' in rdict.keys():
+ if 'retry-after' in rdict:
time.sleep(int(rdict['retry-after']))
continue
else:
diff --git a/gracedb/alerts/migrations/0004_auto_20190919_1957.py b/gracedb/alerts/migrations/0004_auto_20190919_1957.py
new file mode 100644
index 0000000000000000000000000000000000000000..9dfbdf114a3d0d78ad64fa572e07d6797e08ffeb
--- /dev/null
+++ b/gracedb/alerts/migrations/0004_auto_20190919_1957.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-09-19 19:57
+# This was auto-generated after moving to Python 3, with no changes to the
+# actual models. See the commit message for more details.
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('alerts', '0003_add_created_updated_time_fields_to_notification'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='contact',
+ name='phone_method',
+ field=models.CharField(blank=True, choices=[('C', 'Call'), ('T', 'Text'), ('B', 'Call and text')], default=None, max_length=1, null=True),
+ ),
+ migrations.AlterField(
+ model_name='notification',
+ name='category',
+ field=models.CharField(choices=[('E', 'Event'), ('S', 'Superevent')], default='S', max_length=1),
+ ),
+ ]
diff --git a/gracedb/alerts/models.py b/gracedb/alerts/models.py
index a21988b752c9ad5b0238218b730ac6f574da4304..ab4bc8858f8ad80c939ae97611899aa618bdd98d 100644
--- a/gracedb/alerts/models.py
+++ b/gracedb/alerts/models.py
@@ -8,7 +8,8 @@ from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
from django.core.mail import EmailMessage
from django.db import models
-from django.utils import timezone
+from django.utils import six, timezone
+from django.utils.encoding import python_2_unicode_compatible
from django.utils.http import urlencode
from django_twilio.client import twilio_client
@@ -55,7 +56,6 @@ class Contact(CleanSaveModel):
updated = models.DateTimeField(auto_now=True)
verified_time = models.DateTimeField(null=True, blank=True, editable=False)
-
def __str__(self):
return "{0}: {1}".format(self.user.username, self.description)
@@ -168,6 +168,7 @@ class Contact(CleanSaveModel):
###############################################################################
# Notifications ###############################################################
###############################################################################
+@python_2_unicode_compatible
class Notification(models.Model):
# Notification categories
NOTIFICATION_CATEGORY_EVENT = 'E'
@@ -196,10 +197,12 @@ class Notification(models.Model):
pipelines = models.ManyToManyField('events.pipeline', blank=True)
searches = models.ManyToManyField('events.search', blank=True)
- def __unicode__(self):
- return (u"%s: %s") % (
- self.user.username,
- self.display()
+ def __str__(self):
+ return six.text_type(
+ "{username}: {display}".format(
+ username=self.user.username,
+ display=self.display()
+ )
)
def display(self):
diff --git a/gracedb/alerts/recipients.py b/gracedb/alerts/recipients.py
index 43f9356fe2055ce4d3a7a9351b7366a961464dc9..18f6b70811547fc2c5b06b252fdee13e59a34fbe 100644
--- a/gracedb/alerts/recipients.py
+++ b/gracedb/alerts/recipients.py
@@ -1,3 +1,8 @@
+try:
+ from functools import reduce
+except ImportError: # python < 3
+ pass
+
from django.conf import settings
from django.db.models import Q
diff --git a/gracedb/alerts/tests/test_access.py b/gracedb/alerts/tests/test_access.py
index cbd068b1f788227ec91cb188314980cf4e93e019..fb6da47674572c8621cf5f5546e578ebf7e788dc 100644
--- a/gracedb/alerts/tests/test_access.py
+++ b/gracedb/alerts/tests/test_access.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.contrib.auth.models import Group as AuthGroup
diff --git a/gracedb/alerts/tests/test_email.py b/gracedb/alerts/tests/test_email.py
index ea2a655592c709cca68cf512022656feb189febb..23831750598d224acae59c88aaa141f8bf7ffbea 100644
--- a/gracedb/alerts/tests/test_email.py
+++ b/gracedb/alerts/tests/test_email.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.test import override_settings
diff --git a/gracedb/alerts/tests/test_phone.py b/gracedb/alerts/tests/test_phone.py
index 16366477608165ced9b1767a78eb0b66c9557215..96b458e4d2c7ca1c886b3ea344074129eae4d8a6 100644
--- a/gracedb/alerts/tests/test_phone.py
+++ b/gracedb/alerts/tests/test_phone.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.test import override_settings
diff --git a/gracedb/alerts/tests/test_views.py b/gracedb/alerts/tests/test_views.py
index aeb6e689756a45432ff9f571a3d2ab1f31387606..488734d29b6d11b8e1d0df34e854a4862987d74e 100644
--- a/gracedb/alerts/tests/test_views.py
+++ b/gracedb/alerts/tests/test_views.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
import pytest
from django.conf import settings
diff --git a/gracedb/alerts/views.py b/gracedb/alerts/views.py
index 0ea11cce0e7225e540eb5543c8106af8d5609a5d..df94588a203516dcf52b331748f96cd34d3cd353 100644
--- a/gracedb/alerts/views.py
+++ b/gracedb/alerts/views.py
@@ -65,8 +65,7 @@ class CreateNotificationView(MultipleFormView):
return kw
def form_valid(self, form):
- if form.cleaned_data.has_key('key_field'):
- form.cleaned_data.pop('key_field')
+ form.cleaned_data.pop('key_field', None)
# Add user (from request) and category (stored on form class) to
# the form instance, then save
@@ -169,8 +168,7 @@ class CreateContactView(MultipleFormView):
def form_valid(self, form):
# Remove key_field, add user, and save form
- if form.cleaned_data.has_key('key_field'):
- form.cleaned_data.pop('key_field')
+ form.cleaned_data.pop('key_field', None)
form.instance.user = self.request.user
form.save()
diff --git a/gracedb/alerts/xmpp.py b/gracedb/alerts/xmpp.py
index 59355090ef69dc09dfba8c4dbcf74c05d1b59567..09445b45690ee1d802422f57edbc984f5c5fec57 100644
--- a/gracedb/alerts/xmpp.py
+++ b/gracedb/alerts/xmpp.py
@@ -108,7 +108,7 @@ def issue_xmpp_alerts(event_or_superevent, alert_type, serialized_object,
for node_name in node_names:
# Calculate unique message_id and log
- message_id = sha1(node_name + msg).hexdigest()
+ message_id = sha1((node_name + msg).encode()).hexdigest()
# Log message
logger.info(("issue_xmpp_alerts: sending alert type {alert_type} "
diff --git a/gracedb/annotations/__init__.py b/gracedb/annotations/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..9541c0b39194fabdee665a13bdc6bf3b17635145
--- /dev/null
+++ b/gracedb/annotations/voevent_utils.py
@@ -0,0 +1,725 @@
+# See the VOEvent specification for details
+# http://www.ivoa.net/Documents/latest/VOEvent.html
+
+import datetime
+import logging
+import os
+
+from scipy.constants import c, G, pi
+import voeventparse as vp
+
+from django.conf import settings
+from django.urls import reverse
+
+from core.time_utils import gpsToUtc
+from core.urls import build_absolute_uri
+from events.models import VOEventBase, Event
+from events.models import CoincInspiralEvent, MultiBurstEvent, \
+ LalInferenceBurstEvent
+from superevents.shortcuts import is_superevent
+
+# Set up logger
+logger = logging.getLogger(__name__)
+
+
+###############################################################################
+# SETUP #######################################################################
+###############################################################################
+# Dict of VOEvent type abbreviations and full strings
+VOEVENT_TYPE_DICT = dict(VOEventBase.VOEVENT_TYPE_CHOICES)
+
+
+# Used to create the Packet_Type parameter block
+PACKET_TYPES = {
+ VOEventBase.VOEVENT_TYPE_PRELIMINARY: (150, 'LVC_PRELIMINARY'),
+ VOEventBase.VOEVENT_TYPE_INITIAL: (151, 'LVC_INITIAL'),
+ VOEventBase.VOEVENT_TYPE_UPDATE: (152, 'LVC_UPDATE'),
+ VOEventBase.VOEVENT_TYPE_RETRACTION: (164, 'LVC_RETRACTION'),
+}
+
+
+# Description strings
+DEFAULT_DESCRIPTION = \
+ "Candidate gravitational wave event identified by low-latency analysis"
+INSTRUMENT_DESCRIPTIONS = {
+ "H1": "H1: LIGO Hanford 4 km gravitational wave detector",
+ "L1": "L1: LIGO Livingston 4 km gravitational wave detector",
+ "V1": "V1: Virgo 3 km gravitational wave detector",
+}
+
+
+###############################################################################
+# MAIN ########################################################################
+###############################################################################
+def construct_voevent_file(obj, voevent, request=None):
+
+ # Setup ###################################################################
+ ## Determine event or superevent
+ obj_is_superevent = False
+ if is_superevent(obj):
+ obj_is_superevent = True
+ event = obj.preferred_event
+ graceid = obj.default_superevent_id
+ obj_view_name = "superevents:view"
+ fits_view_name = "api:default:superevents:superevent-file-detail"
+ else:
+ event = obj
+ graceid = obj.graceid
+ obj_view_name = "view"
+ fits_view_name = "api:default:events:files"
+
+ # Get the event subclass (CoincInspiralEvent, MultiBurstEvent, etc.) and
+ # set that as the event
+ event = event.get_subclass_or_self()
+
+ ## Let's convert that voevent_type to something nicer looking
+ voevent_type = VOEVENT_TYPE_DICT[voevent.voevent_type]
+
+ ## Now build the IVORN.
+ type_string = voevent_type.capitalize()
+ voevent_id = '{gid}-{N}-{type_str}'.format(type_str=type_string,
+ gid=graceid, N=voevent.N)
+
+ ## Determine role
+ if event.is_mdc() or event.is_test():
+ role = vp.definitions.roles.test
+ else:
+ role = vp.definitions.roles.observation
+
+ ## Instantiate VOEvent
+ v = vp.Voevent(settings.VOEVENT_STREAM, voevent_id, role)
+
+ ## Set root Description
+ if voevent_type != 'retraction':
+ v.Description = "Report of a candidate gravitational wave event"
+
+ # Who #####################################################################
+ ## Remove Who.Description
+ v.Who.remove(v.Who.Description)
+
+ ## Set Who.Date
+ vp.set_who(
+ v,
+ date=datetime.datetime.utcnow()
+ )
+
+ ## Set Who.Author
+ vp.set_author(
+ v,
+ contactName="LIGO Scientific Collaboration and Virgo Collaboration"
+ )
+
+ # How #####################################################################
+ if voevent_type != 'retraction':
+ descriptions = [DEFAULT_DESCRIPTION]
+
+ # Add instrument descriptions
+ instruments = event.instruments.split(',')
+ for inst in INSTRUMENT_DESCRIPTIONS:
+ if inst in instruments:
+ descriptions.append(INSTRUMENT_DESCRIPTIONS[inst])
+ if voevent.coinc_comment:
+ descriptions.append("A gravitational wave trigger identified a "
+ "possible counterpart GRB")
+ vp.add_how(v, descriptions=descriptions)
+
+ # What ####################################################################
+ # UCD = Unified Content Descriptors
+ # http://monet.uni-sw.gwdg.de/twiki/bin/view/VOEvent/UnifiedContentDescriptors
+ # OR -- (from VOTable document, [21] below)
+ # http://www.ivoa.net/twiki/bin/view/IVOA/IvoaUCD
+ # http://cds.u-strasbg.fr/doc/UCD.htx
+ #
+ # which somehow gets you to:
+ # http://www.ivoa.net/Documents/REC/UCD/UCDlist-20070402.html
+ # where you might find some actual information.
+
+ # Unit / Section 4.3 of [21] which relies on [25]
+ # [21] http://www.ivoa.net/Documents/latest/VOT.html
+ # [25] http://vizier.u-strasbg.fr/doc/catstd-3.2.htx
+ #
+ # Basically, a string that makes sense to humans about what units a value
+ # is. eg. "m/s"
+
+ ## Packet_Type param
+ p_packet_type = vp.Param(
+ "Packet_Type",
+ value=PACKET_TYPES[voevent.voevent_type][0],
+ ac=True
+ )
+ p_packet_type.Description = ("The Notice Type number is assigned/used "
+ "within GCN, eg type={typenum} is an {typedesc} notice").format(
+ typenum=PACKET_TYPES[voevent.voevent_type][0],
+ typedesc=PACKET_TYPES[voevent.voevent_type][1]
+ )
+ v.What.append(p_packet_type)
+
+ # Internal param
+ p_internal = vp.Param(
+ "internal",
+ value=int(voevent.internal),
+ ac=True
+ )
+ p_internal.Description = ("Indicates whether this event should be "
+ "distributed to LSC/Virgo members only")
+ v.What.append(p_internal)
+
+ ## Packet serial number
+ p_serial_num = vp.Param(
+ "Pkt_Ser_Num",
+ value=voevent.N,
+ ac=True
+ )
+ p_serial_num.Description = ("A number that increments by 1 each time a "
+ "new revision is issued for this event")
+ v.What.append(p_serial_num)
+
+ ## Event graceid or superevent ID
+ p_gid = vp.Param(
+ "GraceID",
+ value=graceid,
+ ucd="meta.id",
+ dataType="string"
+ )
+ p_gid.Description = "Identifier in GraceDB"
+ v.What.append(p_gid)
+
+ ## Alert type parameter
+ p_alert_type = vp.Param(
+ "AlertType",
+ value = voevent_type.capitalize(),
+ ucd="meta.version",
+ dataType="string"
+ )
+ p_alert_type.Description = "VOEvent alert type"
+ v.What.append(p_alert_type)
+
+ ## Whether the event is a hardware injection or not
+ p_hardware_inj = vp.Param(
+ "HardwareInj",
+ value=int(voevent.hardware_inj),
+ ucd="meta.number",
+ ac=True
+ )
+ p_hardware_inj.Description = ("Indicates that this event is a hardware "
+ "injection if 1, no if 0")
+ v.What.append(p_hardware_inj)
+
+ ## Open alert parameter
+ p_open_alert = vp.Param(
+ "OpenAlert",
+ value=int(voevent.open_alert),
+ ucd="meta.number",
+ ac=True
+ )
+ p_open_alert.Description = ("Indicates that this event is an open alert "
+ "if 1, no if 0")
+ v.What.append(p_open_alert)
+
+ ## Superevent page
+ p_detail_url = vp.Param(
+ "EventPage",
+ value=build_absolute_uri(
+ reverse(obj_view_name, args=[graceid]),
+ request
+ ),
+ ucd="meta.ref.url",
+ dataType="string"
+ )
+ p_detail_url.Description = ("Web page for evolving status of this GW "
+ "candidate")
+ v.What.append(p_detail_url)
+
+ ## Only for non-retractions
+ if voevent_type != 'retraction':
+ ## Instruments
+ p_instruments = vp.Param(
+ "Instruments",
+ value=event.instruments,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_instruments.Description = ("List of instruments used in analysis to "
+ "identify this event")
+ v.What.append(p_instruments)
+
+ ## False alarm rate
+ if event.far:
+ p_far = vp.Param(
+ "FAR",
+ value=float(max(event.far, settings.VOEVENT_FAR_FLOOR)),
+ ucd="arith.rate;stat.falsealarm",
+ unit="Hz",
+ ac=True
+ )
+ p_far.Description = ("False alarm rate for GW candidates with "
+ "this strength or greater")
+ v.What.append(p_far)
+
+ ## Analysis group
+ p_group = vp.Param(
+ "Group",
+ value=event.group.name,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_group.Description = "Data analysis working group"
+ v.What.append(p_group)
+
+ ## Analysis pipeline
+ p_pipeline = vp.Param(
+ "Pipeline",
+ value=event.pipeline.name,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_pipeline.Description = "Low-latency data analysis pipeline"
+ v.What.append(p_pipeline)
+
+ ## Search type
+ if event.search:
+ p_search = vp.Param(
+ "Search",
+ value=event.search.name,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_search.Description = "Specific low-latency search"
+ v.What.append(p_search)
+
+ ## RAVEN specific entries
+ if (is_superevent(obj) and voevent.raven_coinc):
+ ext_id = obj.em_type
+ ext_event = Event.getByGraceid(ext_id)
+ emcoinc_params = []
+
+ ## External GCN ID
+ if ext_event.trigger_id:
+ p_extid = vp.Param(
+ "External_GCN_Notice_Id",
+ value=ext_event.trigger_id,
+ ucd="meta.id",
+ dataType="string"
+ )
+ p_extid.Description = ("GCN trigger ID of external event")
+ emcoinc_params.append(p_extid)
+
+ ## External IVORN
+ if ext_event.ivorn:
+ p_extivorn = vp.Param(
+ "External_Ivorn",
+ value=ext_event.ivorn,
+ ucd="meta.id",
+ dataType="string"
+ )
+ p_extivorn.Description = ("IVORN of external event")
+ emcoinc_params.append(p_extivorn)
+
+ ## External Pipeline
+ if ext_event.pipeline:
+ p_extpipeline = vp.Param(
+ "External_Observatory",
+ value=ext_event.pipeline.name,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_extpipeline.Description = ("External Observatory")
+ emcoinc_params.append(p_extpipeline)
+
+ ## External Search
+ if ext_event.search:
+ p_extsearch = vp.Param(
+ "External_Search",
+ value=ext_event.search.name,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_extsearch.Description = ("External astrophysical search")
+ emcoinc_params.append(p_extsearch)
+
+ ## Time Difference
+ if ext_event.gpstime and obj.t_0:
+ deltat = round(ext_event.gpstime - obj.t_0, 2)
+ p_deltat = vp.Param(
+ "Time_Difference",
+ value=float(deltat),
+ ucd="meta.code",
+ #dataType="float"
+ #AEP--> figure this out
+ ac=True,
+ )
+ p_deltat.Description = ("Time difference between GW candidate "
+ "and external event, centered on the "
+ "GW candidate")
+ emcoinc_params.append(p_deltat)
+
+ ## Temporal Coinc FAR
+ if obj.coinc_far:
+ p_coincfar = vp.Param(
+ "Time_Coincidence_FAR",
+ value=obj.coinc_far,
+ ucd="arith.rate;stat.falsealarm",
+ #dataType="float",
+ #AEP--> figure this out
+ ac=True,
+ unit="Hz"
+ )
+ p_coincfar.Description = ("Estimated coincidence false alarm "
+ "rate in Hz using timing")
+ emcoinc_params.append(p_coincfar)
+
+ ## Spatial-Temporal Coinc FAR
+ ## FIXME: Find a way to supply this value
+ if False:
+ p_coincfar_space = vp.Param(
+ "Time_Sky_Position_Coincidence_FAR",
+ value=obj.coinc_far_space,
+ ucd="arith.rate;stat.falsealarm",
+ #dataType="float",
+ #AEP--> figure this out
+ ac=True,
+ unit="Hz"
+ )
+ p_coincfar_space.Description = ("Estimated coincidence false alarm "
+ "rate in Hz using timing and sky "
+ "position")
+ emcoinc_params.append(p_coincfar_space)
+
+ ## RAVEN combined sky map
+ if voevent.combined_skymap_filename:
+ ## Skymap group
+ ### fits skymap URL
+ fits_skymap_url_comb = build_absolute_uri(
+ reverse(fits_view_name, args=[graceid,
+ voevent.combined_skymap_filename]),
+ request
+ )
+ p_fits_url_comb = vp.Param(
+ "joint_skymap_fits",
+ value=fits_skymap_url_comb,
+ ucd="meta.ref.url",
+ dataType="string"
+ )
+ p_fits_url_comb.Description = "Combined GW-External Sky Map FITS"
+ emcoinc_params.append(p_fits_url_comb)
+
+ ## Create EMCOINC group
+ emcoinc_group = vp.Group(
+ emcoinc_params,
+ name='External Coincidence',
+ type='External Coincidence' # keep this only for backwards compatibility
+ )
+ emcoinc_group.Description = \
+ ("Properties of joint coincidence found by RAVEN")
+ v.What.append(emcoinc_group)
+
+ # initial and update VOEvents must have a skymap.
+ # new feature (10/24/2016): preliminary VOEvents can have a skymap,
+ # but they don't have to.
+ if (voevent_type in ["initial", "update"] or
+ (voevent_type == "preliminary" and voevent.skymap_filename != None)):
+
+ ## Skymap group
+ ### fits skymap URL
+ fits_skymap_url = build_absolute_uri(
+ reverse(fits_view_name, args=[graceid, voevent.skymap_filename]),
+ request
+ )
+ p_fits_url = vp.Param(
+ "skymap_fits",
+ value=fits_skymap_url,
+ ucd="meta.ref.url",
+ dataType="string"
+ )
+ p_fits_url.Description = "Sky Map FITS"
+
+ ### Create skymap group with params
+ skymap_group = vp.Group(
+ [p_fits_url],
+ name="GW_SKYMAP",
+ type="GW_SKYMAP",
+ )
+
+ ### Add to What
+ v.What.append(skymap_group)
+
+ ## Analysis specific attributes
+ if voevent_type != 'retraction':
+ ### Classification group (EM-Bright params; CBC only)
+ em_bright_params = []
+ source_properties_params = []
+ if (isinstance(event, CoincInspiralEvent) and
+ voevent_type != 'retraction'):
+
+ # EM-Bright mass classifier information for CBC event candidates
+ if voevent.prob_bns is not None:
+ p_pbns = vp.Param(
+ "BNS",
+ value=voevent.prob_bns,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pbns.Description = \
+ ("Probability that the source is a binary neutron star "
+ "merger (both objects lighter than 3 solar masses)")
+ em_bright_params.append(p_pbns)
+
+ if voevent.prob_nsbh is not None:
+ p_pnsbh = vp.Param(
+ "NSBH",
+ value=voevent.prob_nsbh,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pnsbh.Description = \
+ ("Probability that the source is a neutron star-black "
+ "hole merger (primary heavier than 5 solar masses, "
+ "secondary lighter than 3 solar masses)")
+ em_bright_params.append(p_pnsbh)
+
+ if voevent.prob_bbh is not None:
+ p_pbbh = vp.Param(
+ "BBH",
+ value=voevent.prob_bbh,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pbbh.Description = ("Probability that the source is a "
+ "binary black hole merger (both objects "
+ "heavier than 5 solar masses)")
+ em_bright_params.append(p_pbbh)
+
+ if voevent.prob_mass_gap is not None:
+ p_pmassgap = vp.Param(
+ "MassGap",
+ value=voevent.prob_mass_gap,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pmassgap.Description = ("Probability that the source has at "
+ "least one object between 3 and 5 "
+ "solar masses")
+ em_bright_params.append(p_pmassgap)
+
+ if voevent.prob_terrestrial is not None:
+ p_pterr = vp.Param(
+ "Terrestrial",
+ value=voevent.prob_terrestrial,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pterr.Description = ("Probability that the source is "
+ "terrestrial (i.e., a background noise "
+ "fluctuation or a glitch)")
+ em_bright_params.append(p_pterr)
+
+ # Add to source properties group
+ if voevent.prob_has_ns is not None:
+ p_phasns = vp.Param(
+ name="HasNS",
+ value=voevent.prob_has_ns,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_phasns.Description = ("Probability that at least one object "
+ "in the binary has a mass that is "
+ "less than 3 solar masses")
+ source_properties_params.append(p_phasns)
+
+ if voevent.prob_has_remnant is not None:
+ p_phasremnant = vp.Param(
+ "HasRemnant",
+ value=voevent.prob_has_remnant,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_phasremnant.Description = ("Probability that a nonzero mass "
+ "was ejected outside the central "
+ "remnant object")
+ source_properties_params.append(p_phasremnant)
+
+ elif isinstance(event, MultiBurstEvent):
+ ### Central frequency
+ p_central_freq = vp.Param(
+ "CentralFreq",
+ value=float(event.central_freq),
+ ucd="gw.frequency",
+ unit="Hz",
+ dataType="float"
+ )
+ p_central_freq.Description = \
+ "Central frequency of GW burst signal"
+ v.What.append(p_central_freq)
+
+ ### Duration
+ p_duration = vp.Param(
+ "Duration",
+ value=float(event.duration),
+ unit="s",
+ ucd="time.duration",
+ dataType="float"
+ )
+ p_duration.Description = "Measured duration of GW burst signal"
+ v.What.append(p_duration)
+
+ # XXX Calculate the fluence. Unfortunately, this requires parsing the trigger.txt
+ # file for hrss values. These should probably be pulled into the database.
+ # But there is no consensus on whether hrss or fluence is meaningful. So I will
+ # put off changing the schema for now.
+ try:
+ # Go find the data file.
+ log = event.eventlog_set.filter(comment__startswith="Original Data").all()[0]
+ filename = log.filename
+ filepath = os.path.join(event.datadir,filename)
+ if os.path.isfile(filepath):
+ datafile = open(filepath,"r")
+ else:
+ raise VOEventBase.VOEventBuilderException(
+ "No file found.")
+ # Now parse the datafile.
+ # The line we want looks like:
+ # hrss: 1.752741e-23 2.101590e-23 6.418900e-23
+ for line in datafile:
+ if line.startswith('hrss:'):
+ hrss_values = [float(hrss) for hrss in line.split()[1:]]
+ max_hrss = max(hrss_values)
+ # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
+ # Note that hrss here actually has units of s^(-1/2)
+ fluence = pi * pow(c,3) * pow(event.central_freq,2)
+ fluence = fluence * pow(max_hrss,2)
+ fluence = fluence / (4.0*G)
+
+ p_fluence = vp.Param(
+ "Fluence",
+ value=fluence,
+ ucd="gw.fluence",
+ unit="erg/cm^2",
+ ac=True,
+ )
+ p_fluence.Description = "Estimated fluence of GW burst signal"
+ v.What.append(p_fluence)
+ except Exception as e:
+ logger.exception(e)
+
+ elif isinstance(event, LalInferenceBurstEvent):
+ p_freq = vp.Param(
+ "frequency",
+ value=float(event.frequency_mean),
+ ucd="gw.frequency",
+ unit="Hz",
+ dataType="float"
+ )
+ p_freq.Description = "Mean frequency of GW burst signal"
+ v.What.append(p_freq)
+
+ # Calculate the fluence.
+ # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
+ # Note that hrss here actually has units of s^(-1/2)
+ # XXX obviously need to refactor here.
+ try:
+ fluence = pi * pow(c,3) * pow(event.frequency,2)
+ fluence = fluence * pow(event.hrss,2)
+ fluence = fluence / (4.0*G)
+
+ p_fluence = vp.Param(
+ "Fluence",
+ value=fluence,
+ ucd="gw.fluence",
+ unit="erg/cm^2",
+ ac=True
+ )
+ p_fluence.Description = "Estimated fluence of GW burst signal"
+ v.What.append(p_fluence)
+ except Exception as e:
+ logger.exception(e)
+
+ ## Create classification group
+ classification_group = vp.Group(
+ em_bright_params,
+ name='Classification',
+ type='Classification' # keep this only for backwards compatibility
+ )
+ classification_group.Description = \
+ ("Source classification: binary neutron star (BNS), neutron star-"
+ "black hole (NSBH), binary black hole (BBH), MassGap, or "
+ "terrestrial (noise)")
+ v.What.append(classification_group)
+
+ ## Create properties group
+ properties_group = vp.Group(
+ source_properties_params,
+ name='Properties',
+ type='Properties' # keep this only for backwards compatibility
+ )
+ properties_group.Description = \
+ ("Qualitative properties of the source, conditioned on the "
+ "assumption that the signal is an astrophysical compact binary "
+ "merger")
+ v.What.append(properties_group)
+
+ # WhereWhen ###############################################################
+ # NOTE: we use a fake ra, dec, err, and units for creating the coords
+ # object. We are required to provide them by the voeventparse code, but
+ # our "format" for VOEvents didn't have a Position2D entry. So to make
+ # the code work but maintain the same format, we add fake information here,
+ # then remove it later.
+ coords = vp.Position2D(
+ ra=1, dec=2, err=3, units='degrees',
+ system=vp.definitions.sky_coord_system.utc_fk5_geo
+ )
+ observatory_id = 'LIGO Virgo'
+ vp.add_where_when(
+ v,
+ coords,
+ gpsToUtc(event.gpstime),
+ observatory_id
+ )
+ # NOTE: now remove position 2D so the fake ra, dec, err, and units
+ # don't show up.
+ ol = v.WhereWhen.ObsDataLocation.ObservationLocation
+ ol.AstroCoords.remove(ol.AstroCoords.Position2D)
+
+ # Citations ###############################################################
+ if obj.voevent_set.count() > 1:
+
+ ## Loop over previous VOEvents for this event or superevent and
+ ## add them to citations
+ event_ivorns_list = []
+ for ve in obj.voevent_set.all():
+ # Oh, actually we need to exclude *this* voevent.
+ if ve.N == voevent.N:
+ continue
+
+ # Get cite type
+ if voevent_type == 'retraction':
+ cite_type = vp.definitions.cite_types.retraction
+ else:
+ cite_type = vp.definitions.cite_types.supersedes
+
+ # Set up event ivorn
+ ei = vp.EventIvorn(ve.ivorn, cite_type)
+
+ # Add event ivorn
+ event_ivorns_list.append(ei)
+
+ # Add citations
+ vp.add_citations(
+ v,
+ event_ivorns_list
+ )
+ # Get description for citation
+ desc = None
+ if voevent_type == 'preliminary':
+ desc = 'Initial localization is now available (preliminary)'
+ elif voevent_type == 'initial':
+ desc = 'Initial localization is now available'
+ elif voevent_type == 'update':
+ desc = 'Updated localization is now available'
+ elif voevent_type == 'retraction':
+ desc = 'Determined to not be a viable GW event candidate'
+ if desc is not None:
+ v.Citations.Description = desc
+
+ # Return the document as a string, along with the IVORN ###################
+ xml = vp.dumps(v, pretty_print=True)
+ return xml, v.get('ivorn')
diff --git a/gracedb/api/backends.py b/gracedb/api/backends.py
index 9274ae56d5ee099e8ca1cdb7915a66d67e396ff9..3766ecdb5a9cbb34bb91121e610dc706a2d2c445 100644
--- a/gracedb/api/backends.py
+++ b/gracedb/api/backends.py
@@ -298,19 +298,22 @@ class GraceDbX509FullCertAuthentication(GraceDbX509Authentication):
@staticmethod
def get_certificate_subject_string(certificate):
subject = certificate.get_subject()
+ subject_decoded = [[word.decode("utf8") for word in sets]
+ for sets in subject.get_components()]
subject_string = '/' + "/".join(["=".join(c) for c in
- subject.get_components()])
+ subject_decoded])
return subject_string
@staticmethod
def get_certificate_issuer_string(certificate):
issuer = certificate.get_issuer()
+ issuer_decoded = [[word.decode("utf8") for word in sets]
+ for sets in issuer.get_components()]
issuer_string = '/' + "/".join(["=".join(c) for c in
- issuer.get_components()])
+ issuer_decoded])
return issuer_string
-
class GraceDbAuthenticatedAuthentication(authentication.BaseAuthentication):
"""
If user is already authenticated by the main Django middleware,
diff --git a/gracedb/api/exceptions.py b/gracedb/api/exceptions.py
index 07c86d032de2f9a8546d0832984ddd6dd563e83a..2716683b8ceb3f8b5a82200fe4934dca320bfb6c 100644
--- a/gracedb/api/exceptions.py
+++ b/gracedb/api/exceptions.py
@@ -13,7 +13,8 @@ def gracedb_exception_handler(exc, context):
if hasattr(exc, 'detail') and hasattr(exc.detail, 'values'):
# Combine values into one list
- exc_out = [item for sublist in exc.detail.values() for item in sublist]
+ exc_out = [item for sublist in list(exc.detail.values())
+ for item in sublist]
# For only one exception, just print it rather than the list
if len(exc_out) == 1:
diff --git a/gracedb/api/tests/test_authentication.py b/gracedb/api/tests/test_authentication.py
index d1f0c9f7e9a62bb3d402865667140633e1c7d4c7..8d650fc3f0d9a581d976ef8a73cc4116c9192bf5 100644
--- a/gracedb/api/tests/test_authentication.py
+++ b/gracedb/api/tests/test_authentication.py
@@ -1,5 +1,8 @@
from base64 import b64encode
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.urls import reverse
@@ -31,9 +34,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can authenticate to API with correct password"""
# Set up and make request
url = api_reverse('api:root')
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
headers = {
'HTTP_AUTHORIZATION': 'Basic {0}'.format(user_and_pass),
}
@@ -53,16 +59,20 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can't authenticate with wrong password"""
# Set up and make request
url = api_reverse('api:root')
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password='b4d')).decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password='b4d'
+ ).encode()
+ ).decode("ascii")
headers = {
'HTTP_AUTHORIZATION': 'Basic {0}'.format(user_and_pass),
}
response = self.client.get(url, data=None, **headers)
# Check response
- self.assertEqual(response.status_code, 403)
- self.assertIn('Invalid username/password', response.content)
+ self.assertContains(response, 'Invalid username/password',
+ status_code=403)
def test_user_authenticate_to_api_with_expired_password(self):
"""User can't authenticate with expired password"""
@@ -73,17 +83,20 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
# Set up and make request
url = api_reverse('api:root')
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
headers = {
'HTTP_AUTHORIZATION': 'Basic {0}'.format(user_and_pass),
}
response = self.client.get(url, data=None, **headers)
# Check response
- self.assertEqual(response.status_code, 403)
- self.assertIn('Your password has expired', response.content)
+ self.assertContains(response, 'Your password has expired',
+ status_code=403)
class TestGraceDbX509Authentication(GraceDbApiTestBase):
@@ -139,8 +152,8 @@ class TestGraceDbX509Authentication(GraceDbApiTestBase):
response = self.client.get(url, data=None, **headers)
# Check response
- self.assertEqual(response.status_code, 401)
- self.assertIn("Invalid certificate subject", response.content)
+ self.assertContains(response, 'Invalid certificate subject',
+ status_code=401)
def test_inactive_user_authenticate(self):
"""Inactive user can't authenticate"""
@@ -156,8 +169,8 @@ class TestGraceDbX509Authentication(GraceDbApiTestBase):
response = self.client.get(url, data=None, **headers)
# Check response
- self.assertEqual(response.status_code, 401)
- self.assertIn("User inactive or deleted", response.content)
+ self.assertContains(response, 'User inactive or deleted',
+ status_code=401)
def test_authenticate_cert_with_proxy(self):
"""User can authenticate to API with proxied X509 certificate"""
diff --git a/gracedb/api/tests/test_backends.py b/gracedb/api/tests/test_backends.py
index e0caec2b4f54f0527693e445d3812e2c58e4cbdd..7c451a6d1f86dbd0818466b5c6fc8a716ab9451a 100644
--- a/gracedb/api/tests/test_backends.py
+++ b/gracedb/api/tests/test_backends.py
@@ -45,9 +45,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can authenticate to API with correct password"""
# Set up request
request = self.factory.get(api_reverse('api:root'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Authentication attempt
@@ -60,8 +63,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can't authenticate with wrong password"""
# Set up request
request = self.factory.get(api_reverse('api:root'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password='b4d')).decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password='b4d'
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Authentication attempt should fail
@@ -77,9 +84,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
# Set up request
request = self.factory.get(api_reverse('api:root'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Authentication attempt should fail
@@ -91,9 +101,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can't authenticate to a non-API URL path"""
# Set up request
request = self.factory.get(reverse('home'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Try to authenticate
@@ -108,9 +121,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
# Set up request
request = self.factory.get(api_reverse('api:root'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Authentication attempt should fail
diff --git a/gracedb/api/tests/test_throttling.py b/gracedb/api/tests/test_throttling.py
index 6f2d6e5932cbe584b6cd5600f31cf75c9309f963..9c8323f8d122147af7f8f5e628b79c47d33ba1d4 100644
--- a/gracedb/api/tests/test_throttling.py
+++ b/gracedb/api/tests/test_throttling.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.core.cache import caches
@@ -24,5 +27,4 @@ class TestThrottling(GraceDbApiTestBase):
# Second response should get throttled
response = self.request_as_user(url, "GET")
- self.assertEqual(response.status_code, 429)
- self.assertIn('Request was throttled', response.content)
+ self.assertContains(response, 'Request was throttled', status_code=429)
diff --git a/gracedb/api/tests/utils.py b/gracedb/api/tests/utils.py
index bef02a5db0d70d1c4214677a02a29430e8bbf48e..014902207412908696dd27e1164431304817c7d3 100644
--- a/gracedb/api/tests/utils.py
+++ b/gracedb/api/tests/utils.py
@@ -1,5 +1,12 @@
from copy import deepcopy
-import mock
+try:
+ from functools import reduce
+except ImportError: # python < 3
+ pass
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.core.cache import caches
diff --git a/gracedb/api/v1/events/tests/test_access.py b/gracedb/api/v1/events/tests/test_access.py
index 1cd2b54c009441ca845c61be24ef888bf17051d7..fb25c523cf08e58fac4cb3a4b114c7cd59d7a048 100644
--- a/gracedb/api/v1/events/tests/test_access.py
+++ b/gracedb/api/v1/events/tests/test_access.py
@@ -22,9 +22,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_detail(self):
"""Unauthenticated user can't access event detail"""
@@ -32,9 +34,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_log_list(self):
"""Unauthenticated user can't access event log list"""
@@ -42,9 +46,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_log_detail(self):
"""Unauthenticated user can't access event log detail"""
@@ -52,9 +58,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_voevent_list(self):
"""Unauthenticated user can't access event VOEvent list"""
@@ -62,9 +70,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_voevent_detail(self):
"""Unauthenticated user can't access event VOEvent detail"""
@@ -72,9 +82,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_embbeventlog_list(self):
"""Unauthenticated user can't access event EMBBEventLog list"""
@@ -82,9 +94,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_embbeventlog_detail(self):
"""Unauthenticated user can't access event EMBBEventLog detail"""
@@ -92,9 +106,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_emobservation_list(self):
"""Unauthenticated user can't access event EMObservation list"""
@@ -102,9 +118,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_emobservation_detail(self):
"""Unauthenticated user can't access event EMObservation detail"""
@@ -112,9 +130,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_tag_list(self):
"""Unauthenticated user can't access event tag list"""
@@ -122,9 +142,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_tag_detail(self):
"""Unauthenticated user can't access event tag detail"""
@@ -132,9 +154,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_log_tag_list(self):
"""Unauthenticated user can't access event log tag list"""
@@ -142,9 +166,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_log_tag_detail(self):
"""Unauthenticated user can't access event log tag detail"""
@@ -153,9 +179,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT", "DELETE"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_permission_list(self):
"""Unauthenticated user can't access event permission list"""
@@ -163,9 +191,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_group_permission_list(self):
"""Unauthenticated user can't access event group permission list"""
@@ -174,9 +204,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_group_permission_detail(self):
"""Unauthenticated user can't access event group permission list"""
@@ -185,9 +217,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT", "DELETE"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_files(self):
"""Unauthenticated user can't access event files (list or detail)"""
@@ -195,9 +229,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_labels(self):
"""Unauthenticated user can't access event labels (list or detail)"""
@@ -205,9 +241,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT", "DELETE"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_neighbors(self):
"""Unauthenticated user can't access event neighbors list"""
@@ -215,9 +253,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_signoff_list(self):
"""Unauthenticated user can't access event signoff list"""
@@ -225,7 +265,8 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
-
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
diff --git a/gracedb/api/v1/events/tests/test_eventgraceidfield.py b/gracedb/api/v1/events/tests/test_eventgraceidfield.py
index bff49b44575807136b7dbdb73ee35a868c6adf69..ca274422377ef6be4cb43d12ed4cab9a9eda2cd3 100644
--- a/gracedb/api/v1/events/tests/test_eventgraceidfield.py
+++ b/gracedb/api/v1/events/tests/test_eventgraceidfield.py
@@ -50,4 +50,4 @@ def test_valid_graceids(graceid):
call_args, _ = mock_super_tiv.call_args
assert mock_super_tiv.call_count == 1
assert len(call_args) == 1
- assert call_args[0].encode() == graceid.upper().strip()
+ assert call_args[0] == graceid.upper().strip()
diff --git a/gracedb/api/v1/events/tests/test_update_grbevent_view.py b/gracedb/api/v1/events/tests/test_update_grbevent_view.py
index 64078c84d5a445e07c3342beef9e937417c33434..06d68be1e89c92c7fa8da22a82c3f690e1c3a60c 100644
--- a/gracedb/api/v1/events/tests/test_update_grbevent_view.py
+++ b/gracedb/api/v1/events/tests/test_update_grbevent_view.py
@@ -147,7 +147,8 @@ def test_update_with_no_new_data(grb_user, internal_group, data):
# Check response
assert response.status_code == 400
- assert 'Request would not modify the GRB event' in response.content
+ assert 'Request would not modify the GRB event' \
+ in response.content.decode()
@pytest.mark.parametrize("data",
@@ -176,7 +177,7 @@ def test_update_with_bad_data(grb_user, internal_group, data):
# Check response
assert response.status_code == 400
- assert 'must be a float' in response.content
+ assert 'must be a float' in response.content.decode()
@pytest.mark.django_db
@@ -206,4 +207,4 @@ def test_update_non_grbevent(grb_user, internal_group):
# Check response
assert response.status_code == 400
assert 'Cannot update GRB event parameters for non-GRB event' \
- in response.content
+ in response.content.decode()
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index bfbd37e8c573244ab4da7e3e560759f829a0faa8..c9b292c9bebf7b402c00674b156877f132345cf2 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -1,11 +1,12 @@
from __future__ import absolute_import
-import exceptions
import json
import logging
import os
import shutil
-import StringIO
-import urllib
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
from django.conf import settings
from django.contrib.auth.models import User, Permission, Group as DjangoGroup
@@ -16,6 +17,7 @@ from django.http import HttpResponse, HttpResponseForbidden, \
HttpResponseNotFound, HttpResponseServerError, HttpResponseBadRequest
from django.http.request import QueryDict
from django.utils.functional import wraps
+from django.utils.http import urlencode
# Stuff for the LigoLwRenderer
from glue.ligolw import ligolw
@@ -36,10 +38,10 @@ from rest_framework.views import APIView
from alerts.issuers.events import EventAlertIssuer, EventLogAlertIssuer, \
EventVOEventAlertIssuer, EventPermissionsAlertIssuer
+from annotations.voevent_utils import construct_voevent_file
from api.throttling import BurstAnonRateThrottle
from core.http import check_and_serve_file
-from core.vfile import VersionedFile
-from events.buildVOEvent import buildVOEvent, VOEventBuilderException
+from core.vfile import create_versioned_file
from events.forms import CreateEventForm
from events.models import Event, Group, Search, Pipeline, EventLog, Tag, \
Label, Labelling, EMGroup, EMBBEventLog, EMSPECTRUM, VOEvent, GrbEvent
@@ -243,7 +245,8 @@ class CoincAccess(Exception):
return repr(self.detail)
def assembleLigoLw(data):
- if 'events' in data.keys():
+ # data is a dict
+ if 'events' in data:
eventDictList = data['events']
else:
# There is only one event.
@@ -269,12 +272,12 @@ class LigoLwRenderer(BaseRenderer):
# XXX If there was an error, we will return the error message
# in plain text, effectively ignoring the accepts header.
# Somewhat irregular?
- if 'error' in data.keys():
+ if 'error' in data:
return data['error']
xmldoc = assembleLigoLw(data)
# XXX Aaargh! Just give me the contents of the xml doc. Annoying.
- output = StringIO.StringIO()
+ output = StringIO()
xmldoc.write(output)
return output.getvalue()
@@ -283,12 +286,12 @@ class TSVRenderer(BaseRenderer):
format = 'tsv'
def render(self, data, media_type=None, renderer_context=None):
- if 'error' in data.keys():
+ if 'error' in data:
return data['error']
accessFun = {
"labels" : lambda e: \
- ",".join(e['labels'].keys()),
+ ",".join(list(e['labels'])),
"dataurl" : lambda e: e['links']['files'],
}
def defaultAccess(e,a):
@@ -381,7 +384,7 @@ class EventList(InheritPermissionsAPIView):
except ParseException:
d = {'error': 'Invalid query' }
return Response(d,status=status.HTTP_400_BAD_REQUEST)
- except Exception, e:
+ except Exception as e:
d = {'error': str(e) }
return Response(d,status=status.HTTP_500_INTERNAL_SERVER_ERROR)
form = SimpleSearchForm(request.GET)
@@ -404,7 +407,7 @@ class EventList(InheritPermissionsAPIView):
d = {'error': 'Too many events.' }
return Response(d, status=status.HTTP_400_BAD_REQUEST)
- last = max(0, (numRows / count)) * count
+ last = max(0, (numRows // count)) * count
rv = {}
links = {}
rv['links'] = links
@@ -416,14 +419,14 @@ class EventList(InheritPermissionsAPIView):
d = { 'start' : 0, "count": count, "sort": sort }
if query: d['query'] = query
- links['first'] = baseuri + "?" + urllib.urlencode(d)
+ links['first'] = baseuri + "?" + urlencode(d)
d['start'] = last
- links['last'] = baseuri + "?" + urllib.urlencode(d)
+ links['last'] = baseuri + "?" + urlencode(d)
if start != last:
d['start'] = start+count
- links['next'] = baseuri + "?" + urllib.urlencode(d)
+ links['next'] = baseuri + "?" + urlencode(d)
rv['numRows'] = events.count()
response = Response(rv)
@@ -447,7 +450,7 @@ class EventList(InheritPermissionsAPIView):
# the function we are presently inside.
response = self.finalize_response(request, response, *args, **kwargs)
response.render()
- except Exception, e:
+ except Exception as e:
try:
status_code = e.status_code
except:
@@ -489,7 +492,7 @@ class EventList(InheritPermissionsAPIView):
# TP (21 Nov 2017): Hack to allow basic event submission without
# labels to function with versions of gracedb-client before 1.26.
# Should be removed eventually.
- if (request.data.has_key('labels') and request.data['labels'] == ''):
+ if (request.data.get('labels') == ''):
request.data.pop('labels', None)
form = CreateEventForm(request.data, request.data)
@@ -567,7 +570,7 @@ class EventDetail(InheritPermissionsAPIView):
# the function we are presently inside.
response = self.finalize_response(request, response)
response.render()
- except Exception, e:
+ except Exception as e:
try:
status_code = e.status_code
except:
@@ -585,7 +588,7 @@ class EventDetail(InheritPermissionsAPIView):
if request.user != event.submitter:
msg = "You (%s) Them (%s)" % (request.user, event.submitter)
return HttpResponseForbidden("You did not create this event. %s" %msg)
- except Exception, e:
+ except Exception as e:
return Response(str(e))
# Compile far and nscand for alerts
@@ -607,29 +610,14 @@ class EventDetail(InheritPermissionsAPIView):
# return Response("\n".join(messages),
# status=status.HTTP_400_BAD_REQUEST)
- # XXX handle duplicate file names.
+ # Create versioned file
f = request.data['eventFile']
- uploadDestination = os.path.join(event.datadir, f.name)
- fdest = VersionedFile(uploadDestination, 'w')
- #for chunk in f.chunks():
- # fdest.write(chunk)
- #fdest.close()
- shutil.copyfileobj(f, fdest)
- fdest.close()
+ version = create_versioned_file(f.name, event.datadir, f)
# Extract Info from uploaded data
- try:
- handle_uploaded_data(event, uploadDestination)
- event.submitter = request.user
- except:
- # XXX Bad news. If the log file fails to save because of
- # race conditions, then this will also be the the message
- # returned. Somehow, I think there are other things that
- # could go wrong inside handle_uploaded_data besides just
- # bad data. We should probably check for different types
- # of exceptions here.
- return Response("Bad Data",
- status=status.HTTP_400_BAD_REQUEST)
+ uploadDestination = os.path.join(event.datadir, f.name)
+ handle_uploaded_data(event, uploadDestination)
+ event.submitter = request.user
# Save event
event.save()
@@ -746,13 +734,13 @@ class EventNeighbors(InheritPermissionsAPIView):
# and TSV renderers.
@event_and_auth_required
def get(self, request, event):
- if request.query_params.has_key('neighborhood'):
+ if 'neighborhood' in request.query_params:
delta = request.query_params['neighborhood']
try:
if delta.find(',') < 0:
neighborhood = (int(delta), int(delta))
else:
- neighborhood = map(int, delta.split(','))
+ neighborhood = list(map(int, delta.split(',')))
except ValueError:
pass
else:
@@ -810,7 +798,7 @@ class EventLabel(InheritPermissionsAPIView):
try:
rv, label_created = create_label(event, request, label)
except (ValueError, Label.ProtectedLabelError) as e:
- return Response(e.message,
+ return Response(str(e),
status=status.HTTP_400_BAD_REQUEST)
# Return response and status code
@@ -825,9 +813,9 @@ class EventLabel(InheritPermissionsAPIView):
try:
rv = delete_label(event, request, label)
except Labelling.DoesNotExist as e:
- return Response(e.message, status=status.HTTP_404_NOT_FOUND)
+ return Response(str(e), status=status.HTTP_404_NOT_FOUND)
except (ValueError, Label.ProtectedLabelError) as e:
- return Response(e.message, status=status.HTTP_400_BAD_REQUEST)
+ return Response(str(e), status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_204_NO_CONTENT)
@@ -891,17 +879,12 @@ class EventLogList(InheritPermissionsAPIView):
file_version = None
if uploadedFile:
filename = uploadedFile.name
- filepath = os.path.join(event.datadir, filename)
try:
# Open / Write the file.
- fdest = VersionedFile(filepath, 'w')
- for chunk in uploadedFile.chunks():
- fdest.write(chunk)
- fdest.close()
- # Ascertain the version assigned to this particular file.
- file_version = fdest.version
- except Exception, e:
+ file_version = create_versioned_file(filename, event.datadir,
+ uploadedFile)
+ except Exception as e:
# XXX This needs some thought.
response = Response(str(e), status=status.HTTP_400_BAD_REQUEST)
@@ -954,7 +937,7 @@ class EventLogList(InheritPermissionsAPIView):
rv = eventLogToDict(logentry, request=request)
response = Response(rv, status=status.HTTP_201_CREATED)
response['Location'] = rv['self']
- #if 'tagWarning' in tw_dict.keys():
+ #if 'tagWarning' in tw_dict:
# response['tagWarning'] = tw_dict['tagWarning']
# Issue alert.
@@ -1013,12 +996,12 @@ class EMBBEventLogList(InheritPermissionsAPIView):
try:
# Alert is issued in this code
eel = create_eel(request.data, event, request.user)
- except ValueError, e:
+ except ValueError as e:
return Response("%s" % str(e), status=status.HTTP_400_BAD_REQUEST)
- except IntegrityError, e:
+ except IntegrityError as e:
return Response("Failed to save EMBB entry: %s" % str(e),
status=status.HTTP_503_SERVICE_UNAVAILABLE)
- except Exception, e:
+ except Exception as e:
return Response("Problem creating EEL: %s" % str(e),
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -1061,7 +1044,7 @@ class EMObservationList(InheritPermissionsAPIView):
# XXX Note the following hack.
# If this JSON information is requested for skymapViewer, use a different
# representation for backwards compatibility.
- if 'skymapViewer' in request.query_params.keys():
+ if 'skymapViewer' in request.query_params:
emo = [ skymapViewerEMObservationToDict(emo, request)
for emo in emo_set.iterator() ]
@@ -1097,12 +1080,12 @@ class EMObservationList(InheritPermissionsAPIView):
try:
# Create EMObservation - alert is issued inside this code
emo = create_emobservation(request, event)
- except ValueError, e:
+ except ValueError as e:
return Response("%s" % str(e), status=status.HTTP_400_BAD_REQUEST)
- except IntegrityError, e:
+ except IntegrityError as e:
return Response("Failed to save EMBB observation record: %s" % str(e),
status=status.HTTP_503_SERVICE_UNAVAILABLE)
- except Exception, e:
+ except Exception as e:
return Response("Problem creating EMBB Observation: %s" % str(e),
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -1277,7 +1260,7 @@ class EventLogTagDetail(InheritPermissionsAPIView):
# client that the creation was sucessful when, in fact, the database
# was unchanged.
tag = eventlog.tags.filter(name=tagname)[0]
- msg = "Log already has tag %s" % unicode(tag)
+ msg = "Log already has tag {0}".format(tag.name)
return Response(msg,status=status.HTTP_409_CONFLICT)
except:
# Check authorization
@@ -1462,7 +1445,7 @@ class GroupEventPermissionDetail(InheritPermissionsAPIView):
permission=underlying_permission)
underlying_event.refresh_perms()
- except Exception, e:
+ except Exception as e:
# We're gonna blame the user here.
return Response("Problem creating permission: %" % str(e),
status=status.HTTP_400_BAD_REQUEST)
@@ -1528,7 +1511,7 @@ class GroupEventPermissionDetail(InheritPermissionsAPIView):
except GroupObjectPermission.DoesNotExist:
return Response("GroupObjectPermission not found.",
status=status.HTTP_404_NOT_FOUND)
- except Exception, e:
+ except Exception as e:
return Response("Problem deleting permission: %s" % str(e),
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -1610,25 +1593,20 @@ class Files(InheritPermissionsAPIView):
def put(self, request, event, filename=""):
""" File uploader. Implements file versioning. """
filename = filename or ""
- filepath = os.path.join(event.datadir, filename)
try:
# Open / Write the file.
- fdest = VersionedFile(filepath, 'w')
f = request.data['upload']
- for chunk in f.chunks():
- fdest.write(chunk)
- fdest.close()
- file_version = fdest.version
+ file_version = create_versioned_file(filename, event.datadir, f)
rv = {}
# XXX this seems wobbly.
- longname = fdest.name
+ longname = os.path.join(event.datadir, filename)
shortname = longname[longname.rfind(filename):]
rv['permalink'] = api_reverse(
"events:files", args=[event.graceid, shortname], request=request)
response = Response(rv, status=status.HTTP_201_CREATED)
- except Exception, e:
+ except Exception as e:
# XXX This needs some thought.
response = Response(str(e), status=status.HTTP_400_BAD_REQUEST)
# XXX Uhm, we don't to try creating a log message for this, right?
@@ -1695,16 +1673,11 @@ class VOEventList(InheritPermissionsAPIView):
@event_and_auth_required
def post(self, request, event):
+ # Get data from request
voevent_type = request.data.get('voevent_type', None)
- if not voevent_type:
- msg = "You must provide a valid voevent_type."
- return Response({'error': msg}, status = status.HTTP_400_BAD_REQUEST)
-
internal = request.data.get('internal', 1)
-
skymap_type = request.data.get('skymap_type', None)
skymap_filename = request.data.get('skymap_filename', None)
-
open_alert = request.data.get('open_alert', 0)
hardware_inj = request.data.get('hardware_inj', 0)
CoincComment = request.data.get('CoincComment', None)
@@ -1716,9 +1689,65 @@ class VOEventList(InheritPermissionsAPIView):
Terrestrial = request.data.get('Terrestrial', None)
MassGap = request.data.get('MassGap', None)
- if (skymap_filename and not skymap_type) or (skymap_type and not skymap_filename):
- msg = "Both or neither of skymap_type and skymap_filename must be specified."
- return Response({'error': msg}, status = status.HTTP_400_BAD_REQUEST)
+ # Get RAVEN data
+ ext_gcn = request.data.get('ext_gcn', None)
+ ext_pipeline = request.data.get('ext_pipeline', None)
+ ext_search = request.data.get('ext_search', None)
+ time_coinc_far = request.data.get('time_coinc_far', None)
+ space_coinc_far = request.data.get('space_coinc_far', None)
+ combined_skymap_filename = request.data.get('combined_skymap_filename',
+ None)
+ delta_t = request.data.get('delta_t', None)
+ raven_coinc = request.data.get('raven_coinc', None)
+
+ # Get VOEvent types as a dict (key = short form, value = long form)
+ VOEVENT_TYPE_DICT = dict(VOEvent.VOEVENT_TYPE_CHOICES)
+
+ # Check data
+ error = False
+ if not voevent_type or voevent_type not in VOEVENT_TYPE_DICT:
+ error = True
+ msg = "You must provide a valid voevent_type."
+ elif ((skymap_filename and not skymap_type) or
+ (skymap_type and not skymap_filename)):
+ error = True
+ msg = ("Both or neither of skymap_type and skymap_filename must "
+ "be specified.")
+ elif not event.gpstime:
+ error = True
+ msg = "Cannot build a VOEvent because event has no gpstime."
+ elif not event.far:
+ error = True
+ msg = "Cannot build a VOEvent because event has no FAR."
+ elif (voevent_type in ["IN", "UP"] or
+ voevent_type == "PR" and skymap_filename is not None):
+ if skymap_filename is None:
+ error = True
+ msg = "Skymap filename not provided."
+ if skymap_type is None:
+ error = True
+ msg = "Skymap type must be provided."
+
+ # Check if skymap file exists
+ skymap_file_path = os.path.join(event.datadir, skymap_filename)
+ if not os.path.exists(skymap_file_path):
+ error = True
+ msg = "Skymap file {fname} does not exist".format(
+ fname=skymap_filename)
+ elif time_coinc_far or space_coinc_far:
+ if not ext_gcn:
+ error = True
+ msg = "External GCN ID not provided"
+ elif not ext_pipeline:
+ error = True
+ msg = "External Pipeline not provided"
+ elif not ext_search:
+ error = True
+ msg = "External Search not provided"
+
+ # If there's an error, return a 400 response
+ if error:
+ return Response({'error': msg}, status=status.HTTP_400_BAD_REQUEST)
# Instantiate the voevent and save in order to get the serial number
voevent = VOEvent(event=event, issuer=request.user,
@@ -1738,19 +1767,13 @@ class VOEventList(InheritPermissionsAPIView):
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# Now, you need to actually build the VOEvent.
- try:
- voevent_text, ivorn = buildVOEvent(event, voevent, request=request)
- except VOEventBuilderException, e:
- msg = "Problem building VOEvent: %s" % str(e)
- return Response({'error': msg}, status = status.HTTP_400_BAD_REQUEST)
+ voevent_text, ivorn = construct_voevent_file(event, voevent,
+ request=request)
- voevent_display_type = dict(VOEvent.VOEVENT_TYPE_CHOICES)[voevent_type].capitalize()
+ voevent_display_type = VOEVENT_TYPE_DICT[voevent_type].capitalize()
filename = "%s-%d-%s.xml" % (event.graceid, voevent.N, voevent_display_type)
- filepath = os.path.join(event.datadir, filename)
- fdest = VersionedFile(filepath, 'w')
- fdest.write(voevent_text)
- fdest.close()
- file_version = fdest.version
+ file_version = create_versioned_file(filename, event.datadir,
+ voevent_text)
voevent.filename = filename
voevent.file_version = file_version
diff --git a/gracedb/api/v1/fields.py b/gracedb/api/v1/fields.py
index ebc3ec316f19d3f1eb08ba919a6859c8f77b266d..7fc167bc56f0b1c2bbdadd3108759a3df3bc8d7b 100644
--- a/gracedb/api/v1/fields.py
+++ b/gracedb/api/v1/fields.py
@@ -125,8 +125,8 @@ class GenericField(fields.Field):
err_msg = self.get_does_not_exist_error(data)
else:
err_msg = '{model} with {lf}={data} does not exist'.format(
- model=self.model.__name__, lf=model_dict.keys()[0],
- data=model_dict.values()[0]
+ model=self.model.__name__, lf=list(model_dict)[0],
+ data=list(model_dict.values())[0]
)
raise exceptions.ValidationError(err_msg)
diff --git a/gracedb/api/v1/main/tests/test_access.py b/gracedb/api/v1/main/tests/test_access.py
index 19854c43837d443dad9144905d1d2b1f8cac9b41..4d94ed1a7284124c17aaa9ed49aad04ced03c295 100644
--- a/gracedb/api/v1/main/tests/test_access.py
+++ b/gracedb/api/v1/main/tests/test_access.py
@@ -31,14 +31,18 @@ class TestPublicAccess(GraceDbApiTestBase):
"""Unauthenticated user can't access performance info"""
url = v_reverse('performance-info')
response = self.request_as_user(url, "GET")
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_lvem_user_performance_info(self):
"""LV-EM user can't access performance info"""
url = v_reverse('performance-info')
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Forbidden", response.content)
-
+ self.assertContains(
+ response,
+ 'Forbidden',
+ status_code=403
+ )
diff --git a/gracedb/api/v1/main/tests/test_views.py b/gracedb/api/v1/main/tests/test_views.py
index 74d8c9810fe19e6f6f8b29279408a45150473db5..728d34526f20ad96686fa565a115ef801d82e04a 100644
--- a/gracedb/api/v1/main/tests/test_views.py
+++ b/gracedb/api/v1/main/tests/test_views.py
@@ -46,5 +46,5 @@ class TestUserInfoView(GraceDbApiTestBase):
self.assertEqual(response.status_code, 200)
# Test information
- self.assertEqual(response.data.keys(), ['username'])
+ self.assertEqual(list(response.data), ['username'])
self.assertEqual(response.data['username'], 'AnonymousUser')
diff --git a/gracedb/api/v1/main/views.py b/gracedb/api/v1/main/views.py
index 2cef09eb66f6e34f2c57d7a0dc1494c374d4b601..8387bc6579bca22ddd093851c9eb3534b50b59c4 100644
--- a/gracedb/api/v1/main/views.py
+++ b/gracedb/api/v1/main/views.py
@@ -185,7 +185,7 @@ class PerformanceInfo(InheritDefaultPermissionsMixin, APIView):
try:
performance_info = get_performance_info()
- except Exception, e:
+ except Exception as e:
return Response(str(e),
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
diff --git a/gracedb/api/v1/paginators.py b/gracedb/api/v1/paginators.py
index 6fa5e67775bc13b30d876740eaa129d8dac0d330..d099a4b0dab196ea321c1e5756561eaffc92d19f 100644
--- a/gracedb/api/v1/paginators.py
+++ b/gracedb/api/v1/paginators.py
@@ -1,6 +1,7 @@
from collections import OrderedDict
import logging
-import urllib
+
+from django.utils.http import urlencode
from rest_framework import pagination
from rest_framework.response import Response
@@ -80,7 +81,7 @@ class CustomSupereventPagination(pagination.LimitOffsetPagination):
'start': last,
self.limit_query_param: self.limit,
}
- last_uri = base_uri + '?' + urllib.urlencode(param_dict)
+ last_uri = base_uri + '?' + urlencode(param_dict)
output = OrderedDict([
('numRows', numRows),
diff --git a/gracedb/api/v1/superevents/filters.py b/gracedb/api/v1/superevents/filters.py
index cb6a15306752ae1e498fb899d920443f17efe131..8ff62a00443eae5eeb35686d1605d627c01c1e04 100644
--- a/gracedb/api/v1/superevents/filters.py
+++ b/gracedb/api/v1/superevents/filters.py
@@ -58,7 +58,7 @@ class SupereventOrderingFilter(filters.OrderingFilter):
for f in fields:
prefix = '-' if f.startswith('-') else ''
f_s = f.lstrip('-')
- if f_s in self.field_map.keys():
+ if f_s in self.field_map:
mapped_fields = self.field_map[f_s]
if not isinstance(mapped_fields, list):
mapped_fields = [mapped_fields]
diff --git a/gracedb/api/v1/superevents/paginators.py b/gracedb/api/v1/superevents/paginators.py
index 599682a17d4b8a02e5b15c8c6f02c1ebf0b0bfde..8cc96729262ef1c07f1f191c0b969361988024e1 100644
--- a/gracedb/api/v1/superevents/paginators.py
+++ b/gracedb/api/v1/superevents/paginators.py
@@ -1,6 +1,7 @@
from collections import OrderedDict
import logging
-import urllib
+
+from django.utils.http import urlencode
from rest_framework import pagination
from rest_framework.response import Response
@@ -26,7 +27,7 @@ class CustomSupereventPagination(pagination.LimitOffsetPagination):
'start': last,
self.limit_query_param: self.limit,
}
- last_uri = base_uri + '?' + urllib.urlencode(param_dict)
+ last_uri = base_uri + '?' + urlencode(param_dict)
output = OrderedDict([
('numRows', numRows),
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index 8009b79114a2fd599357cede78d62026378a5723..c38e2e260b94e6abb7232be50392b1de21d6bc10 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -71,8 +71,9 @@ class SupereventSerializer(serializers.ModelSerializer):
class Meta:
model = Superevent
fields = ('superevent_id', 'gw_id', 'category', 'created', 'submitter',
- 'preferred_event', 'events', 't_start', 't_0', 't_end',
- 'gw_events', 'em_events', 'far', 'labels', 'links', 'user')
+ 'preferred_event', 'events', 'em_type', 't_start', 't_0', 't_end',
+ 'gw_events', 'em_events', 'far', 'coinc_far', 'labels', 'links',
+ 'user')
def validate(self, data):
data = super(SupereventSerializer, self).validate(data)
@@ -171,7 +172,8 @@ class SupereventUpdateSerializer(SupereventSerializer):
Used for updates ONLY (PUT/PATCH). Overrides validation which is needed
for object creation.
"""
- allowed_fields = ('t_start', 't_0', 't_end', 'preferred_event')
+ allowed_fields = ('t_start', 't_0', 't_end', 'preferred_event',
+ 'em_type', 'coinc_far')
def __init__(self, *args, **kwargs):
super(SupereventUpdateSerializer, self).__init__(*args, **kwargs)
@@ -538,6 +540,12 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
'superevent.'),
'skymap_image_not_found': _('Skymap image file {filename} not found '
'for this superevent.'),
+ 'em_type_none': _('em_type for superevent {s_event} not set (is None)'),
+ 'invalid_em_type': _('em_type for superevent {s_event} is not a valid '
+ 'graceid'),
+ 'em_type_not_found': _('event for em_type={em_type} not found'),
+ 'comb_skymap_not_found': _('Combined skymap file {filename} not found '
+ 'for this superevent.'),
}
# Read only fields
issuer = serializers.SlugRelatedField(slug_field='username',
@@ -575,6 +583,20 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
MassGap = serializers.FloatField(write_only=True, min_value=0,
max_value=1, required=False)
+ # Additional RAVEN fields
+ raven_coinc = serializers.BooleanField(default=False)
+ ext_gcn = serializers.CharField(required=False)
+ ext_pipeline = serializers.CharField(required=False)
+ ext_search = serializers.CharField(required=False)
+ time_coinc_far = serializers.FloatField(write_only=True, min_value=0,
+ max_value=1000, required=False)
+ space_coinc_far = serializers.FloatField(write_only=True, min_value=0,
+ max_value=1000, required=False)
+ combined_skymap_filename = serializers.CharField(required=False)
+ delta_t = serializers.FloatField(write_only=True, min_value=-1000,
+ max_value=1000, required=False)
+ ivorn = serializers.CharField(required=False)
+
class Meta:
model = VOEvent
fields = ('voevent_type', 'file_version', 'ivorn', 'created',
@@ -585,11 +607,18 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
'prob_has_remnant', 'prob_bns', 'prob_nsbh', 'prob_bbh',
'prob_terrestrial', 'prob_mass_gap', 'superevent', 'user')
+ raven_fields = ('raven_coinc','ext_gcn', 'ext_pipeline', 'ext_search',
+ 'time_coinc_far', 'space_coinc_far', 'combined_skymap_filename',
+ 'delta_t', 'ivorn')
+
+ # Combine the fields:
+ fields = fields + raven_fields
+
def __init__(self, *args, **kwargs):
super(SupereventVOEventSerializer, self).__init__(*args, **kwargs)
read_only_fields = ['file_version', 'filename', 'ivorn',
'coinc_comment', 'prob_has_ns', 'prob_has_remnant', 'prob_bns',
- 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap']
+ 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap', ]
for f in read_only_fields:
self.fields.get(f).read_only = True
@@ -617,6 +646,9 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
voevent_type = data.get('voevent_type')
skymap_filename = data.get('skymap_filename', None)
skymap_type = data.get('skymap_type', None)
+ raven_coinc = data.get('raven_coinc')
+ combined_smfn = data.get('combined_skymap_filename',None)
+
# Checks to do:
# Preferred event must have gpstime
@@ -642,6 +674,23 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
if not os.path.exists(full_skymap_path):
self.fail('skymap_not_found', filename=skymap_filename)
+ if raven_coinc:
+ if superevent.em_type is None:
+ self.fail('em_type_none', s_event=superevent.graceid)
+ else:
+ try:
+ em_event = Event.getByGraceid(superevent.em_type)
+ except ValueError:
+ self.fail('invalid_em_type', s_event=superevent.graceid)
+ except Event.DoesNotExist:
+ self.fail('em_type_not_found',em_type=superevent.em_type)
+ if combined_smfn != None:
+ comb_skymap_path = os.path.join(superevent.datadir,
+ combined_smfn)
+ if not os.path.exists(comb_skymap_path):
+ self.fail('comb_skymap_not_found', filename=combined_smfn)
+
+
return data
def create(self, validated_data):
@@ -653,7 +702,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
issuer = validated_data.pop('user')
# Call create function - creates VOEvent object and also runs
- # buildVOEvent to create the related file.
+ # construct_voevent_file to create the related file.
voevent = create_voevent_for_superevent(superevent, issuer,
**validated_data)
@@ -783,7 +832,7 @@ class SupereventEMObservationSerializer(serializers.ModelSerializer):
list_length = len(ra_list)
all_lists = (ra_list, dec_list, ra_width_list, dec_width_list,
start_time_list, duration_list)
- if not all(map(lambda l: len(l) == list_length, all_lists)):
+ if not all(list(map(lambda l: len(l) == list_length, all_lists))):
self.fail('list_lengths')
return data
diff --git a/gracedb/api/v1/superevents/tests/test_access.py b/gracedb/api/v1/superevents/tests/test_access.py
index 85c49b920ebe4a9e88835055b57cbbf4cbfa0e13..958e9e2e1bfadb95808d91e8c10c364dfa39c8b0 100644
--- a/gracedb/api/v1/superevents/tests/test_access.py
+++ b/gracedb/api/v1/superevents/tests/test_access.py
@@ -503,9 +503,11 @@ class TestSupereventConfirmAsGw(SupereventManagersGroupAndUserSetup,
assign_perm('superevents.view_superevent', self.lvem_obs_group,
obj=self.production_superevent)
response = self.request_as_user(url, "POST", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to confirm superevents as GWs',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to confirm superevents as GWs',
+ status_code=403
+ )
def test_lvem_user_confirm_mdc_superevent(self):
"""LV-EM user can't confirm mdc superevent as GW"""
@@ -520,9 +522,11 @@ class TestSupereventConfirmAsGw(SupereventManagersGroupAndUserSetup,
assign_perm('superevents.view_superevent', self.lvem_obs_group,
obj=self.mdc_superevent)
response = self.request_as_user(url, "POST", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to confirm MDC superevents as GWs',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to confirm MDC superevents as GWs',
+ status_code=403
+ )
def test_lvem_user_confirm_test_superevent(self):
"""LV-EM user can't confirm test superevent as GW"""
@@ -537,9 +541,11 @@ class TestSupereventConfirmAsGw(SupereventManagersGroupAndUserSetup,
assign_perm('superevents.view_superevent', self.lvem_obs_group,
obj=self.test_superevent)
response = self.request_as_user(url, "POST", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to confirm test superevents as GWs',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to confirm test superevents as GWs',
+ status_code=403
+ )
def test_public_user_confirm_superevents(self):
"""Public user can't confirm any superevents as GWs"""
@@ -551,16 +557,22 @@ class TestSupereventConfirmAsGw(SupereventManagersGroupAndUserSetup,
url = v_reverse('superevents:superevent-confirm-as-gw',
args=[s.superevent_id])
response = self.request_as_user(url, "POST")
- self.assertEqual(response.status_code, 403)
- self.assertIn('Authentication credentials', response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials',
+ status_code=403
+ )
# Expose it, should still get same 403 since the permission
# checking process still fails at the outset
assign_perm('superevents.view_superevent', self.public_group,
obj=self.test_superevent)
response = self.request_as_user(url, "POST")
- self.assertEqual(response.status_code, 403)
- self.assertIn('Authentication credentials', response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials',
+ status_code=403
+ )
class TestSupereventLabelList(SupereventSetup, GraceDbApiTestBase):
@@ -738,8 +750,11 @@ class TestSupereventLabelList(SupereventSetup, GraceDbApiTestBase):
args=[self.public_superevent.superevent_id])
data = {'name': label.name}
response = self.request_as_user(url, "POST", data=data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('Authentication credentials', response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials',
+ status_code=403
+ )
class TestSupereventLabelDetail(SupereventSetup, GraceDbApiTestBase):
@@ -873,9 +888,11 @@ class TestSupereventLabelDetail(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-label-detail',
args=[self.public_superevent.superevent_id, l.name])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('You do not have permission to remove labels',
- response.content)
+ self.assertContains(
+ response,
+ 'You do not have permission to remove labels',
+ status_code=403
+ )
def test_public_user_delete_label(self):
"""Public user cannot remove labels from any superevents"""
@@ -905,8 +922,11 @@ class TestSupereventLabelDetail(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-label-detail',
args=[self.public_superevent.superevent_id, l.name])
response = self.request_as_user(url, "DELETE")
- self.assertEqual(response.status_code, 403)
- self.assertIn('Authentication credentials', response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials',
+ status_code=403
+ )
class TestSupereventEventList(SupereventManagersGroupAndUserSetup,
@@ -1128,9 +1148,11 @@ class TestSupereventEventList(SupereventManagersGroupAndUserSetup,
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.lvem_user,
data={'event': ev.graceid})
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to add events to superevents',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to add events to superevents',
+ status_code=403
+ )
def test_public_user_add_event_to_superevent(self):
"""Public user can't add events to superevents"""
@@ -1151,8 +1173,11 @@ class TestSupereventEventList(SupereventManagersGroupAndUserSetup,
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST",
data={'event': ev.graceid})
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventEventDetail(SupereventManagersGroupAndUserSetup,
@@ -1351,9 +1376,11 @@ class TestSupereventEventDetail(SupereventManagersGroupAndUserSetup,
args=[self.lvem_superevent.superevent_id,
self.event2.graceid])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to remove events from superevents',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to remove events from superevents',
+ status_code=403
+ )
def test_public_user_remove_event_from_superevent(self):
"""
@@ -1378,8 +1405,11 @@ class TestSupereventEventDetail(SupereventManagersGroupAndUserSetup,
args=[self.public_superevent.superevent_id,
self.event2.graceid])
response = self.request_as_user(url, "DELETE")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventLogList(AccessManagersGroupAndUserSetup,
@@ -1547,10 +1577,11 @@ class TestSupereventLogList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data=log_data)
# Check response and data
- self.assertEqual(response.status_code, 403)
- # Make sure correct 403 error is provided
- self.assertIn('not allowed to expose superevent log messages to LV-EM',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevent log messages to LV-EM',
+ status_code=403
+ )
def test_internal_user_create_log_with_public_tag(self):
"""Basic internal user can't create logs with public access tag"""
@@ -1567,10 +1598,11 @@ class TestSupereventLogList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data=log_data)
# Check response and data
- self.assertEqual(response.status_code, 403)
- # Make sure correct 403 error is provided
- self.assertIn(('not allowed to expose superevent log messages to the '
- 'public'), response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevent log messages to the public',
+ status_code=403
+ )
def test_access_manager_create_log_with_lvem_tag(self):
"""Access manager user can create logs with external access tag"""
@@ -1672,9 +1704,11 @@ class TestSupereventLogList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.lvem_user,
data=log_data)
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('You are not allowed to post log messages with tags',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to post log messages with tags',
+ status_code=403
+ )
def test_public_user_create_log(self):
"""Public user can't create any logs"""
@@ -1699,8 +1733,11 @@ class TestSupereventLogList(AccessManagersGroupAndUserSetup,
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", data=log_data)
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventLogDetail(SupereventSetup, GraceDbApiTestBase):
@@ -1890,8 +1927,10 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
self.assertEqual(response.status_code, 200)
# Lists of log tags from
log_tags = [t.name for t in l.tags.all()]
- self.assertItemsEqual(
- [t['name'] for t in response.data['tags']], log_tags)
+ self.assertEqual(
+ sorted([t['name'] for t in response.data['tags']]),
+ sorted(log_tags)
+ )
def test_lvem_user_get_log_tag_list_for_hidden_superevent(self):
"""LV-EM user can't get tags for any logs on hidden superevent"""
@@ -2015,10 +2054,11 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data={'name': settings.EXTERNAL_ACCESS_TAGNAME})
# Check response and data
- self.assertEqual(response.status_code, 403)
- # Make sure correct 403 error is provided
- self.assertIn('not allowed to expose superevent log messages to LV-EM',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevent log messages to LV-EM',
+ status_code=403
+ )
def test_internal_user_tag_log_with_public(self):
"""Basic internal user add public access tag"""
@@ -2035,10 +2075,11 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data={'name': settings.PUBLIC_ACCESS_TAGNAME})
# Check response and data
- self.assertEqual(response.status_code, 403)
- # Make sure correct 403 error is provided
- self.assertIn(('not allowed to expose superevent log messages to the '
- 'public'), response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevent log messages to the public',
+ status_code=403
+ )
def test_access_manager_tag_log_with_lvem(self):
"""Access manager user can tag logs with external access tag"""
@@ -2076,8 +2117,11 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.am_user,
data={'name': settings.PUBLIC_ACCESS_TAGNAME})
# Check response and data
- self.assertEqual(response.status_code, 201)
- self.assertIn(response.data['name'], settings.PUBLIC_ACCESS_TAGNAME)
+ self.assertContains(
+ response,
+ settings.PUBLIC_ACCESS_TAGNAME,
+ status_code=201
+ )
def test_lvem_user_tag_log(self):
"""LV-EM user can't tag logs for any superevents"""
@@ -2147,8 +2191,11 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST",
data={'name': self.tag1.name})
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventLogTagDetail(AccessManagersGroupAndUserSetup,
@@ -2398,8 +2445,11 @@ class TestSupereventLogTagDetail(AccessManagersGroupAndUserSetup,
args=[self.public_superevent.superevent_id,
self.log_dict['public_public'].N, self.tag1.name])
response = self.request_as_user(url, "DELETE")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventVOEventList(SupereventSetup, GraceDbApiTestBase):
@@ -2544,9 +2594,11 @@ class TestSupereventVOEventList(SupereventSetup, GraceDbApiTestBase):
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.lvem_user,
data=self.voevent_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to create VOEvents',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to create VOEvents',
+ status_code=403
+ )
def test_public_user_create_voevent_for_hidden_superevent(self):
"""Public user can't create VOEvents for hidden superevents"""
@@ -2560,9 +2612,11 @@ class TestSupereventVOEventList(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-voevent-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", data=self.voevent_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventVOEventDetail(SupereventSetup, GraceDbApiTestBase):
@@ -2683,8 +2737,8 @@ class TestSupereventEMObservationList(SupereventSetup, GraceDbApiTestBase):
# Define EMObservation data for POST-ing
cls.emgroup_name = 'fake_emgroup'
now = datetime.datetime.now()
- start_time_list = map(lambda i:
- (now + datetime.timedelta(seconds=i)).isoformat(), [0, 1, 2, 3])
+ start_time_list = list(map(lambda i:
+ (now + datetime.timedelta(seconds=i)).isoformat(), [0, 1, 2, 3]))
cls.emobservation_data = {
'group': cls.emgroup_name,
'ra_list': [1, 2, 3, 4],
@@ -2743,7 +2797,7 @@ class TestSupereventEMObservationList(SupereventSetup, GraceDbApiTestBase):
api_emo_nums = [emo['N'] for emo in response.data['observations']]
db_emo_nums = [emo.N for emo in
self.public_superevent.emobservation_set.all()]
- self.assertItemsEqual(api_emo_nums, db_emo_nums)
+ self.assertEqual(sorted(api_emo_nums), sorted(db_emo_nums))
def test_public_user_get_list_for_hidden_superevent(self):
"""Public user can't get any EMObservations for hidden superevents"""
@@ -2771,7 +2825,7 @@ class TestSupereventEMObservationList(SupereventSetup, GraceDbApiTestBase):
api_emo_nums = [emo['N'] for emo in response.data['observations']]
db_emo_nums = [emo.N for emo in
self.public_superevent.emobservation_set.all()]
- self.assertItemsEqual(api_emo_nums, db_emo_nums)
+ self.assertEqual(sorted(api_emo_nums), sorted(db_emo_nums))
def test_internal_user_create_emobservation(self):
"""Internal user can create EMObservations for all superevents"""
@@ -2844,8 +2898,11 @@ class TestSupereventEMObservationList(SupereventSetup, GraceDbApiTestBase):
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST",
data=self.emobservation_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventEMObservationDetail(SupereventSetup, GraceDbApiTestBase):
@@ -2945,8 +3002,8 @@ class TestSupereventFileList(SupereventSetup, GraceDbApiTestBase):
super(TestSupereventFileList, cls).setUpTestData()
# Create files for internal superevent
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_superevent, filename=cls.file1['filename'],
@@ -3088,8 +3145,8 @@ class TestSupereventFileDetail(SupereventSetup, GraceDbApiTestBase):
super(TestSupereventFileDetail, cls).setUpTestData()
# Create files for internal superevent
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_superevent, filename=cls.file1['filename'],
@@ -3307,18 +3364,22 @@ class TestSupereventGroupObjectPermissionList(SupereventSetup,
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "GET", self.lvem_user)
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to view superevent permissions',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to view superevent permissions',
+ status_code=403
+ )
# Public superevent
url = v_reverse('superevents:superevent-permission-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "GET", self.lvem_user)
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to view superevent permissions',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to view superevent permissions',
+ status_code=403
+ )
def test_public_user_get_permissions(self):
"""Public user can't get permission list"""
@@ -3341,8 +3402,11 @@ class TestSupereventGroupObjectPermissionList(SupereventSetup,
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "GET")
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventGroupObjectPermissionModify(SupereventSetup,
@@ -3356,9 +3420,11 @@ class TestSupereventGroupObjectPermissionModify(SupereventSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data={'action': 'expose'})
# Check response
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to expose superevents',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevents',
+ status_code=403
+ )
def test_internal_user_hide_exposed_superevent(self):
"""Internal user can't modify permissions to hide superevent"""
@@ -3367,9 +3433,11 @@ class TestSupereventGroupObjectPermissionModify(SupereventSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data={'action': 'hide'})
# Check response
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to hide superevents',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to hide superevents',
+ status_code=403
+ )
def test_access_manager_expose_internal_superevent(self):
"""Access manager can modify permissions to expose superevent"""
@@ -3410,8 +3478,11 @@ class TestSupereventGroupObjectPermissionModify(SupereventSetup,
response = self.request_as_user(url, "POST", self.lvem_user,
data={'action': 'hide'})
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to hide superevent', response.content)
+ self.assertContains(
+ response,
+ 'not allowed to hide superevents',
+ status_code=403
+ )
def test_public_user_modify_permissions(self):
"""Public user can't modify permissions"""
@@ -3427,8 +3498,11 @@ class TestSupereventGroupObjectPermissionModify(SupereventSetup,
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", data={'action': 'hide'})
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffList(SupereventSetup, GraceDbApiTestBase):
@@ -3477,17 +3551,21 @@ class TestSupereventSignoffList(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-signoff-list',
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to view superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to view superevent signoffs',
+ status_code=403
+ )
# Public superevent
url = v_reverse('superevents:superevent-signoff-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to view superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to view superevent signoffs',
+ status_code=403
+ )
def test_public_get_hidden_signoff_list(self):
"""Public user can't view list of signoffs for hidden superevent"""
@@ -3509,9 +3587,11 @@ class TestSupereventSignoffList(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-signoff-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "GET")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffDetail(SupereventSetup, GraceDbApiTestBase):
@@ -3570,9 +3650,11 @@ class TestSupereventSignoffDetail(SupereventSetup, GraceDbApiTestBase):
args=[self.lvem_superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to view superevent signoffs',
- response.content)
+ self.assertContains(
+ response,
+ 'do not have permission to view superevent signoffs',
+ status_code=403
+ )
# Public superevent
signoff = self.public_superevent.signoff_set.first()
@@ -3580,9 +3662,11 @@ class TestSupereventSignoffDetail(SupereventSetup, GraceDbApiTestBase):
args=[self.public_superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to view superevent signoffs',
- response.content)
+ self.assertContains(
+ response,
+ 'do not have permission to view superevent signoffs',
+ status_code=403
+ )
def test_public_user_get_signoff_detail_for_hidden_superevent(self):
"""Public user can't view signoff detail for hidden superevent"""
@@ -3609,8 +3693,11 @@ class TestSupereventSignoffDetail(SupereventSetup, GraceDbApiTestBase):
args=[self.public_superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "GET")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
@@ -3657,9 +3744,11 @@ class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
args=[self.internal_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.internal_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to create superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to create superevent signoffs',
+ status_code=403
+ )
def test_H1_control_room_create_H1_signoff(self):
"""H1 control room user can create H1 signoffs"""
@@ -3679,9 +3768,11 @@ class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
args=[self.internal_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.H1_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to do L1 signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to do L1 signoffs',
+ status_code=403
+ )
def test_advocate_create_adv_signoff(self):
"""EM advocate user can create advocate signoffs"""
@@ -3711,18 +3802,22 @@ class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.lvem_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to create superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to create superevent signoffs',
+ status_code=403
+ )
# Public superevent
url = v_reverse('superevents:superevent-signoff-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.lvem_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to create superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to create superevent signoffs',
+ status_code=403
+ )
def test_public_user_create_signoff_for_hidden_superevent(self):
"""Public user can't create signoffs for hidden superevents"""
@@ -3743,9 +3838,11 @@ class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
url = v_reverse('superevents:superevent-signoff-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
@@ -3813,9 +3910,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", self.internal_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to change superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to change superevent signoffs',
+ status_code=403
+ )
def test_H1_control_room_update_H1_signoff(self):
"""H1 control room user can update H1 signoffs"""
@@ -3842,9 +3941,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", self.H1_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to do L1 signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to do L1 signoffs',
+ status_code=403
+ )
def test_advocate_update_adv_signoff(self):
"""EM advocate user can update advocate signoffs"""
@@ -3881,9 +3982,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", self.lvem_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to change superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to change superevent signoffs',
+ status_code=403
+ )
# Public superevent
signoff = self.public_signoff
@@ -3892,9 +3995,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", self.lvem_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to change superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to change superevent signoffs',
+ status_code=403
+ )
def test_public_user_update_signoff_for_hidden_superevent(self):
"""Public user can't update signoffs for hidden superevents"""
@@ -3921,8 +4026,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
@@ -3979,9 +4087,11 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[self.internal_superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE", self.internal_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to delete superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to delete superevent signoffs',
+ status_code=403
+ )
def test_H1_control_room_delete_H1_signoff(self):
"""H1 control room user can delete H1 signoffs"""
@@ -4004,9 +4114,11 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE", self.H1_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to do L1 signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to do L1 signoffs',
+ status_code=403
+ )
def test_advocate_delete_adv_signoff(self):
"""EM advocate user can delete advocate signoffs"""
@@ -4038,9 +4150,11 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to delete superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to delete superevent signoffs',
+ status_code=403
+ )
# Public superevent
signoff = self.public_signoff
@@ -4048,9 +4162,11 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to delete superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to delete superevent signoffs',
+ status_code=403
+ )
def test_public_user_delete_signoff_for_hidden_superevent(self):
"""Public user can't delete signoffs for hidden superevents"""
@@ -4077,5 +4193,8 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
diff --git a/gracedb/api/v1/superevents/tests/test_serializers.py b/gracedb/api/v1/superevents/tests/test_serializers.py
index 809d3ea24bf985d18d5b845774b394a77463f3ed..939bf64e0c9765c7d3e458d5ff1318a01881e8e5 100644
--- a/gracedb/api/v1/superevents/tests/test_serializers.py
+++ b/gracedb/api/v1/superevents/tests/test_serializers.py
@@ -45,12 +45,12 @@ class TestSupereventSerializerViaWeb(SupereventSetup, GraceDbApiTestBase):
response = self.request_as_user(url, "GET", self.internal_user)
self.assertEqual(response.status_code, 200)
# Check data on page
- response_keys = response.json().keys()
+ response_keys = list(response.json())
response_links = response.json()['links']
self.assertIn('preferred_event', response_keys)
self.assertIn('gw_events', response_keys)
self.assertIn('em_events', response_keys)
- self.assertIn('events', response_links.keys())
+ self.assertIn('events', response_links)
def test_lvem_user_get_superevent_detail(self):
"""LV-EM user sees events link and all event graceids"""
@@ -61,12 +61,12 @@ class TestSupereventSerializerViaWeb(SupereventSetup, GraceDbApiTestBase):
response = self.request_as_user(url, "GET", self.lvem_user)
self.assertEqual(response.status_code, 200)
# Check data on page
- response_keys = response.json().keys()
+ response_keys = list(response.json())
response_links = response.json()['links']
self.assertIn('preferred_event', response_keys)
self.assertIn('gw_events', response_keys)
self.assertIn('em_events', response_keys)
- self.assertIn('events', response_links.keys())
+ self.assertIn('events', response_links)
def test_public_user_get_superevent_detail(self):
"""Public user does not see events link or all event graceids"""
@@ -77,9 +77,9 @@ class TestSupereventSerializerViaWeb(SupereventSetup, GraceDbApiTestBase):
response = self.request_as_user(url, "GET")
self.assertEqual(response.status_code, 200)
# Check data on page
- response_keys = response.json().keys()
+ response_keys = list(response.json())
response_links = response.json()['links']
self.assertNotIn('preferred_event', response_keys)
self.assertNotIn('gw_events', response_keys)
self.assertNotIn('em_events', response_keys)
- self.assertNotIn('events', response_links.keys())
+ self.assertNotIn('events', response_links)
diff --git a/gracedb/api/v1/superevents/url_templates.py b/gracedb/api/v1/superevents/url_templates.py
index 49433e0bb5fe910f418ee4a3b78b96a1eacbaa6b..9085b757a19e222e54867f7d08ac845a5630f858 100644
--- a/gracedb/api/v1/superevents/url_templates.py
+++ b/gracedb/api/v1/superevents/url_templates.py
@@ -60,12 +60,12 @@ def construct_url_templates(request=None):
# keys are like '{view_name}-template'
# values are URLs with placeholder parameters
templates = {view_name + '-template': sr(view_name, args=args)
- for view_name, args in views.iteritems()}
+ for view_name, args in views.items()}
# Replace URL placeholder parameters with string formatting placeholders
# Ex: replace 'G1234' with '{graceid}'
- for k,v in templates.iteritems():
- for pattern,placeholder in PH.iteritems():
+ for k,v in templates.items():
+ for pattern,placeholder in PH.items():
if placeholder in v:
v = v.replace(placeholder, "{{{0}}}".format(pattern))
templates[k] = v
diff --git a/gracedb/api/v1/superevents/views.py b/gracedb/api/v1/superevents/views.py
index 2575d53aff2f0332a3790d86e1fdb4a1b665f1cd..86e36582a5eeae5fdb0e243456316a0eba1129ff 100644
--- a/gracedb/api/v1/superevents/views.py
+++ b/gracedb/api/v1/superevents/views.py
@@ -15,12 +15,11 @@ from rest_framework.views import APIView
from core.file_utils import get_file_list
from core.http import check_and_serve_file
-from core.vfile import VersionedFile, FileVersionError, FileVersionNameError
+from core.vfile import FileVersionError, FileVersionNameError
from events.models import Event, Label
from events.view_utils import reverse as gracedb_reverse
from ligoauth.utils import is_internal
-from superevents.buildVOEvent import VOEventBuilderException
-from superevents.models import Superevent, Log, Signoff
+from superevents.models import Superevent, Log, Signoff, VOEvent
from superevents.utils import remove_tag_from_log, \
remove_event_from_superevent, remove_label_from_superevent, \
confirm_superevent_as_gw, get_superevent_by_date_id_or_404, \
@@ -73,7 +72,8 @@ class SupereventViewSet(SafeCreateMixin, InheritDefaultPermissionsMixin,
SupereventSearchFilter, SupereventOrderingFilter,)
ordering_fields = ('created', 't_0', 't_start', 't_end',
'preferred_event__id', 't_0_date', 'is_gw', 'base_date_number',
- 'gw_date_number', 'category')
+ 'gw_date_number', 'category',
+ 'coinc_far','em_type')
def get_serializer_class(self):
"""Select a different serializer for updates"""
@@ -321,7 +321,7 @@ class SupereventVOEventViewSet(SafeCreateMixin, InheritDefaultPermissionsMixin,
serializer_class = SupereventVOEventSerializer
pagination_class = BasePaginationFactory(results_name='voevents')
permission_classes = (SupereventVOEventModelPermissions,)
- create_error_classes = (VOEventBuilderException)
+ create_error_classes = (VOEvent.VOEventBuilderException)
lookup_url_kwarg = 'N'
lookup_field = 'N'
list_view_order_by = ('N',)
diff --git a/gracedb/core/forms.py b/gracedb/core/forms.py
index 45a5e8462fe50aea3bd9b1ef4117e8dc47edbbbc..4e49be5bdc21c74316e2c58c2c43d28c7d6174d9 100644
--- a/gracedb/core/forms.py
+++ b/gracedb/core/forms.py
@@ -37,8 +37,8 @@ class ModelFormUpdateMixin(forms.ModelForm):
# Insert instance data for missing fields only
instance_data = self.get_instance_data()
- for key in self.fields.keys():
- if not self.data.has_key(key) and instance_data[key]:
+ for key in self.fields:
+ if not key in self.data and instance_data[key]:
self.data[key] = instance_data[key]
diff --git a/gracedb/core/middleware/profiling.py b/gracedb/core/middleware/profiling.py
index 811c41aafc2a19dc5305de10d0c1d03eca918d5b..b78257d8c763cde05c586527f40748e142ccc82c 100644
--- a/gracedb/core/middleware/profiling.py
+++ b/gracedb/core/middleware/profiling.py
@@ -5,9 +5,14 @@
import sys
import os
import re
-import hotshot, hotshot.stats
import tempfile
-import StringIO
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
+
+import hotshot, hotshot.stats
+
from django.utils.deprecation import MiddlewareMixin
from django.conf import settings
@@ -91,7 +96,7 @@ class ProfileMiddleware(MiddlewareMixin):
if (settings.DEBUG or request.user.is_superuser) and 'prof' in request.GET:
self.prof.close()
- out = StringIO.StringIO()
+ out = StringIO()
old_stdout = sys.stdout
sys.stdout = out
diff --git a/gracedb/core/middleware/proxy.py b/gracedb/core/middleware/proxy.py
index 0f989656a28da60e1037d22a287041a5e2c04e48..93c05a0665a6794ced42fb3d36eef54c2923c5a3 100644
--- a/gracedb/core/middleware/proxy.py
+++ b/gracedb/core/middleware/proxy.py
@@ -11,7 +11,7 @@ class XForwardedForMiddleware(object):
def __call__(self, request):
# Process request -----------------------------------------------------
- if request.META.has_key('HTTP_X_FORWARDED_FOR'):
+ if 'HTTP_X_FORWARDED_FOR' in request.META:
request.META['REMOTE_ADDR'] = \
request.META['HTTP_X_FORWARDED_FOR'].split(",")[0].strip()
diff --git a/gracedb/core/models.py b/gracedb/core/models.py
index b3f36fb00ddff9cdcb694e81f41e6d88f079dd6a..7d346f658520fd4dd199eb374fed57a715abd162 100644
--- a/gracedb/core/models.py
+++ b/gracedb/core/models.py
@@ -115,7 +115,7 @@ class AutoIncrementModel(models.Model):
qn = compiler.quote_name_unless_alias
# Compile multiple constraints with AND
- constraint_fields = map(meta.get_field, self.AUTO_CONSTRAINTS)
+ constraint_fields = list(map(meta.get_field, self.AUTO_CONSTRAINTS))
constraint_list = ["{0}=%s".format(qn(f.column))
for f in constraint_fields]
constraint_values = [f.get_db_prep_value(getattr(self, f.column),
diff --git a/gracedb/core/time_utils.py b/gracedb/core/time_utils.py
index 6c1f93f5f28a4a05b1070769bc0b8439f476c2af..360dec80861982f1e978e0adfe2aafea4c127f57 100644
--- a/gracedb/core/time_utils.py
+++ b/gracedb/core/time_utils.py
@@ -14,7 +14,7 @@ import calendar
gpsEpoch = calendar.timegm((1980, 1, 6, 0, 0, 0, 0, 0, 0))
-leapSeconds = map(calendar.timegm, [
+leapSeconds = list(map(calendar.timegm, [
(1981, 7, 0, 0, 0, 0, 0, 0, 0),
(1982, 7, 0, 0, 0, 0, 0, 0, 0),
(1983, 7, 0, 0, 0, 0, 0, 0, 0),
@@ -33,7 +33,7 @@ leapSeconds = map(calendar.timegm, [
(2012, 7, 0, 0, 0, 0, 0, 0, 0),
(2015, 7, 0, 0, 0, 0, 0, 0, 0),
(2017, 1, 0, 0, 0, 0, 0, 0, 0),
-])
+]))
def gpsToPosixTime(gpsTime):
if gpsTime is None:
@@ -68,28 +68,10 @@ def gpsToUtc(gpsTime):
t = gpsToPosixTime(gpsTime)
return datetime.datetime.fromtimestamp(t, pytz.utc)
-def isoToGps(t):
- # The input is a string in ISO time format: 2012-10-28T05:04:31.91
- # First strip out whitespace, then split off the factional
- # second. We'll add that back later.
- if not t:
- return None
- t=t.strip()
- ISOTime = t.split('.')[0]
- ISOTime = datetime.datetime.strptime(ISOTime,"%Y-%m-%dT%H:%M:%S")
- # Need to set UTC time zone or this is interpreted as local time.
- ISOTime = ISOTime.replace(tzinfo=pytz.utc)
- sec_substr = t.split('.')[1]
- if sec_substr:
- fracSec = float('0.' + sec_substr)
- else:
- fracSec = 0
- posixTime = calendar.timegm(ISOTime.utctimetuple()) + fracSec
- return int(round(posixToGpsTime(posixTime)))
def isoToGpsFloat(t):
# The input is a string in ISO time format: 2012-10-28T05:04:31.91
- # First strip out whitespace, then split off the factional
+ # First strip out whitespace, then split off the fractional
# second. We'll add that back later.
if not t:
return None
@@ -105,3 +87,12 @@ def isoToGpsFloat(t):
fracSec = 0
posixTime = calendar.timegm(ISOTime.utctimetuple()) + fracSec
return posixToGpsTime(posixTime)
+
+
+def isoToGps(t):
+ return int(round(isoToGpsFloat(t)))
+
+
+def utc_datetime_to_gps_float(dt):
+ posix_time = calendar.timegm(dt.timetuple()) + (dt.microsecond * 1e-6)
+ return posixToGpsTime(posix_time)
diff --git a/gracedb/core/utils.py b/gracedb/core/utils.py
index 40fe4e781b345798815dbc919ec7ace070692d48..0c629ea0eaff4398c481e3ae16588ed9139d6b68 100644
--- a/gracedb/core/utils.py
+++ b/gracedb/core/utils.py
@@ -19,7 +19,7 @@ def int_to_letters(num, positive_only=True):
"""
# Argument checking
- if not isinstance(num, (int, long)):
+ if not isinstance(num, int):
# Coerce to int
logger.warning('Coercing argument of type {0} to int'.format(
type(num)))
diff --git a/gracedb/core/vfile.py b/gracedb/core/vfile.py
index 4ff97d3cd6ef843e1ffab8a94b16b3ffb4829728..a362993b9b701ee9e9c248d23dce9e8f380644de 100644
--- a/gracedb/core/vfile.py
+++ b/gracedb/core/vfile.py
@@ -24,7 +24,7 @@ class FileVersionNameError(Exception):
pass
-class VersionedFile(file):
+class VersionedFile(object):
"""
Open a versioned file.
@@ -74,7 +74,7 @@ class VersionedFile(file):
# one scoped inside of this __init__). But I'm reluctant to mess with
# Brian's code too much.
self.version = version
- file.__init__(self, actual_name, *args, **kwargs)
+ self.file = open(actual_name, *args, **kwargs)
# Otherwise...
@@ -122,13 +122,13 @@ class VersionedFile(file):
# os.O_EXCL causes the open to fail if the file already exists.
fd = os.open(actual_name,
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
- 0644)
+ 0o644)
# re-open
- file.__init__(self, actual_name, *args, **kwargs)
+ self.file = open(actual_name, *args, **kwargs)
# lose fd we used to ensure file creation.
os.close(fd)
break
- except OSError, e:
+ except OSError as e:
if e.errno != errno.EEXIST:
raise
version += 1
@@ -183,7 +183,7 @@ class VersionedFile(file):
try:
# XXX Another race condition. File will not exist for a very brief time.
os.unlink(self.fullname)
- except OSError, e:
+ except OSError as e:
# Do not care if file does not exist, otherwise raise exception.
if e.errno != errno.ENOENT:
raise
@@ -208,6 +208,13 @@ class VersionedFile(file):
return [int(f.split(',')[1])
for f in os.listdir(d) if f.startswith(name + ',')]
+ def write(self, s):
+ self.file.write(s)
+
+ @property
+ def closed(self):
+ return self.file.closed
+
def close(self):
if self.writing:
# no need to update symlink if we were only reading.
@@ -215,13 +222,13 @@ class VersionedFile(file):
# file -- trying to discover the lastest version fails
# painfully. (max(known_versions()) => max([]))
self._repoint_symlink()
- if not self.closed:
- file.close(self)
+ if not self.file.closed:
+ self.file.close()
def __del__(self):
# XXX file does not have a __del__ method. Should we?
- if not self.closed:
- self.close()
+ if not self.file.closed:
+ self.file.close()
@staticmethod
def guess_mimetype(filename):
@@ -243,18 +250,24 @@ class VersionedFile(file):
def create_versioned_file(filename, file_dir, file_contents):
-
# Get full file path
full_path = os.path.join(file_dir, filename)
# Create file
- fdest = VersionedFile(full_path, 'w')
if isinstance(file_contents, six.string_types):
+ fdest = VersionedFile(full_path, 'w')
+ fdest.write(file_contents)
+ elif isinstance(file_contents, bytes):
+ fdest = VersionedFile(full_path, 'wb')
fdest.write(file_contents)
elif isinstance(file_contents, (UploadedFile, InMemoryUploadedFile,
TemporaryUploadedFile, SimpleUploadedFile)):
+ fdest = VersionedFile(full_path, 'wb')
for chunk in file_contents.chunks():
fdest.write(chunk)
+ else:
+ raise TypeError('Unexpected file contents in '
+ 'core.vfile.create_versioned_file')
fdest.close()
return fdest.version
diff --git a/gracedb/events/buildVOEvent.py b/gracedb/events/buildVOEvent.py
deleted file mode 100644
index 1ef4f27e3375c1f30e16440dd3893e0bdd771b4d..0000000000000000000000000000000000000000
--- a/gracedb/events/buildVOEvent.py
+++ /dev/null
@@ -1,482 +0,0 @@
-
-# Taken from VOEventLib example code, which is:
-# Copyright 2010 Roy D. Williams
-# then modified
-"""
-buildVOEvent: Creates a complex VOEvent with tables
-See the VOEvent specification for details
-http://www.ivoa.net/Documents/latest/VOEvent.html
-"""
-from scipy.constants import c, G, pi
-
-from VOEventLib.VOEvent import VOEvent, Who, Author, Param, How, What, Group
-from VOEventLib.VOEvent import Citations, EventIVORN
-#from VOEventLib.VOEvent import Why
-#from VOEventLib.Vutil import makeWhereWhen, stringVOEvent
-from VOEventLib.Vutil import stringVOEvent
-
-from VOEventLib.VOEvent import AstroCoords, AstroCoordSystem
-from VOEventLib.VOEvent import ObservationLocation, ObservatoryLocation
-from VOEventLib.VOEvent import ObsDataLocation, WhereWhen
-from VOEventLib.VOEvent import Time, TimeInstant
-
-from core.time_utils import gpsToUtc
-from core.urls import build_absolute_uri
-from django.utils import timezone
-from django.conf import settings
-from django.urls import reverse
-from .models import CoincInspiralEvent, MultiBurstEvent
-from .models import VOEvent as GraceDBVOEvent
-from .models import LalInferenceBurstEvent
-
-import os
-
-class VOEventBuilderException(Exception):
- pass
-
-# Used to create the Packet_Type parameter block
-PACKET_TYPES = {
- GraceDBVOEvent.VOEVENT_TYPE_PRELIMINARY: (150, 'LVC_PRELIMINARY'),
- GraceDBVOEvent.VOEVENT_TYPE_INITIAL: (151, 'LVC_INITIAL'),
- GraceDBVOEvent.VOEVENT_TYPE_UPDATE: (152, 'LVC_UPDATE'),
- GraceDBVOEvent.VOEVENT_TYPE_RETRACTION: (164, 'LVC_RETRACTION'),
-}
-
-VOEVENT_TYPE_DICT = dict(GraceDBVOEvent.VOEVENT_TYPE_CHOICES)
-
-def get_voevent_type(short_name):
- for t in GraceDBVOEvent.VOEVENT_TYPE_CHOICES:
- if short_name in t:
- return t[1]
- return None
-
-def buildVOEvent(event, voevent, request=None):
-
-# XXX Branson commenting out. Reed's MDC events do not have FAR for some reason.
-# if not event.far:
-# raise VOEventBuilderException("Cannot build a VOEvent because event has no FAR.")
-
- if not event.gpstime:
- raise VOEventBuilderException("Cannot build a VOEvent because event has no gpstime.")
-
- if not voevent.voevent_type in VOEVENT_TYPE_DICT.keys():
- raise VOEventBuilderException("voevent_type must be preliminary, initial, update, or retraction")
-
- # Let's convert that voevent_type to something nicer looking
- voevent_type = VOEVENT_TYPE_DICT[voevent.voevent_type]
-
- objid = event.graceid
-
- # Now build the IVORN.
- type_string = voevent_type.capitalize()
- event_id = "%s-%d-%s" % (objid, voevent.N, type_string)
- ivorn = settings.IVORN_PREFIX + event_id
-
- ############ VOEvent header ############################
- v = VOEvent(version="2.0")
- v.set_ivorn(ivorn)
-
- if event.search and event.search.name == 'MDC':
- v.set_role("test")
- elif event.group.name == 'Test':
- v.set_role("test")
- else:
- v.set_role("observation")
- if voevent_type != 'retraction':
- v.set_Description(settings.SKYALERT_DESCRIPTION)
-
- ############ Who ############################
- w = Who()
- a = Author()
- a.add_contactName("LIGO Scientific Collaboration and Virgo Collaboration")
- #a.add_contactEmail("postmaster@ligo.org")
- w.set_Author(a)
- w.set_Date(timezone.now().strftime("%Y-%m-%dT%H:%M:%S"))
- v.set_Who(w)
-
- ############ Why ############################
- # Moving this information into the 'How' section.
- #if voevent_type != 'retraction':
- # y = Why()
- # y.add_Description("Candidate gravitational wave event identified by low-latency analysis")
- # v.set_Why(y)
-
- ############ How ############################
-
- if voevent_type != 'retraction':
- h = How()
- h.add_Description("Candidate gravitational wave event identified by low-latency analysis")
- instruments = event.instruments.split(',')
- if 'H1' in instruments:
- h.add_Description("H1: LIGO Hanford 4 km gravitational wave detector")
- if 'L1' in instruments:
- h.add_Description("L1: LIGO Livingston 4 km gravitational wave detector")
- if 'V1' in instruments:
- h.add_Description("V1: Virgo 3 km gravitational wave detector")
- if int(voevent.coinc_comment) == 1:
- h.add_Description("A gravitational wave trigger identified a possible counterpart GRB")
- v.set_How(h)
-
- ############ What ############################
- w = What()
-
- # UCD = Unified Content Descriptors
- # http://monet.uni-sw.gwdg.de/twiki/bin/view/VOEvent/UnifiedContentDescriptors
- # OR -- (from VOTable document, [21] below)
- # http://www.ivoa.net/twiki/bin/view/IVOA/IvoaUCD
- # http://cds.u-strasbg.fr/doc/UCD.htx
- #
- # which somehow gets you to: http://www.ivoa.net/Documents/REC/UCD/UCDlist-20070402.html
- # where you might find some actual information.
-
- # Unit / Section 4.3 of [21] which relies on [25]
- # [21] http://www.ivoa.net/Documents/latest/VOT.html
- # [25] http://vizier.u-strasbg.fr/doc/catstd-3.2.htx
- #
- # basically, a string that makes sense to humans about what units a value is. eg. "m/s"
-
- # Add Packet_Type for GCNs
- w.add_Param(Param(name="Packet_Type",
- value=PACKET_TYPES[voevent.voevent_type][0], dataType="int",
- Description=[("The Notice Type number is assigned/used within GCN, eg "
- "type={typenum} is an {typedesc} notice").format(
- typenum=PACKET_TYPES[voevent.voevent_type][0],
- typedesc=PACKET_TYPES[voevent.voevent_type][1])]))
-
- # Whether the alert is internal or not
- w.add_Param(Param(name="internal", value=int(voevent.internal),
- dataType="int", Description=['Indicates whether this event should be '
- 'distributed to LSC/Virgo members only']))
-
- # The serial number
- w.add_Param(Param(name="Pkt_Ser_Num", value=voevent.N,
- Description=["A number that increments by 1 each time a new revision "
- "is issued for this event"]))
-
- # The GraceID
- w.add_Param(Param(name="GraceID",
- dataType="string",
- ucd="meta.id",
- value=objid,
- Description=["Identifier in GraceDB"]))
-
- # Alert type parameter
- w.add_Param(Param(name="AlertType",
- dataType="string",
- ucd="meta.version",
- value = voevent_type.capitalize(),
- Description=["VOEvent alert type"]))
-
- # Shib protected event page
- # Whether the event is a hardware injection or not
- w.add_Param(Param(name="HardwareInj",
- dataType="int",
- ucd="meta.number",
- value=int(voevent.hardware_inj),
- Description=['Indicates that this event is a hardware injection if 1, no if 0']))
-
- w.add_Param(Param(name="OpenAlert",
- dataType="int",
- ucd="meta.number",
- value=int(voevent.open_alert),
- Description=['Indicates that this event is an open alert if 1, no if 0']))
-
- w.add_Param(Param(name="EventPage",
- ucd="meta.ref.url",
- value=build_absolute_uri(reverse('view', args=[objid]), request),
- Description=["Web page for evolving status of this candidate event"]))
-
- if voevent_type != 'retraction':
- # Instruments
- w.add_Param(Param(name="Instruments",
- dataType="string",
- ucd="meta.code",
- value=event.instruments,
- Description=["List of instruments used in analysis to identify this event"]))
-
- # False alarm rate
- if event.far:
- w.add_Param(Param(name="FAR",
- dataType="float",
- ucd="arith.rate;stat.falsealarm",
- unit="Hz",
- value=float(max(event.far, settings.VOEVENT_FAR_FLOOR)),
- Description=["False alarm rate for GW candidates with this strength or greater"]))
-
- # Group
- w.add_Param(Param(name="Group",
- dataType="string",
- ucd="meta.code",
- value=event.group.name,
- Description=["Data analysis working group"]))
-
- # Pipeline
- w.add_Param(Param(name="Pipeline",
- dataType="string",
- ucd="meta.code",
- value=event.pipeline.name,
- Description=["Low-latency data analysis pipeline"]))
-
- # Search
- if event.search:
- w.add_Param(Param(name="Search",
- ucd="meta.code",
- dataType="string",
- value=event.search.name,
- Description=["Specific low-latency search"]))
-
- # initial and update VOEvents must have a skymap.
- # new feature (10/24/5/2016): preliminary VOEvents can have a skymap,
- # but they don't have to.
- if (voevent_type in ["initial", "update"] or
- (voevent_type == "preliminary" and voevent.skymap_filename != None)):
- if not voevent.skymap_filename:
- raise VOEventBuilderException("Skymap filename not provided.")
-
- fits_name = voevent.skymap_filename
- fits_path = os.path.join(event.datadir, fits_name)
- if not os.path.exists(fits_path):
- raise VOEventBuilderException("Skymap file does not exist: %s" % voevent.skymap_filename)
-
- if not voevent.skymap_type:
- raise VOEventBuilderException("Skymap type must be provided.")
-
- # Skymaps. Create group and set fits file name
- g = Group('GW_SKYMAP', voevent.skymap_type)
-
- fits_skymap_url = build_absolute_uri(reverse(
- "api:default:events:files", args=[objid, fits_name]), request)
-
- # Add parameters to the skymap group
- g.add_Param(Param(name="skymap_fits", dataType="string",
- ucd="meta.ref.url", value=fits_skymap_url,
- Description=["Sky Map FITS"]))
-
- w.add_Group(g)
-
- # Analysis specific attributes
- if voevent_type != 'retraction':
- classification_group = Group('Classification', Description=["Source "
- "classification: binary neutron star (BNS), neutron star-black "
- "hole (NSBH), binary black hole (BBH), MassGap, or terrestrial "
- "(noise)"])
- properties_group = Group('Properties', Description=["Qualitative "
- "properties of the source, conditioned on the assumption that the "
- "signal is an astrophysical compact binary merger"])
- if isinstance(event,CoincInspiralEvent) and voevent_type != 'retraction':
- # get mchirp and mass
- mchirp = float(event.mchirp)
- mass = float(event.mass)
- # calculate eta = (mchirp/total_mass)**(5/3)
- eta = pow((mchirp/mass),5.0/3.0)
-
- # EM-Bright mass classifier information for CBC event candidates
- if voevent.prob_bns is not None:
- classification_group.add_Param(Param(name="BNS",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_bns, Description=["Probability that "
- "the source is a binary neutron star merger (both objects "
- "lighter than 3 solar masses)"]))
-
- if voevent.prob_nsbh is not None:
- classification_group.add_Param(Param(name="NSBH",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_nsbh, Description=["Probability that "
- "the source is a neutron star-black hole merger (primary "
- "heavier than 5 solar masses, secondary lighter than 3 "
- "solar masses)"]))
-
- if voevent.prob_bbh is not None:
- classification_group.add_Param(Param(name="BBH",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_bbh, Description=["Probability that "
- "the source is a binary black hole merger (both objects "
- "heavier than 5 solar masses)"]))
-
- if voevent.prob_mass_gap is not None:
- classification_group.add_Param(Param(name="MassGap",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_mass_gap, Description=["Probability "
- "that the source has at least one object between 3 and 5 "
- "solar masses"]))
-
- if voevent.prob_terrestrial is not None:
- classification_group.add_Param(Param(name="Terrestrial",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_terrestrial, Description=["Probability "
- "that the source is terrestrial (i.e., a background noise "
- "fluctuation or a glitch)"]))
-
- # Add to source properties group
- if voevent.prob_has_ns is not None:
- properties_group.add_Param(Param(name="HasNS",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_has_ns,
- Description=["Probability that at least one object in the "
- "binary has a mass that is less than 3 solar masses"]))
-
- if voevent.prob_has_remnant is not None:
- properties_group.add_Param(Param(name="HasRemnant",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_has_remnant, Description=["Probability "
- "that a nonzero mass was ejected outside the central "
- "remnant object"]))
-
- # build up MaxDistance. event.singleinspiral_set.all()?
- # Each detector calculates an effective distance assuming the inspiral is
- # optimally oriented. It is the maximum distance at which a source of the
- # given parameters would've been seen by that particular detector. To get
- # an effective 'maximum distance', we just find the minumum over detectors
- max_distance = float('inf')
- for obj in event.singleinspiral_set.all():
- if obj.eff_distance < max_distance:
- max_distance = obj.eff_distance
-# if max_distance < float('inf'):
-# w.add_Param(Param(name="MaxDistance",
-# dataType="float",
-# ucd="pos.distance",
-# unit="Mpc",
-# value=max_distance,
-# Description=["Estimated maximum distance for CBC event"]))
-
- elif isinstance(event,MultiBurstEvent):
- w.add_Param(Param(name="CentralFreq",
- dataType="float",
- ucd="gw.frequency",
- unit="Hz",
- value=float(event.central_freq),
- Description=["Central frequency of GW burst signal"]))
- w.add_Param(Param(name="Duration",
- dataType="float",
- ucd="time.duration",
- unit="s",
- value=float(event.duration),
- Description=["Measured duration of GW burst signal"]))
-
- # XXX Calculate the fluence. Unfortunately, this requires parsing the trigger.txt
- # file for hrss values. These should probably be pulled into the database.
- # But there is no consensus on whether hrss or fluence is meaningful. So I will
- # put off changing the schema for now.
- try:
- # Go find the data file.
- log = event.eventlog_set.filter(comment__startswith="Original Data").all()[0]
- filename = log.filename
- filepath = os.path.join(event.datadir,filename)
- if os.path.isfile(filepath):
- datafile = open(filepath,"r")
- else:
- raise Exception("No file found.")
- # Now parse the datafile.
- # The line we want looks like:
- # hrss: 1.752741e-23 2.101590e-23 6.418900e-23
- for line in datafile:
- if line.startswith('hrss:'):
- hrss_values = [float(hrss) for hrss in line.split()[1:]]
- max_hrss = max(hrss_values)
- # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
- # Note that hrss here actually has units of s^(-1/2)
- fluence = pi * pow(c,3) * pow(event.central_freq,2)
- fluence = fluence * pow(max_hrss,2)
- fluence = fluence / (4.0*G)
-
- w.add_Param(Param(name="Fluence",
- dataType="float",
- ucd="gw.fluence",
- unit="erg/cm^2",
- value=fluence,
- Description=["Estimated fluence of GW burst signal"]))
- except Exception:
- pass
- elif isinstance(event,LalInferenceBurstEvent):
- w.add_Param(Param(name="frequency",
- dataType="float",
- ucd="gw.frequency",
- unit="Hz",
- value=float(event.frequency_mean),
- Description=["Mean frequency of GW burst signal"]))
-
- # Calculate the fluence.
- # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
- # Note that hrss here actually has units of s^(-1/2)
- # XXX obviously need to refactor here.
- try:
- fluence = pi * pow(c,3) * pow(event.frequency,2)
- fluence = fluence * pow(event.hrss,2)
- fluence = fluence / (4.0*G)
-
- w.add_Param(Param(name="Fluence",
- dataType="float",
- ucd="gw.fluence",
- unit="erg/cm^2",
- value=fluence,
- Description=["Estimated fluence of GW burst signal"]))
- except:
- pass
- else:
- pass
- # Add Groups to What block
- w.add_Group(classification_group)
- w.add_Group(properties_group)
-
- v.set_What(w)
-
- ############ Wherewhen ############################
-# The old way of making the WhereWhen section led to a pointless position
-# location.
-# wwd = {'observatory': 'LIGO Virgo',
-# 'coord_system': 'UTC-FK5-GEO',
-# # XXX time format
-# 'time': str(gpsToUtc(event.gpstime).isoformat())[:-6], #'1918-11-11T11:11:11',
-# #'timeError': 1.0,
-# 'longitude': 0.0,
-# 'latitude': 0.0,
-# 'positionalError': 180.0,
-# }
-#
-# ww = makeWhereWhen(wwd)
-# if ww: v.set_WhereWhen(ww)
-
- coord_system_id = 'UTC-FK5-GEO'
- event_time = str(gpsToUtc(event.gpstime).isoformat())[:-6]
- observatory_id = 'LIGO Virgo'
- ac = AstroCoords(coord_system_id=coord_system_id)
- acs = AstroCoordSystem(id=coord_system_id)
- ac.set_Time(Time(TimeInstant = TimeInstant(event_time)))
-
- onl = ObservationLocation(acs, ac)
- oyl = ObservatoryLocation(id=observatory_id)
- odl = ObsDataLocation(oyl, onl)
- ww = WhereWhen()
- ww.set_ObsDataLocation(odl)
- v.set_WhereWhen(ww)
-
- ############ Citation ############################
- if event.voevent_set.count()>1:
- c = Citations()
- for ve in event.voevent_set.all():
- # Oh, actually we need to exclude *this* voevent.
- if ve.N == voevent.N:
- continue
- if voevent_type == 'initial':
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Initial localization is now available')
- elif voevent_type == 'update':
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Updated localization is now available')
- elif voevent_type == 'retraction':
- ei = EventIVORN('retraction', ve.ivorn)
- c.set_Description('Determined to not be a viable GW event candidate')
- elif voevent_type == 'preliminary':
- # For cases when an additional preliminary VOEvent is sent
- # in order to add a preliminary skymap.
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Initial localization is now available (preliminary)')
- c.add_EventIVORN(ei)
-
- v.set_Citations(c)
-
- ############ output the event ############################
- xml = stringVOEvent(v)
- #schemaURL = "http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd")
- return xml, ivorn
-
diff --git a/gracedb/events/migrations/0005_initial_label_data.py b/gracedb/events/migrations/0005_initial_label_data.py
index c1e5e92c2053fa3fdd85573f4a3f046befc81b31..1b4a1520092547f66317f16c77707692037ec88f 100644
--- a/gracedb/events/migrations/0005_initial_label_data.py
+++ b/gracedb/events/migrations/0005_initial_label_data.py
@@ -53,8 +53,8 @@ def remove_labels(apps, schema_editor):
try:
l = Label.objects.get(name=label_dict['name'])
except Label.DoesNotExist:
- print('Label {0} not found to be deleted, skipping.' \
- .format(label_dict['name']))
+ print(('Label {0} not found to be deleted, skipping.' \
+ .format(label_dict['name'])))
break
l.delete()
diff --git a/gracedb/events/migrations/0011_add_O2VirgoTest_search.py b/gracedb/events/migrations/0011_add_O2VirgoTest_search.py
index 8c625f3498d8f7c402804df6441f8280152d8412..3bb7f6cb35319286173957ae06765617fa43ddd5 100644
--- a/gracedb/events/migrations/0011_add_O2VirgoTest_search.py
+++ b/gracedb/events/migrations/0011_add_O2VirgoTest_search.py
@@ -18,7 +18,7 @@ def add_search(apps, schema_editor):
# Create search
new_search, created = Search.objects.get_or_create(name=SEARCH['name'])
if created:
- for key in SEARCH.keys():
+ for key in SEARCH:
setattr(new_search, key, SEARCH[key])
new_search.save()
diff --git a/gracedb/events/migrations/0036_populate_voevent_fields.py b/gracedb/events/migrations/0036_populate_voevent_fields.py
index f5da68f22e319554e3eb894bdf29c81dfd1f6ba3..32ac1bacbf9c3ce7e799bf6d4b4b6140fd029dbe 100644
--- a/gracedb/events/migrations/0036_populate_voevent_fields.py
+++ b/gracedb/events/migrations/0036_populate_voevent_fields.py
@@ -1,7 +1,10 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.20 on 2019-06-10 16:58
from __future__ import unicode_literals
-from cStringIO import StringIO
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
from hashlib import sha1
import os
from lxml import etree
@@ -66,7 +69,7 @@ def get_datadir(event_or_superevent):
hash_input = str(event_or_superevent.id)
if event_or_superevent.__class__.__name__.lower() == 'superevent':
hash_input = 'superevent' + hash_input
- hdf = StringIO(sha1(hash_input).hexdigest())
+ hdf = StringIO(sha1(hash_input.encode()).hexdigest())
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
diff --git a/gracedb/events/migrations/0040_auto_20190919_1957.py b/gracedb/events/migrations/0040_auto_20190919_1957.py
new file mode 100644
index 0000000000000000000000000000000000000000..c201cdb3ec9e93ba8703581b8b01b8ed647a630c
--- /dev/null
+++ b/gracedb/events/migrations/0040_auto_20190919_1957.py
@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-09-19 19:57
+# This was auto-generated after moving to Python 3, with no changes to the
+# actual models. See the commit message for more details.
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('events', '0039_specify_pipeline_types'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='coincinspiralevent',
+ name='ifos',
+ field=models.CharField(default='', max_length=20),
+ ),
+ migrations.AlterField(
+ model_name='embbeventlog',
+ name='eel_status',
+ field=models.CharField(choices=[('FO', 'FOOTPRINT'), ('SO', 'SOURCE'), ('CO', 'COMMENT'), ('CI', 'CIRCULAR')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='embbeventlog',
+ name='obs_status',
+ field=models.CharField(choices=[('NA', 'NOT APPLICABLE'), ('OB', 'OBSERVATION'), ('TE', 'TEST'), ('PR', 'PREDICTION')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='embbeventlog',
+ name='waveband',
+ field=models.CharField(choices=[('em.gamma', 'Gamma rays part of the spectrum'), ('em.gamma.soft', 'Soft gamma ray (120 - 500 keV)'), ('em.gamma.hard', 'Hard gamma ray (>500 keV)'), ('em.X-ray', 'X-ray part of the spectrum'), ('em.X-ray.soft', 'Soft X-ray (0.12 - 2 keV)'), ('em.X-ray.medium', 'Medium X-ray (2 - 12 keV)'), ('em.X-ray.hard', 'Hard X-ray (12 - 120 keV)'), ('em.UV', 'Ultraviolet part of the spectrum'), ('em.UV.10-50nm', 'Ultraviolet between 10 and 50 nm'), ('em.UV.50-100nm', 'Ultraviolet between 50 and 100 nm'), ('em.UV.100-200nm', 'Ultraviolet between 100 and 200 nm'), ('em.UV.200-300nm', 'Ultraviolet between 200 and 300 nm'), ('em.UV.FUV', 'Far-Infrared, 30-100 microns'), ('em.opt', 'Optical part of the spectrum'), ('em.opt.U', 'Optical band between 300 and 400 nm'), ('em.opt.B', 'Optical band between 400 and 500 nm'), ('em.opt.V', 'Optical band between 500 and 600 nm'), ('em.opt.R', 'Optical band between 600 and 750 nm'), ('em.opt.I', 'Optical band between 750 and 1000 nm'), ('em.IR', 'Infrared part of the spectrum'), ('em.IR.NIR', 'Near-Infrared, 1-5 microns'), ('em.IR.J', 'Infrared between 1.0 and 1.5 micron'), ('em.IR.H', 'Infrared between 1.5 and 2 micron'), ('em.IR.K', 'Infrared between 2 and 3 micron'), ('em.IR.MIR', 'Medium-Infrared, 5-30 microns'), ('em.IR.3-4um', 'Infrared between 3 and 4 micron'), ('em.IR.4-8um', 'Infrared between 4 and 8 micron'), ('em.IR.8-15um', 'Infrared between 8 and 15 micron'), ('em.IR.15-30um', 'Infrared between 15 and 30 micron'), ('em.IR.30-60um', 'Infrared between 30 and 60 micron'), ('em.IR.60-100um', 'Infrared between 60 and 100 micron'), ('em.IR.FIR', 'Far-Infrared, 30-100 microns'), ('em.mm', 'Millimetric part of the spectrum'), ('em.mm.1500-3000GHz', 'Millimetric between 1500 and 3000 GHz'), ('em.mm.750-1500GHz', 'Millimetric between 750 and 1500 GHz'), ('em.mm.400-750GHz', 'Millimetric between 400 and 750 GHz'), ('em.mm.200-400GHz', 'Millimetric between 200 and 400 GHz'), ('em.mm.100-200GHz', 'Millimetric between 100 and 200 GHz'), ('em.mm.50-100GHz', 'Millimetric between 50 and 100 GHz'), ('em.mm.30-50GHz', 'Millimetric between 30 and 50 GHz'), ('em.radio', 'Radio part of the spectrum'), ('em.radio.12-30GHz', 'Radio between 12 and 30 GHz'), ('em.radio.6-12GHz', 'Radio between 6 and 12 GHz'), ('em.radio.3-6GHz', 'Radio between 3 and 6 GHz'), ('em.radio.1500-3000MHz', 'Radio between 1500 and 3000 MHz'), ('em.radio.750-1500MHz', 'Radio between 750 and 1500 MHz'), ('em.radio.400-750MHz', 'Radio between 400 and 750 MHz'), ('em.radio.200-400MHz', 'Radio between 200 and 400 MHz'), ('em.radio.100-200MHz', 'Radio between 100 and 200 MHz'), ('em.radio.20-100MHz', 'Radio between 20 and 100 MHz')], max_length=25),
+ ),
+ migrations.AlterField(
+ model_name='event',
+ name='instruments',
+ field=models.CharField(default='', max_length=20),
+ ),
+ migrations.AlterField(
+ model_name='eventlog',
+ name='filename',
+ field=models.CharField(blank=True, default='', max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='label',
+ name='defaultColor',
+ field=models.CharField(default='black', max_length=20),
+ ),
+ migrations.AlterField(
+ model_name='multiburstevent',
+ name='ifos',
+ field=models.CharField(default='', max_length=20),
+ ),
+ migrations.AlterField(
+ model_name='multiburstevent',
+ name='single_ifo_times',
+ field=models.CharField(default='', max_length=255),
+ ),
+ migrations.AlterField(
+ model_name='pipeline',
+ name='pipeline_type',
+ field=models.CharField(choices=[('E', 'external'), ('O', 'other'), ('SO', 'non-production search'), ('SP', 'production search')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='pipelinelog',
+ name='action',
+ field=models.CharField(choices=[('D', 'disable'), ('E', 'enable')], max_length=10),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='instrument',
+ field=models.CharField(blank=True, choices=[('H1', 'LHO'), ('L1', 'LLO'), ('V1', 'Virgo')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='signoff_type',
+ field=models.CharField(choices=[('OP', 'operator'), ('ADV', 'advocate')], max_length=3),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='status',
+ field=models.CharField(choices=[('OK', 'OKAY'), ('NO', 'NOT OKAY')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='destination_channel',
+ field=models.CharField(blank=True, default='', max_length=50, null=True),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='numrel_data',
+ field=models.CharField(blank=True, default='', max_length=50),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='source',
+ field=models.CharField(blank=True, default='', max_length=50),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='source_channel',
+ field=models.CharField(blank=True, default='', max_length=50, null=True),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='taper',
+ field=models.CharField(blank=True, default='', max_length=50),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='waveform',
+ field=models.CharField(blank=True, default='', max_length=50),
+ ),
+ migrations.AlterField(
+ model_name='tag',
+ name='name',
+ field=models.CharField(max_length=100, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_tag_name', message='Tag names can only include [0-9a-zA-z_-]', regex='^[0-9a-zA-Z_\\-]*$')]),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='filename',
+ field=models.CharField(blank=True, default='', editable=False, max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=200),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='voevent_type',
+ field=models.CharField(choices=[('PR', 'preliminary'), ('IN', 'initial'), ('UP', 'update'), ('RE', 'retraction')], max_length=2),
+ ),
+ ]
diff --git a/gracedb/events/migrations/0041_add_raven_voevent_fields.py b/gracedb/events/migrations/0041_add_raven_voevent_fields.py
new file mode 100644
index 0000000000000000000000000000000000000000..c81bfdbc9bc5bbbe0d7b20656b90d34176a84b3e
--- /dev/null
+++ b/gracedb/events/migrations/0041_add_raven_voevent_fields.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-11-12 21:18
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('events', '0040_auto_20190919_1957'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='voevent',
+ name='combined_skymap_filename',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='delta_t',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(-1000), django.core.validators.MaxValueValidator(1000)]),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_gcn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_pipeline',
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_search',
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='raven_coinc',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='space_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='time_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
+ ),
+ ]
diff --git a/gracedb/events/migrations/0042_auto_20191119_1730.py b/gracedb/events/migrations/0042_auto_20191119_1730.py
new file mode 100644
index 0000000000000000000000000000000000000000..0841069715291bccb85abd79518e460ac1933ff2
--- /dev/null
+++ b/gracedb/events/migrations/0042_auto_20191119_1730.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-11-19 17:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('events', '0041_add_raven_voevent_fields'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=300),
+ ),
+ ]
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index 2b4f11ed0e1e18e500286ad5961b738ee7c1ca56..6413e548d44b7d5a5eed2e58f6d0c452ec786c39 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -4,6 +4,8 @@ import numbers
from django.db import models, IntegrityError
from django.urls import reverse
from django.core.exceptions import ValidationError
+from django.utils import six
+from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from model_utils.managers import InheritanceManager
@@ -33,7 +35,11 @@ from django.conf import settings
import pytz
import calendar
-from cStringIO import StringIO
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
+
from hashlib import sha1
import shutil
@@ -50,25 +56,16 @@ SERVER_TZ = pytz.timezone(settings.TIME_ZONE)
#
schema_version = "1.1"
-#class User(models.Model):
- #name = models.CharField(max_length=100)
- #email = models.EmailField()
- #principal = models.CharField(max_length=100)
- #dn = models.CharField(max_length=100)
- #unixid = models.CharField(max_length=25)
-
- #class Meta:
- #ordering = ["name"]
-
- #def __unicode__(self):
- #return self.name
+@python_2_unicode_compatible
class Group(models.Model):
name = models.CharField(max_length=20)
- def __unicode__(self):
- return self.name
+
+ def __str__(self):
+ return six.text_type(self.name)
+@python_2_unicode_compatible
class Pipeline(models.Model):
PIPELINE_TYPE_EXTERNAL = 'E'
PIPELINE_TYPE_OTHER = 'O'
@@ -97,8 +94,8 @@ class Pipeline(models.Model):
('manage_pipeline', 'Can enable or disable pipeline'),
)
- def __unicode__(self):
- return self.name
+ def __str__(self):
+ return six.text_type(self.name)
class PipelineLog(models.Model):
@@ -115,16 +112,20 @@ class PipelineLog(models.Model):
choices=PIPELINE_LOG_ACTION_CHOICES)
+@python_2_unicode_compatible
class Search(models.Model):
name = models.CharField(max_length=100)
description = models.TextField(blank=True)
# XXX Need any additional fields? Like a PI email? Or perhaps even fk?
- def __unicode__(self):
- return self.name
+
+ def __str__(self):
+ return six.text_type(self.name)
+
# Label color will be used in CSS, see
# https://www.w3schools.com/colors/colors_names.asp for
# allowed color choices
+@python_2_unicode_compatible
class Label(models.Model):
name = models.CharField(max_length=20, unique=True)
# XXX really, does this belong here? probably not.
@@ -137,8 +138,8 @@ class Label(models.Model):
# signoffs, for examples.
protected = models.BooleanField(default=False)
- def __unicode__(self):
- return self.name
+ def __str__(self):
+ return six.text_type(self.name)
class ProtectedLabelError(Exception):
# To be raised when an attempt is made to apply or remove a
@@ -153,6 +154,7 @@ class Label(models.Model):
pass
+@python_2_unicode_compatible
class Event(models.Model):
objects = InheritanceManager() # Queries can return subclasses, if available.
@@ -242,7 +244,8 @@ class Event(models.Model):
@property
def datadir(self):
# Create a file-like object which is the SHA-1 hexdigest of the Event's primary key
- hdf = StringIO(sha1(str(self.id)).hexdigest())
+ hid = sha1(str(self.id).encode()).hexdigest()
+ hdf = StringIO(hid)
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
@@ -336,8 +339,8 @@ class Event(models.Model):
return e
raise cls.DoesNotExist("Event matching query does not exist")
- def __unicode__(self):
- return self.graceid
+ def __str__(self):
+ return six.text_type(self.graceid)
# Return a list of distinct tags associated with the log messages of this
# event.
@@ -482,6 +485,7 @@ class EventLog(CleanSaveModel, LogBase, AutoIncrementModel):
return None
+@python_2_unicode_compatible
class EMGroup(models.Model):
name = models.CharField(max_length=50, unique=True)
@@ -492,8 +496,8 @@ class EMGroup(models.Model):
# purposes.
#liasons = models.ManyToManyField(UserModel)
- def __unicode__(self):
- return self.name
+ def __str__(self):
+ return six.text_type(self.name)
class EMObservationBase(models.Model):
@@ -561,6 +565,7 @@ class EMObservationBase(models.Model):
self.decWidth = decmax-decmin
+@python_2_unicode_compatible
class EMObservation(EMObservationBase, AutoIncrementModel):
"""EMObservation class for events"""
AUTO_FIELD = 'N'
@@ -570,9 +575,14 @@ class EMObservation(EMObservationBase, AutoIncrementModel):
class Meta(EMObservationBase.Meta):
unique_together = (('event', 'N'),)
- def __unicode__(self):
- return "{event_id} | {group} | {N}".format(
- event_id=self.event.graceid, group=self.group.name, N=self.N)
+ def __str__(self):
+ return six.text_type(
+ "{event_id} | {group} | {N}".format(
+ event_id=self.event.graceid,
+ group=self.group.name,
+ N=self.N
+ )
+ )
def calculateCoveringRegion(self):
footprints = self.emfootprint_set.all()
@@ -620,6 +630,7 @@ class EMFootprint(EMFootprintBase, AutoIncrementModel):
unique_together = (('observation', 'N'),)
+@python_2_unicode_compatible
class Labelling(m2mThroughBase):
"""
Model which provides the "through" relationship between Events and Labels.
@@ -627,9 +638,13 @@ class Labelling(m2mThroughBase):
event = models.ForeignKey(Event)
label = models.ForeignKey(Label)
- def __unicode__(self):
- return "{graceid} | {label}".format(graceid=self.event.graceid,
- label=self.label.name)
+ def __str__(self):
+ return six.text_type(
+ "{graceid} | {label}".format(
+ graceid=self.event.graceid,
+ label=self.label.name
+ )
+ )
## Analysis Specific Attributes.
@@ -828,8 +843,8 @@ class SingleInspiral(models.Model):
return cls._field_names
except AttributeError: pass
model_field_names = set([ x.name for x in cls._meta.get_fields(include_parents=False) ])
- ligolw_field_names = set(
- glue.ligolw.lsctables.SnglInspiralTable.validcolumns.keys())
+ ligolw_field_names = set(list(
+ glue.ligolw.lsctables.SnglInspiralTable.validcolumns))
cls._field_names = model_field_names.intersection(ligolw_field_names)
return cls._field_names
@@ -905,6 +920,7 @@ class SimInspiralEvent(Event):
return cls._field_names
# Tags (user-defined log message attributes)
+@python_2_unicode_compatible
class Tag(CleanSaveModel):
"""
Model for tags attached to EventLogs.
@@ -926,8 +942,10 @@ class Tag(CleanSaveModel):
])
displayName = models.CharField(max_length=200, null=True, blank=True)
- def __unicode__(self):
- return self.displayName if self.displayName else self.name
+ def __str__(self):
+ return six.text_type(
+ self.displayName if self.displayName else self.name
+ )
class VOEventBase(CleanSaveModel):
@@ -990,10 +1008,34 @@ class VOEventBase(CleanSaveModel):
validators=[models.fields.validators.MinValueValidator(0.0),
models.fields.validators.MaxValueValidator(1.0)])
+ # Additional RAVEN Fields
+ raven_coinc = models.BooleanField(null=False, default=False, blank=True)
+ ext_gcn = models.CharField(max_length=20, default="", blank=True,
+ editable=False)
+ ext_pipeline = models.CharField(max_length=20, default="", blank=True,
+ editable=False)
+ ext_search = models.CharField(max_length=20, default="", blank=True,
+ editable=False)
+ time_coinc_far = models.FloatField(null=True, default=None, blank=True,
+ validators=[models.fields.validators.MinValueValidator(0.0)])
+ space_coinc_far = models.FloatField(null=True, default=None, blank=True,
+ validators=[models.fields.validators.MinValueValidator(0.0)])
+ combined_skymap_filename = models.CharField(max_length=100, null=True,
+ default=None, blank=True)
+ delta_t = models.FloatField(null=True, default=None, blank=True,
+ validators=[models.fields.validators.MinValueValidator(-1000),
+ models.fields.validators.MaxValueValidator(1000)])
+ ivorn = models.CharField(max_length=300, default="", blank=True,
+ editable=False)
+
+
def fileurl(self):
# Override this method on derived classes
return NotImplemented
+ class VOEventBuilderException(Exception):
+ pass
+
class VOEvent(VOEventBase, AutoIncrementModel):
"""VOEvent class for events"""
@@ -1104,7 +1146,7 @@ class SignoffBase(models.Model):
return 'ADV' + self.opposite_status
-
+@python_2_unicode_compatible
class Signoff(SignoffBase):
"""Class for Event signoffs"""
@@ -1113,9 +1155,15 @@ class Signoff(SignoffBase):
class Meta:
unique_together = ('event', 'instrument')
- def __unicode__(self):
- return "%s | %s | %s" % (self.event.graceid, self.instrument,
- self.status)
+ def __str__(self):
+ return six.text_type(
+ "{gid} | {instrument} | {status}".format(
+ self.event.graceid,
+ self.instrument,
+ self.status
+ )
+ )
+
EMSPECTRUM = (
('em.gamma', 'Gamma rays part of the spectrum'),
@@ -1170,8 +1218,10 @@ EMSPECTRUM = (
('em.radio.20-100MHz', 'Radio between 20 and 100 MHz'),
)
+
# TP (2 Apr 2018): pretty sure this class is deprecated - most recent
# production use is T137114 = April 2015.
+@python_2_unicode_compatible
class EMBBEventLog(AutoIncrementModel):
"""EMBB EventLog: A multi-purpose annotation for EM followup.
@@ -1182,8 +1232,14 @@ class EMBBEventLog(AutoIncrementModel):
ordering = ['-created', '-N']
unique_together = ("event","N")
- def __unicode__(self):
- return "%s-%s-%d" % (self.event.graceid, self.group.name, self.N)
+ def __str__(self):
+ return six.text_type(
+ "{gid}-{name}-{N}".format(
+ self.event.graceid,
+ self.group.name,
+ self.N
+ )
+ )
# A counter for Eels associated with a given event. This is
# important for addressibility.
diff --git a/gracedb/events/nltime.py b/gracedb/events/nltime.py
index 0cbcc7338d86dce803530caa5f612c5b403c2427..8515d1e97c1504c2a9102e978e94bf735a25f14a 100755
--- a/gracedb/events/nltime.py
+++ b/gracedb/events/nltime.py
@@ -58,7 +58,7 @@ def convertToAbsTime(toks):
else:
day = pytz.utc.localize(datetime(now.year, now.month, now.day))
if "timeOfDay" in toks:
- if isinstance(toks.timeOfDay,basestring):
+ if isinstance(toks.timeOfDay, str):
timeOfDay = {
"now" : timedelta(0, (now.hour*60+now.minute)*60+now.second, now.microsecond),
"noon" : timedelta(0,0,0,0,0,12),
@@ -91,11 +91,11 @@ def calculateTime(toks):
# grammar definitions
CL = CaselessLiteral
-today, tomorrow, yesterday, noon, midnight, now = map( CL,
- "today tomorrow yesterday noon midnight now".split())
+today, tomorrow, yesterday, noon, midnight, now = list(map( CL,
+ "today tomorrow yesterday noon midnight now".split()))
plural = lambda s : Combine(CL(s) + Optional(CL("s")))
-month, week, day, hour, minute, second = map( plural,
- "month week day hour minute second".split())
+month, week, day, hour, minute, second = list(map(plural,
+ "month week day hour minute second".split()))
am = CL("am")
pm = CL("pm")
COLON = Suppress(':')
@@ -187,11 +187,11 @@ if __name__ == "__main__":
2009/12/22 12:13:14""".splitlines()
for t in tests:
- print t, "(relative to %s)" % timezone.now()
+ print(t, "(relative to %s)" % timezone.now())
res = nlTimeExpression.parseString(t)
if "calculatedTime" in res:
- print res.calculatedTime
+ print(res.calculatedTime)
else:
- print "???"
- print
+ print("???")
+ print()
diff --git a/gracedb/events/reports.py b/gracedb/events/reports.py
index 4edb249792e7cc84befb3bf5cb633fe2177f806c..01b824b41e9d7f0c23d4dbe2c409c7939f07ccd1 100644
--- a/gracedb/events/reports.py
+++ b/gracedb/events/reports.py
@@ -21,7 +21,6 @@ import matplotlib
matplotlib.use('Agg')
import numpy
import matplotlib.pyplot as plot
-import StringIO
import base64
import sys
import calendar
@@ -30,6 +29,10 @@ from core.time_utils import posixToGpsTime
from django.utils import timezone
import pytz
import json
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
@internal_user_required
def histo(request):
@@ -124,7 +127,7 @@ def cluster(events):
return [e for e in events if not quieter(e)]
def to_png_image(out = sys.stdout):
- f = StringIO.StringIO()
+ f = StringIO()
plot.savefig(f, format="png")
return base64.b64encode(f.getvalue())
diff --git a/gracedb/events/serialize.py b/gracedb/events/serialize.py
index e36d9a126ed2973ce58f479cdb6eb1c100eb5d41..f3614334931dfe26a17a6893feb2d204af6731d3 100644
--- a/gracedb/events/serialize.py
+++ b/gracedb/events/serialize.py
@@ -1,6 +1,5 @@
-#!/usr/bin/python
-
from math import log
+import os
from time import gmtime, strftime
from glue.lal import LIGOTimeGPS
@@ -8,7 +7,7 @@ from glue.ligolw import ligolw
from glue.ligolw import table
from glue.ligolw import lsctables
-from core.vfile import VersionedFile
+from core.vfile import VersionedFile, create_versioned_file
##############################################################################
#
@@ -39,7 +38,7 @@ H1_detlist = ['H1']
L1_detlist = ['L1']
V1_detlist = ['V1']
-#this is the subset of SnglInspiralTable.validcolumn.keys() that
+#this is the subset of keys in SnglInspiralTable.validcolumn that
#are assigned from MBTA coinc triggers
MBTA_set_keys = ['ifo', 'search', 'end_time', 'end_time_ns', 'mass1', 'mass2',\
'mchirp', 'mtotal', 'eta', 'snr', 'eff_distance', 'event_id',\
@@ -84,17 +83,19 @@ def compute_mchirp_eta(m1,m2):
def write_output_files(root_dir, xmldoc, log_content, \
xml_fname = 'coinc.xml', log_fname = 'event.log'):
- """
- write the xml-format coinc tables and log file
- """
+ """
+ Write the xml-format coinc tables and log file
+ """
- f = VersionedFile(root_dir+'/'+xml_fname,'w')
- xmldoc.write(f)
- f.close()
+ # Write xml-formatted coinc table
+ # We do it this way instead of using create_versioned_file since the
+ # xmldoc is designed to write to a file object.
+ file_path = os.path.join(root_dir, xml_fname)
+ f = VersionedFile(file_path, 'w')
+ xmldoc.write(f.file)
- f = VersionedFile(root_dir+'/'+log_fname,'w')
- f.write(log_content)
- f.close()
+ # Write log file
+ create_versioned_file(log_fname, root_dir, log_content)
def get_ifos_for_cwb(cwb_ifos):
"""
@@ -132,7 +133,7 @@ def populate_omega_tables(datafile, set_keys = Omega_set_keys):
for line in f.readlines():
if not line.strip(): continue # ignore blank lines
elif '#' in line.strip()[0]: continue # ignore comments
- elif '=' not in line: raise ValueError, "Improperly formatted line"
+ elif '=' not in line: raise ValueError("Improperly formatted line")
else:
omega_list.extend([dat.strip() for dat in line.split('=',1)])
f.close()
@@ -140,7 +141,7 @@ def populate_omega_tables(datafile, set_keys = Omega_set_keys):
# basic error checking
# for key in omega_data:
# if not (key in omega_vars):
-# raise ValueError, "Unknown variable"
+# raise ValueError("Unknown variable")
#create the content for the event.log file
log_data = '\nLog File created '\
@@ -166,7 +167,7 @@ def populate_omega_tables(datafile, set_keys = Omega_set_keys):
row.confidence = -log(float(omega_data['probGlitch']))
cid = lsctables.CoincTable.get_next_id()
row.coinc_event_id = cid
- for key in mb_table.validcolumns.keys():
+ for key in mb_table.validcolumns:
if key not in set_keys:
setattr(row,key,None)
mb_table.append(row)
@@ -215,7 +216,7 @@ def populate_cwb_tables(datafile, set_keys=CWB_set_keys):
row.start_time_ns = st.nanoseconds
cid = lsctables.CoincTable.get_next_id()
row.coinc_event_id = cid
- for key in mb_table.validcolumns.keys():
+ for key in mb_table.validcolumns:
if key not in set_keys:
setattr(row,key,None)
mb_table.append(row)
@@ -255,7 +256,7 @@ def populate_coinc_tables(xmldoc, coinc_event_id, event_id_dict,\
elif 'burst' in CoincDef.search:
row.nevents = 1
else:
- raise ValueError, "Unrecognize CoincDef.search"
+ raise ValueError("Unrecognize CoincDef.search")
row.likelihood = likelihood
coinc_table.append(row)
@@ -270,7 +271,7 @@ def populate_coinc_tables(xmldoc, coinc_event_id, event_id_dict,\
elif 'burst' in CoincDef.search:
row.table_name = lsctables.MultiBurstTable.tableName.split(':')[0]
else:
- raise ValueError, "Unrecognize CoincDef.search"
+ raise ValueError("Unrecognize CoincDef.search")
if event_id_dict:
row.event_id = event_id_dict[ifo]
coinc_map_table.append(row)
diff --git a/gracedb/events/templatetags/logtags.py b/gracedb/events/templatetags/logtags.py
index c19075255941e756ca736c0ad6b0386a404267ed..d07c6188e6ad8b576959205bc400e6bcf3b35366 100644
--- a/gracedb/events/templatetags/logtags.py
+++ b/gracedb/events/templatetags/logtags.py
@@ -1,6 +1,5 @@
from django import template
-from django.utils.encoding import force_unicode
from django.utils.safestring import mark_safe
from ..models import Tag, EventLog
@@ -20,9 +19,6 @@ def getLogsForTag(event,name=None):
# In either case, we want the template to just ignore it.
return None
-@register.filter("tagUnicode")
-def tagUnicode(tag):
- return unicode(tag);
@register.filter("logsForTagHaveImage")
def logsForTagAllHaveImages(event,name=None):
diff --git a/gracedb/events/templatetags/scientific.py b/gracedb/events/templatetags/scientific.py
index 0cdfafad21b94a6d0219535c21ad06eb8f3e6154..ce1281520d5a0801dccaebe52690f43323b16c51 100644
--- a/gracedb/events/templatetags/scientific.py
+++ b/gracedb/events/templatetags/scientific.py
@@ -1,7 +1,7 @@
-
+from builtins import str
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
from django import template
-from django.utils.encoding import force_unicode
+from django.utils.encoding import force_text
from django.utils.safestring import mark_safe
from django.utils import formats
register = template.Library()
@@ -38,7 +38,7 @@ def scientificformat(text):
"""
try:
- input_val = force_unicode(text)
+ input_val = force_text(text)
d = Decimal(input_val)
except UnicodeEncodeError:
return u''
@@ -46,7 +46,7 @@ def scientificformat(text):
if input_val in special_floats:
return input_val
try:
- d = Decimal(force_unicode(float(text)))
+ d = Decimal(force_text(float(text)))
except (ValueError, InvalidOperation, TypeError, UnicodeEncodeError):
return u''
@@ -62,7 +62,7 @@ def scientificformat(text):
elif ( (d > Decimal('-100') and d < Decimal('-0.1')) or ( d > Decimal('0.1') and d < Decimal('100')) ) :
# this is what the original floatformat() function does
sign, digits, exponent = d.quantize(Decimal('.01'), ROUND_HALF_UP).as_tuple()
- digits = [unicode(digit) for digit in reversed(digits)]
+ digits = [str(digit) for digit in reversed(digits)]
while len(digits) <= abs(exponent):
digits.append(u'0')
digits.insert(-exponent, u'.')
@@ -73,7 +73,7 @@ def scientificformat(text):
# for very small and very large numbers
sign, digits, exponent = d.as_tuple()
exponent = d.adjusted()
- digits = [unicode(digit) for digit in digits][:3] # limit to 2 decimal places
+ digits = [str(digit) for digit in digits][:3] # limit to 2 decimal places
while len(digits) < 3:
digits.append(u'0')
digits.insert(1, u'.')
diff --git a/gracedb/events/templatetags/timeutil.py b/gracedb/events/templatetags/timeutil.py
index dc72a5c95f57176622db1e47680bd00567f1255e..95fdf8b6251d83c77454e77419a1e55e824becb2 100644
--- a/gracedb/events/templatetags/timeutil.py
+++ b/gracedb/events/templatetags/timeutil.py
@@ -53,7 +53,7 @@ def get_multitime_value(t, label, autoescape, format):
dt = dt.astimezone(SERVER_TZ)
posix_time = time.mktime(dt.timetuple())
gps_time = int(posixToGpsTime(posix_time))
- elif isinstance(t, int) or isinstance(t, long):
+ elif isinstance(t, int):
gps_time = t
dt = gpsToUtc(t)
# Note: must convert to server timezone before calling mktime
@@ -66,7 +66,7 @@ def get_multitime_value(t, label, autoescape, format):
return "N/A"
return '<time utc="%s" gps="%s" llo="%s" lho="%s" virgo="%s" jsparsable="%s"%s>%s</time>' % \
8*("N/A",)
- # raise ValueError("time must be type int, long or datetime, not '%s'" % type(t))
+ # raise ValueError("time must be type int or datetime, not '%s'" % type(t))
# JavaScript -- parsable by Date() object constructor
# "Jan 2, 1985 00:00:00 UTC"
@@ -167,7 +167,7 @@ def timeSelections(t):
dt = dt.astimezone(SERVER_TZ)
posix_time = time.mktime(dt.timetuple())
gps_time = int(posixToGpsTime(posix_time))
- elif isinstance(t, int) or isinstance(t, long):
+ elif isinstance(t, int):
gps_time = t
dt = gpsToUtc(t)
posix_time = time.mktime(dt.astimezone(SERVER_TZ).timetuple())
@@ -176,7 +176,7 @@ def timeSelections(t):
dt = gpsToUtc(t)
posix_time = time.mktime(dt.astimezone(SERVER_TZ).timetuple())
else:
- raise ValueError("time must be type int, long or datetime, not '%s'" % type(t))
+ raise ValueError("time must be type int or datetime, not '%s'" % type(t))
# JavaScript -- parsable by Date() object constructor
# "Jan 2, 1985 00:00:00 UTC"
diff --git a/gracedb/events/tests/test_access.py b/gracedb/events/tests/test_access.py
index 8e724a6a822f33f140df5389902ed2edf62483f8..0703958e3d5dd5b5a818af15bdc05619bb6b0339 100644
--- a/gracedb/events/tests/test_access.py
+++ b/gracedb/events/tests/test_access.py
@@ -157,8 +157,8 @@ class TestEventFileListView(ExposeLogMixin, EventSetup, GraceDbTestBase):
super(TestEventFileListView, cls).setUpTestData()
# Create files for internal and exposed events
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_event, filename=cls.file1['filename'],
@@ -254,8 +254,8 @@ class TestEventFileDownloadView(ExposeLogMixin, EventSetup, GraceDbTestBase):
super(TestEventFileDownloadView, cls).setUpTestData()
# Create files for internal and exposed events
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_event, filename=cls.file1['filename'],
@@ -389,9 +389,11 @@ class TestEventCreationPage(GraceDbTestBase):
"""Basic internal user can't create production events"""
response = self.request_as_user(self.url, "POST", self.internal_user,
data=self.event_data)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content,
- "You do not have permission to submit events to this pipeline.")
+ self.assertContains(
+ response,
+ 'You do not have permission to submit events to this pipeline',
+ status_code=403
+ )
def test_basic_internal_user_create_test_event(self):
"""Basic internal user can create test events"""
@@ -490,9 +492,11 @@ class TestEventModifyPermissions(EventSetup, GraceDbTestBase):
url = reverse('modify_permissions', args=[self.internal_event.graceid])
response = self.request_as_user(url, "POST", self.internal_user,
data=self.perm_data)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content,
- "You aren't authorized to create permission objects.")
+ self.assertContains(
+ response,
+ "You aren't authorized to create permission objects",
+ status_code=403
+ )
def test_privileged_internal_user_modify_permissions(self):
"""Privileged internal user can modify event permissions"""
@@ -521,9 +525,11 @@ class TestEventModifyPermissions(EventSetup, GraceDbTestBase):
url = reverse('modify_permissions', args=[self.internal_event.graceid])
response = self.request_as_user(url, "POST", self.internal_user,
data=remove_data)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content,
- "You aren't authorized to delete permission objects.")
+ self.assertContains(
+ response,
+ "You aren't authorized to delete permission objects",
+ status_code=403
+ )
# Give remove permission to user and try again - should succeed
p_remove.user_set.add(self.internal_user)
@@ -548,9 +554,11 @@ class TestEventModifyPermissions(EventSetup, GraceDbTestBase):
url = reverse('modify_permissions', args=[self.lvem_event.graceid])
response = self.request_as_user(url, "POST", self.lvem_user,
data=self.perm_data)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content,
- "You aren't authorized to create permission objects.")
+ self.assertContains(
+ response,
+ "You aren't authorized to create permission objects",
+ status_code=403
+ )
def test_public_user_modify_permissions(self):
"""Public user can't modify event permissions"""
@@ -584,9 +592,11 @@ class TestEventModifySignoff(SignoffGroupsAndUsersSetup, EventSetup,
url = reverse('modify_signoff', args=[self.internal_event.graceid])
response = self.request_as_user(url, "POST", self.internal_user,
self.op_signoff_data)
- self.assertEqual(response.status_code, 400)
- self.assertEqual(response.content,
- "Unknown instrument/control room for signoff.")
+ self.assertContains(
+ response,
+ 'Unknown instrument/control room for signoff.',
+ status_code=400
+ )
def test_H1_control_room_modify_signoff(self):
"""H1 control room can create H1 operator signoff for event"""
@@ -624,9 +634,11 @@ class TestEventModifySignoff(SignoffGroupsAndUsersSetup, EventSetup,
url = reverse('modify_signoff', args=[self.lvem_event.graceid])
response = self.request_as_user(url, "POST", self.lvem_user,
self.op_signoff_data)
- self.assertEqual(response.status_code, 400)
- self.assertEqual(response.content,
- "Unknown instrument/control room for signoff.")
+ self.assertContains(
+ response,
+ 'Unknown instrument/control room for signoff.',
+ status_code=400
+ )
def test_public_user_modify_signoff(self):
"""Public user can't modify event signoffs"""
@@ -773,8 +785,11 @@ class TestEventLogTag(EventSetup, GraceDbTestBase):
url = reverse('taglogentry', args=[self.lvem_event.graceid,
log.N, self.tag_name])
response = self.request_as_user(url, "POST", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content, 'Forbidden')
+ self.assertContains(
+ response,
+ 'Forbidden',
+ status_code=403
+ )
# Exposed log on exposed event
log = self.lvem_event.eventlog_set.get(
@@ -843,10 +858,13 @@ class TestEventLogUntag(EventSetup, GraceDbTestBase):
response = self.request_as_user(url, "DELETE", self.internal_user)
# Returns a 200 with a message on success
- self.assertEqual(response.status_code, 200)
- self.assertEqual(response.content,
- "Removed tag {tag} for message {N}.".format(tag=self.test_tag.name,
- N=log.N))
+ self.assertContains(
+ response,
+ "Removed tag {tag} for message {N}.".format(
+ tag=self.test_tag.name, N=log.N
+ ),
+ status_code=200
+ )
def test_lvem_user_untag_log_hidden_event(self):
"""LV-EM user can't untag event logs for hidden events"""
@@ -878,8 +896,11 @@ class TestEventLogUntag(EventSetup, GraceDbTestBase):
url = reverse('taglogentry', args=[self.lvem_event.graceid,
log.N, self.test_tag.name])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content, 'Forbidden')
+ self.assertContains(
+ response,
+ 'Forbidden',
+ status_code=403
+ )
# Exposed log on exposed event
log = self.lvem_event.eventlog_set.get(
@@ -888,10 +909,13 @@ class TestEventLogUntag(EventSetup, GraceDbTestBase):
log.N, self.test_tag.name])
response = self.request_as_user(url, "DELETE", self.lvem_user)
# Test response
- self.assertEqual(response.status_code, 200)
- self.assertEqual(response.content,
- "Removed tag {tag} for message {N}.".format(tag=self.test_tag.name,
- N=log.N))
+ self.assertContains(
+ response,
+ "Removed tag {tag} for message {N}.".format(
+ tag=self.test_tag.name, N=log.N
+ ),
+ status_code=200
+ )
# Test tags on event log
self.assertEqual(log.tags.count(), 1)
self.assertIn(self.lvem_tag, log.tags.all())
@@ -917,8 +941,8 @@ class TestEventCreateEMObservation(EventSetup, GraceDbTestBase):
# Define EMObservation data for POST-ing
cls.emgroup_name = 'fake_emgroup'
now = datetime.datetime.now()
- start_time_list = map(lambda i:
- (now + datetime.timedelta(seconds=i)).isoformat(), [0, 1, 2, 3])
+ start_time_list = list(map(lambda i:
+ (now + datetime.timedelta(seconds=i)).isoformat(), [0, 1, 2, 3]))
cls.emobservation_data = {
'group': cls.emgroup_name,
'ra_list': [1, 2, 3, 4],
@@ -990,5 +1014,8 @@ class TestEventCreateEMObservation(EventSetup, GraceDbTestBase):
for log in ev.eventlog_set.all():
url = reverse('emobservation_entry', args=[ev.graceid, ""])
response = self.request_as_user(url, "POST")
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content, 'Forbidden')
+ self.assertContains(
+ response,
+ 'Forbidden',
+ status_code=403
+ )
diff --git a/gracedb/events/tests/test_label_search.py b/gracedb/events/tests/test_label_search.py
index 3219c5653c6f6540bc3a83f0098ff643a10c7fae..75cdd731045b39a6ef0a652fb02df39128ef3ac6 100644
--- a/gracedb/events/tests/test_label_search.py
+++ b/gracedb/events/tests/test_label_search.py
@@ -53,8 +53,8 @@ class LabelSearchTestCase(TestCase):
Labelling.objects.create(event=e, label=label, creator=submitter)
def test_all_queries(self):
- for key, d in QUERY_CASES.iteritems():
- print "Checking %s ... " % key
+ for key, d in QUERY_CASES.items():
+ print("Checking %s ... " % key)
# Explicitly search for test events
query = 'Test ' + d['query']
self.assertEqual(set(get_pks_for_query(query)), set(d['pk_list']))
diff --git a/gracedb/events/tests/test_perms.py b/gracedb/events/tests/test_perms.py
index a2b0148f7c73ca8914bf5d02af68f66b2f1f3798..72d14684e22592ae38bcbadfc3ce39dd78d3c335 100644
--- a/gracedb/events/tests/test_perms.py
+++ b/gracedb/events/tests/test_perms.py
@@ -4,6 +4,7 @@ from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import Permission, Group, User
from django.conf import settings
from django.urls import reverse
+from django.utils.http import urlencode
from guardian.models import GroupObjectPermission, UserObjectPermission
from guardian.shortcuts import assign_perm
@@ -16,8 +17,7 @@ from events.permission_utils import assign_default_event_perms
import json
import os
import shutil
-from urllib import urlencode
-
+
#------------------------------------------------------------------------------
#------------------------------------------------------------------------------
# Some utilities
@@ -50,7 +50,7 @@ def extra_args(user):
# Need to handle reverse proxy case where headers are used
# instead of Apache environment variables.
if settings.USE_X_FORWARDED_HOST:
- for k in AUTH_DICT.keys():
+ for k in AUTH_DICT:
AUTH_DICT['HTTP_' + k.upper()] = AUTH_DICT.pop(k)
return AUTH_DICT
diff --git a/gracedb/events/translator.py b/gracedb/events/translator.py
index 977a6420b2e2e2a76619f2e3f63cb832ef6b2cf3..288637f58b07cbce7f48dba07feef1ee8867e93f 100644
--- a/gracedb/events/translator.py
+++ b/gracedb/events/translator.py
@@ -1,28 +1,29 @@
-from math import isnan
+import json
+import logging
+from math import isnan, sqrt
import numbers
import os
-from .models import EventLog
-from .models import SingleInspiral
-from .models import SimInspiralEvent
-
from glue.ligolw.utils import load_filename, load_fileobj
from glue.ligolw.lsctables import CoincInspiralTable, SnglInspiralTable, use_in
from glue.ligolw.lsctables import SimInspiralTable, MultiBurstTable, CoincTable
from glue.ligolw.ligolw import LIGOLWContentHandler
+import voeventparse as vp
+from core.time_utils import utc_datetime_to_gps_float
+from core.vfile import create_versioned_file
+from .models import EventLog
+from .models import SingleInspiral
+from .models import SimInspiralEvent
from .serialize import populate_omega_tables, write_output_files
-from VOEventLib.Vutil import parse, getWhereWhen, findParam, getParamNames
-from core.time_utils import isoToGps, isoToGpsFloat
-from core.vfile import VersionedFile
-
-import json
-import StringIO
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
-from math import sqrt
-import logging
+# Set up logger
logger = logging.getLogger(__name__)
use_in(LIGOLWContentHandler)
@@ -44,7 +45,7 @@ def cleanData(val, field_name, table_name='events_event'):
return maxval
else:
return val
- elif isinstance(val, basestring):
+ elif isinstance(val, str):
raise ValueError("Unrecognized string in the %s column" % field_name)
else:
raise ValueError("Unrecognized value in column %s" % field_name)
@@ -76,7 +77,7 @@ def handle_uploaded_data(event, datafilename,
try:
xmldoc = load_filename(datafilename, contenthandler = LIGOLWContentHandler)
- except Exception, e:
+ except Exception as e:
message = "Could not read data (%s)" % str(e)
EventLog(event=event, issuer=event.submitter, comment=message).save()
return
@@ -84,7 +85,7 @@ def handle_uploaded_data(event, datafilename,
# Try reading the CoincInspiralTable
try:
coinc_table = CoincInspiralTable.get_table(xmldoc)[0]
- except Exception, e:
+ except Exception as e:
warnings += "Could not extract coinc inspiral table."
return temp_data_loc, warnings
@@ -124,7 +125,7 @@ def handle_uploaded_data(event, datafilename,
log_data.append("FAR: %0.3e" % far)
else:
log_data.append("FAR: ---")
- except Exception, e:
+ except Exception as e:
log_comment = "Problem Creating Log File"
log_data = ["Cannot create log file", "error was:", str(e)]
@@ -157,7 +158,7 @@ def handle_uploaded_data(event, datafilename,
# Try to get the coinc_event_table
try:
coinc_event_table = CoincTable.get_table(xmldoc)[0]
- except Exception, e:
+ except Exception as e:
warnings += "Could not extract coinc event table."
return temp_data_loc, warnings
event.nevents = coinc_event_table.nevents
@@ -175,9 +176,6 @@ def handle_uploaded_data(event, datafilename,
event.snr = snr
event.false_alarm_rate = getattr(coinc_table, "false_alarm_rate", None)
event.combined_far = far
-
- # XXX xml_filename unused
- #xml_filename = os.path.join(output_dir, coinc_table_filename)
event.save()
# Extract Single Inspiral Information
@@ -192,7 +190,7 @@ def handle_uploaded_data(event, datafilename,
if datafilename:
xmldoc = load_filename(datafilename, contenthandler=LIGOLWContentHandler)
elif file_contents:
- f = StringIO.StringIO(file_contents)
+ f = StringIO(file_contents)
xmldoc, digest = load_fileobj(f, contenthandler=LIGOLWContentHandler)
else:
msg = "If you wanna make an injection event, I'm gonna need a filepath or filecontents."
@@ -220,7 +218,7 @@ def handle_uploaded_data(event, datafilename,
# log_data.append("Component 1 Spin: (%f, %f, %f)" % spin1)
# log_data.append("Component 2 Spin: (%f, %f, %f)" % spin2)
# log_data.append("Geocentric End Time: %d.%09d" % end_time)
-# except Exception, e:
+# except Exception as e:
# log_comment = "Problem Creating Log File"
# log_data = ["Cannot create log file", "error was:", str(e)]
# log_data = "\n".join(log_data)
@@ -281,7 +279,7 @@ def handle_uploaded_data(event, datafilename,
warnings = []
try:
coinc_table = CoincInspiralTable.get_table(xmldoc)[0]
- except Exception, e:
+ except Exception as e:
warnings += "Could not extract coinc inspiral table."
return temp_data_loc, warnings
@@ -289,10 +287,6 @@ def handle_uploaded_data(event, datafilename,
event.instruments = coinc_table.ifos
event.nevents = coinc_event_table.nevents
event.likelihood = cleanData(coinc_event_table.likelihood, 'likelihood')
-
- # XXX xml_filename unused.
- #xml_filename = os.path.join(output_dir, coinc_table_filename)
-
event.save()
elif pipeline in ['CWB', 'CWB2G']:
@@ -314,7 +308,7 @@ def handle_uploaded_data(event, datafilename,
comment="Coinc Table Created")
log.save()
- if data.writeLogfile( os.path.join(outputDataDir, "event.log") ):
+ if data.writeLogfile(outputDataDir, "event.log"):
log = EventLog(event=event,
filename="event.log",
file_version=0,
@@ -342,17 +336,7 @@ def handle_uploaded_data(event, datafilename,
elif pipeline in ['Swift', 'Fermi', 'SNEWS']:
# Get the event time from the VOEvent file
error = None
- try:
- #event.gpstime = getGpsFromVOEvent(datafilename)
- populateGrbEventFromVOEventFile(datafilename, event)
- except Exception, e:
- error = "Problem parsing VOEvent: %s" % e.__repr__()
- event.save()
- if error is not None:
- log = EventLog(event=event,
- issuer=event.submitter,
- comment=error)
- log.save()
+ populateGrbEventFromVOEventFile(datafilename, event)
elif pipeline == 'oLIB':
# lambda function for converting to a type if not None
typecast = lambda t, v: t(v) if v is not None else v
@@ -449,12 +433,10 @@ class Translator(object):
logdata.append("FAR: %s" % val_or_dashes(data.get('far')))
return "\n".join(logdata)
- def writeLogfile(self, path):
+ def writeLogfile(self, data_directory, filename):
data = self.logData()
if data:
- f = VersionedFile(path, 'w')
- f.write(data)
- f.close()
+ create_versioned_file(filename, data_directory, data)
return True
@@ -614,43 +596,46 @@ class CwbData(Translator):
def writeCoincFile(self, path):
pass
-def getGpsFromVOEvent(filename):
- v = parse(filename)
- wwd = getWhereWhen(v)
- gpstime = isoToGps(wwd['time'])
- return gpstime
def populateGrbEventFromVOEventFile(filename, event):
- v = parse(filename)
- wherewhen = getWhereWhen(v)
- event.gpstime = isoToGpsFloat(wherewhen['time'])
- event.ivorn = v.ivorn
-
- event.author_shortname = v.get_Who().Author.shortName[0]
- event.author_ivorn = v.get_Who().AuthorIVORN
-
- event.observatory_location_id = wherewhen['observatory']
- event.coord_system = wherewhen['coord_system']
- event.ra = wherewhen['longitude']
- event.dec = wherewhen['latitude']
- event.error_radius = wherewhen['positionalError']
-
- event.how_description = v.get_How().get_Description()[0]
- event.how_reference_url = v.get_How().get_Reference()[0].uri
-
- # try to find a trigger_duration value
+ # Load file into vp.Voevent instance
+ with open(filename, 'rb') as f:
+ v = vp.load(f)
+
+ # Get gpstime
+ utc_time = vp.convenience.get_event_time_as_utc(v)
+ gpstime = utc_datetime_to_gps_float(utc_time)
+
+ # Get event position
+ pos2d = vp.get_event_position(v)
+
+ # Assign information to event
+ event.gpstime = gpstime
+ event.ivorn = v.get('ivorn')
+ event.author_shortname = v.Who.Author.shortName
+ event.author_ivorn = v.Who.AuthorIVORN
+ event.observatory_location_id = \
+ v.WhereWhen.ObsDataLocation.ObservatoryLocation.get('id')
+ event.coord_system = pos2d.system
+ event.ra = pos2d.ra
+ event.dec = pos2d.dec
+ event.error_radius = pos2d.err
+ event.how_description = v.How.Description
+ event.how_reference_url = v.How.Reference.get('uri')
+
+ # Try to find a trigger_duration value
# Fermi uses Trig_Dur or Data_Integ, while Swift uses Integ_Time
# One or the other may be present, but not both
- VOEvent_params = [pn[1] for pn in getParamNames(v)]
+ VOEvent_params = vp.convenience.get_toplevel_params(v)
trig_dur_params = ["Trig_Dur", "Trans_Duration", "Data_Integ",
"Integ_Time", "Trig_Timescale"]
trigger_duration = None
for param in trig_dur_params:
if (param in VOEvent_params):
- trigger_duration = float(findParam(v, "", param).get_value())
+ trigger_duration = float(VOEvent_params.get(param).get('value'))
break
-
+
# Fermi GCNs (after the first one) often set Trig_Dur or Data_Integ
# to 0.000 (not sure why). We don't want to overwrite the currently
# existing value in the database with 0.000 if this has happened, so
@@ -663,6 +648,9 @@ def populateGrbEventFromVOEventFile(filename, event):
trigger_id_params = ['TrigID', 'Trans_Num', 'EventID']
for param in trigger_id_params:
if (param in VOEvent_params):
- trigger_id = findParam(v, "", param).get_value()
+ trigger_id = VOEvent_params.get(param).get('value')
break
event.trigger_id = trigger_id
+
+ # Save event
+ event.save()
diff --git a/gracedb/events/view_logic.py b/gracedb/events/view_logic.py
index 8987f65a5b7666ad012572e5fa79dc4872e3b656..3c8ab99a09c0effc5bfb9c2b96f721267c716efc 100644
--- a/gracedb/events/view_logic.py
+++ b/gracedb/events/view_logic.py
@@ -17,7 +17,7 @@ from .permission_utils import assign_default_event_perms
from alerts.issuers.events import EventAlertIssuer, EventLabelAlertIssuer, \
EventEMObservationAlertIssuer, EventEMBBEventLogAlertIssuer
-from core.vfile import VersionedFile
+from core.vfile import create_versioned_file
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import Permission
@@ -105,10 +105,7 @@ def _createEventFromForm(request, form):
# Write the event data file to disk.
f = request.FILES['eventFile']
uploadDestination = os.path.join(eventDir, f.name)
- fdest = VersionedFile(uploadDestination, 'w')
- for chunk in f.chunks():
- fdest.write(chunk)
- fdest.close()
+ version = create_versioned_file(f.name, event.datadir, f)
file_contents = None
# Extract Info from uploaded data
@@ -128,7 +125,7 @@ def _createEventFromForm(request, form):
# XXX This reverse will give the web-interface URL, not the REST URL.
# This could be a problem if anybody ever tries to use it.
EventAlertIssuer(event, alert_type='new').issue_alerts()
- except Exception, e:
+ except Exception as e:
message = "Problem issuing an alert (%s)" % e
logger.warning(message)
warnings += [message]
@@ -145,12 +142,12 @@ def _createEventFromForm(request, form):
create_label(event, request, label.name,
can_add_protected=False)
- except Exception, e:
+ except Exception as e:
message = "Problem scanning data. No alert issued (%s)" % e
logger.warning(message)
warnings += [message]
#return HttpResponseRedirect(reverse(view, args=[event.graceid]))
- except Exception, e:
+ except Exception as e:
# something went wrong.
# XXX We need to make sure we clean up EVERYTHING.
# We don't. Wiki page and data directories remain.
@@ -299,28 +296,28 @@ def get_performance_info():
# Localize so we can compare with aware datetimes
dt = SERVER_TZ.localize(dt)
if dt > dt_min:
- if method not in totals_by_method.keys():
+ if method not in totals_by_method:
totals_by_method[method] = 1
totals_by_status[method] = {status: 1}
else:
totals_by_method[method] += 1
- if status not in totals_by_status[method].keys():
+ if status not in totals_by_status[method]:
totals_by_status[method][status] = 1
else:
totals_by_status[method][status] += 1
# Calculate summary information:
summaries = {}
- for method in totals_by_method.keys():
+ for method in totals_by_method:
summaries[method] = {'gt_500': 0, 'btw_300_500': 0}
- for key in totals_by_status[method].keys():
+ for key in totals_by_status[method]:
if key >= 500:
summaries[method]['gt_500'] += totals_by_status[method][key]
elif key >= 300:
summaries[method]['btw_300_500'] += totals_by_status[method][key]
# Normalize
if totals_by_method[method] > 0:
- for key in summaries[method].keys():
+ for key in summaries[method]:
summaries[method][key] = float(summaries[method][key])/totals_by_method[method]
context = {
@@ -491,22 +488,22 @@ def create_emobservation(request, event):
startTimeList = d.get('start_time_list')
durationList = d.get('duration_list')
- except Exception, e:
+ except Exception as e:
raise ValueError('Lacking input: %s' % str(e))
# Handle case where comma-separated strings are submitted rather than lists
if isinstance(raList, six.string_types):
- raList = map(lambda x: float(x.strip()), raList.split(','))
+ raList = list(map(lambda x: float(x.strip()), raList.split(',')))
if isinstance(raWidthList, six.string_types):
- raWidthList = map(lambda x: float(x.strip()), raWidthList.split(','))
+ raWidthList = list(map(lambda x: float(x.strip()), raWidthList.split(',')))
if isinstance(decList, six.string_types):
- decList = map(lambda x: float(x.strip()), decList.split(','))
+ decList = list(map(lambda x: float(x.strip()), decList.split(',')))
if isinstance(decWidthList, six.string_types):
- decWidthList = map(lambda x: float(x.strip()), decWidthList.split(','))
+ decWidthList = list(map(lambda x: float(x.strip()), decWidthList.split(',')))
if isinstance(startTimeList, six.string_types):
- startTimeList = map(lambda x: x.strip(), startTimeList.split(','))
+ startTimeList = list(map(lambda x: x.strip(), startTimeList.split(',')))
if isinstance(durationList, six.string_types):
- durationList = map(lambda x: int(x.strip()), durationList.split(','))
+ durationList = list(map(lambda x: int(x.strip()), durationList.split(',')))
all_lists = (raList, raWidthList, decList, decWidthList, startTimeList,
durationList)
@@ -516,7 +513,7 @@ def create_emobservation(request, event):
# Check all list lengths
list_length = len(all_lists[0])
- if not all(map(lambda l: len(l) == list_length, all_lists)):
+ if not all(list(map(lambda l: len(l) == list_length, all_lists))):
raise ValueError('ra_list, dec_list, ra_width_list, dec_width_list, '
'start_time_list, and duration_list must be the same length.')
@@ -576,7 +573,7 @@ def create_emobservation(request, event):
group=emo.group)
EventEMObservationAlertIssuer(emo, alert_type='emobservation') \
.issue_alerts()
- except Exception, e:
+ except Exception as e:
# XXX Should probably send back warnings, as in the other cases.
logger.error('error sending alert for emobservation: {0}'.format(e))
diff --git a/gracedb/events/views.py b/gracedb/events/views.py
index e05949917045d24fba9780b54c106bb186b4d901..3f0d2cfe69ce1ac2d93d72fbeddaf992910f7f0a 100644
--- a/gracedb/events/views.py
+++ b/gracedb/events/views.py
@@ -48,7 +48,7 @@ log = logging.getLogger(__name__)
import os
from django.conf import settings
-from core.vfile import VersionedFile
+from core.vfile import create_versioned_file
# XXX This should be configurable / moddable or something
MAX_QUERY_RESULTS = 1000
@@ -137,7 +137,7 @@ def index(request):
if request.user and not is_external(request.user) and settings.SHOW_RECENT_EVENTS_ON_HOME:
try:
recent_events = get_recent_events_string(request)
- except Exception, e:
+ except Exception as e:
pass
context['recent_events'] = recent_events
@@ -211,17 +211,10 @@ def logentry(request, event, num=None):
file_version = None
if uploadedFile:
filename = uploadedFile.name
- filepath = os.path.join(event.datadir, filename)
-
try:
- # Open / Write the file.
- fdest = VersionedFile(filepath, 'w')
- for chunk in uploadedFile.chunks():
- fdest.write(chunk)
- fdest.close()
- # Ascertain the version assigned to this particular file.
- file_version = fdest.version
- except Exception, e:
+ file_version = create_versioned_file(filename, event.datadir,
+ uploadedFile)
+ except Exception as e:
return HttpResponseServerError(str(e))
elog.filename = filename
@@ -294,7 +287,7 @@ def logentry(request, event, num=None):
return HttpResponseForbidden("Forbidden")
try:
elog = event.eventlog_set.filter(N=num)[0]
- except Exception, e:
+ except Exception as e:
raise Http404
# Check authorization for this log message
@@ -323,13 +316,13 @@ def logentry(request, event, num=None):
def neighbors(request, event, delta1, delta2=None):
context = {}
try:
- delta1 = long(delta1)
+ delta1 = int(delta1)
if delta2 is None:
delta2 = delta1
delta1 = -delta1
else:
- delta2 = long(delta2)
+ delta2 = int(delta2)
except ValueError: pass
except: pass
@@ -589,7 +582,7 @@ def performance(request):
try:
context = get_performance_info()
- except Exception, e:
+ except Exception as e:
return HttpResponseServerError(str(e))
return render(request, 'gracedb/performance.html', context=context)
@@ -745,9 +738,9 @@ def emobservation_entry(request, event, num=None):
try:
# Alert is issued in this function
create_emobservation(request, event)
- except ValueError, e:
+ except ValueError as e:
return HttpResponseBadRequest(str(e))
- except Exception, e:
+ except Exception as e:
return HttpResponseServerError(str(e))
return HttpResponseRedirect(reverse('view', args=[event.graceid]))
diff --git a/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py b/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py
index 38fd05eda3395f7ee0b00e6d1c9a2501c28a941f..918b9880ec3ca06ed778c18c6dd3d52bb0f410dd 100644
--- a/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py
+++ b/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py
@@ -1,3 +1,4 @@
+from builtins import str
import datetime
import ldap
@@ -32,13 +33,15 @@ class LdapPersonResultProcessor(object):
def extract_user_attributes(self):
if self.ldap_connection is None:
raise RuntimeError('LDAP connection not configured')
+ memberships = [group_name.decode('utf-8') for group_name in
+ self.ldap_result.get('isMemberOf', [])]
self.user_data = {
- 'first_name': unicode(self.ldap_result['givenName'][0], 'utf-8'),
- 'last_name': unicode(self.ldap_result['sn'][0], 'utf-8'),
- 'email': self.ldap_result['mail'][0],
- 'is_active': self.ldap_connection.lvc_group.ldap_name in
- self.ldap_result.get('isMemberOf', []),
- 'username': self.ldap_result['krbPrincipalName'][0],
+ 'first_name': self.ldap_result['givenName'][0].decode('utf-8'),
+ 'last_name': self.ldap_result['sn'][0].decode('utf-8'),
+ 'email': self.ldap_result['mail'][0].decode('utf-8'),
+ 'is_active': (self.ldap_connection.lvc_group.ldap_name in
+ memberships),
+ 'username': self.ldap_result['krbPrincipalName'][0].decode('utf-8'),
}
def check_situation(self, user_exists, l_user_exists):
@@ -142,7 +145,8 @@ class LdapPersonResultProcessor(object):
def update_user_groups(self):
# Get list of group names that the user belongs to from the LDAP result
- memberships = self.ldap_result.get('isMemberOf', [])
+ memberships = [group_name.decode('utf-8') for group_name in
+ self.ldap_result.get('isMemberOf', [])]
# Get groups which are listed for the user in the LDAP and whose
# membership is controlled by the LDAP
@@ -208,7 +212,8 @@ class LdapPersonResultProcessor(object):
# Get two lists of subjects as sets
db_x509_subjects = set(list(self.ligoldapuser.x509cert_set.values_list(
'subject', flat=True)))
- ldap_x509_subjects = set(self.ldap_result.get('gridX509subject', []))
+ ldap_x509_subjects = set([x.decode('utf-8') for x in
+ self.ldap_result.get('gridX509subject', [])])
# Get certs to add and remove
certs_to_add = ldap_x509_subjects.difference(db_x509_subjects)
@@ -238,14 +243,14 @@ class LdapRobotResultProcessor(LdapPersonResultProcessor):
def extract_user_attributes(self):
if self.ldap_connection is None:
raise RuntimeError('LDAP connection not configured')
+ memberships = [group_name.decode('utf-8') for group_name in
+ self.ldap_result.get('isMemberOf', [])]
self.user_data = {
- 'last_name': unicode(self.ldap_result['x-LIGO-TWikiName'][0],
- 'utf-8'),
- 'email': self.ldap_result['mail'][0],
- 'is_active': self.ldap_connection.groups.get(
- name='robot_accounts').name in self.ldap_result.get(
- 'isMemberOf', []),
- 'username': self.ldap_result['cn'][0],
+ 'last_name': self.ldap_result['x-LIGO-TWikiName'][0].decode('utf-8'),
+ 'email': self.ldap_result['mail'][0].decode('utf-8'),
+ 'is_active': (self.ldap_connection.groups.get(
+ name='robot_accounts').name in memberships),
+ 'username': self.ldap_result['cn'][0].decode('utf-8'),
}
def check_situation(self, user_exists, l_user_exists):
diff --git a/gracedb/ligoauth/middleware.py b/gracedb/ligoauth/middleware.py
index a4b879fbf9c2997351e189eb720617da80590603..e08bd5f645cb13c1cd6bdbaf86a72e8c78e2a458 100644
--- a/gracedb/ligoauth/middleware.py
+++ b/gracedb/ligoauth/middleware.py
@@ -125,7 +125,7 @@ class ControlRoomMiddleware(object):
user_ip = self.get_client_ip(request)
# Add user to control room group(s)
- for ifo, ip in settings.CONTROL_ROOM_IPS.iteritems():
+ for ifo, ip in settings.CONTROL_ROOM_IPS.items():
if (ip == user_ip):
control_room_group = DjangoGroup.objects.get(name=
(ifo.lower() + self.control_room_group_suffix))
diff --git a/gracedb/ligoauth/migrations/0005_update_emfollow_accounts.py b/gracedb/ligoauth/migrations/0005_update_emfollow_accounts.py
index f88071fce3d4e0244014a2abd572ed2d599b263f..ce155144eb663e954c7289b9115f712490c29310 100644
--- a/gracedb/ligoauth/migrations/0005_update_emfollow_accounts.py
+++ b/gracedb/ligoauth/migrations/0005_update_emfollow_accounts.py
@@ -49,7 +49,7 @@ def deactivate_old_and_add_new_accounts(apps, schema_editor):
Group = apps.get_model('auth', 'Group')
# Mark old users as inactive and delete their certificates
- for username in OLD_ACCOUNTS.keys():
+ for username in OLD_ACCOUNTS:
user = LocalUser.objects.get(username=username)
for subject in OLD_ACCOUNTS[username]['certs']:
cert = user.x509cert_set.get(subject=subject)
@@ -73,7 +73,7 @@ def activate_old_and_remove_new_accounts(apps, schema_editor):
X509Cert = apps.get_model('ligoauth', 'X509Cert')
# Activate old accounts and add their X509 certificates
- for username in OLD_ACCOUNTS.keys():
+ for username in OLD_ACCOUNTS:
user = LocalUser.objects.get(username=username)
for subject in OLD_ACCOUNTS[username]['certs']:
cert = user.x509cert_set.create(subject=subject)
diff --git a/gracedb/ligoauth/migrations/0019_update_idq_certs.py b/gracedb/ligoauth/migrations/0019_update_idq_certs.py
index 66a6366705582e8a97f36f70b465503d691564ba..0bb25d3e205f0d41241ea0a42f4e08141c908674 100644
--- a/gracedb/ligoauth/migrations/0019_update_idq_certs.py
+++ b/gracedb/ligoauth/migrations/0019_update_idq_certs.py
@@ -19,7 +19,7 @@ ACCOUNTS = {
def update_certs(apps, schema_editor):
User = apps.get_model('auth', 'User')
- for user, certs in ACCOUNTS.iteritems():
+ for user, certs in ACCOUNTS.items():
# Get user
user = User.objects.get(username=user)
@@ -35,7 +35,7 @@ def revert_certs(apps, schema_editor):
User = apps.get_model('auth', 'User')
X509Cert = apps.get_model('ligoauth', 'X509Cert')
- for user, certs in ACCOUNTS.iteritems():
+ for user, certs in ACCOUNTS.items():
# Get user
user = User.objects.get(username=user)
diff --git a/gracedb/ligoauth/migrations/0052_update_gstlalcbc_O3b_cert.py b/gracedb/ligoauth/migrations/0052_update_gstlalcbc_O3b_cert.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc2adc205312cdb3a0774808f4ab940cd3a1df55
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0052_update_gstlalcbc_O3b_cert.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# The new cert and subject line came as a result of gstlalcbc's old
+# certificate expiring on Oct 1, 2019. The new cert was generated
+# on Oct 7, 2019 and expires Oct 7, 2020. So this one should get
+# through O3b.
+
+ACCOUNT = {
+ 'name': 'gstlalcbc',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=cbc.ligo.caltech.edu/CN=gstlalcbc/CN=Chad Hanna/CN=UID:chad.hanna.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('ligoauth', 'RobotUser')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0051_populate_grb_managers_authgroup'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
diff --git a/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py b/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py
new file mode 100644
index 0000000000000000000000000000000000000000..d791b80e08decba08cd62147b926dba829cd90d4
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# The new cert and subject line came as a result of virgo detchar's
+# cert expiring. I (Alex) was contained by Nicolas Arnaud on 10/24/2019
+# via email.
+
+ACCOUNT = {
+ 'name': 'virgo_detchar',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=lscgw.virgo.infn.it/CN=Virgodetchar/CN=Nicolas Arnaud/CN=UID:nicolas.arnaud.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('ligoauth', 'RobotUser')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0052_update_gstlalcbc_O3b_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
diff --git a/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py b/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py
new file mode 100644
index 0000000000000000000000000000000000000000..31cfa20a67dd674172f6729bd4abd75158c15b0e
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+ACCOUNT = {
+ 'name': 'pycbclive',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=ldas-grid.ligo.caltech.edu/CN=pycbclive/CN=Tito Canton/CN=UID:tito.canton.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0053_update_virgodetchar_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
\ No newline at end of file
diff --git a/gracedb/ligoauth/migrations/0055_update_detchar_cert.py b/gracedb/ligoauth/migrations/0055_update_detchar_cert.py
new file mode 100644
index 0000000000000000000000000000000000000000..e63bfa062676d2c201d77536c9a228bc48d23902
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0055_update_detchar_cert.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# Note: the CN in the cert is for detchar-la, not detchar.
+
+ACCOUNT = {
+ 'name': 'detchar',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=detchar.ligo-la.caltech.edu/CN=detchar-la/CN=Alexander Urban/CN=UID:alexander.urban.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0054_update_pycbclive_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
\ No newline at end of file
diff --git a/gracedb/ligoauth/migrations/0056_update_dashboard_cert.py b/gracedb/ligoauth/migrations/0056_update_dashboard_cert.py
new file mode 100644
index 0000000000000000000000000000000000000000..76de8da6d3635320e05c991f4dbc4e778de019e6
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0056_update_dashboard_cert.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# Note: the CN in the cert is for detchar-la, not detchar.
+
+ACCOUNT = {
+ 'name': 'nagios',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=dashboard.ligo.org/CN=NagiosShibScraper/CN=Shawn Kwang/CN=UID:shawn.kwang.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0055_update_detchar_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
diff --git a/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py b/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py
new file mode 100644
index 0000000000000000000000000000000000000000..67459917b6eef0369bde9e4f192a082e16fea6c9
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# Note: the CN in the cert is for detchar-la, not detchar.
+
+ACCOUNT = {
+ 'name': 'gstlalcbc',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=submit.nemo.uwm.edu/CN=gstlal_online_luigi/CN=Duncan Meacher/CN=UID:duncan.meacher.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0056_update_dashboard_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
diff --git a/gracedb/ligoauth/migrations/0058_add_more_detchar_certs.py b/gracedb/ligoauth/migrations/0058_add_more_detchar_certs.py
new file mode 100644
index 0000000000000000000000000000000000000000..dfb8b8e5530af67ee568394266da0f6ff700f8ee
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0058_add_more_detchar_certs.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# detchar is on a tear getting new certs, so I'm doing three
+# at once.
+
+gracedb_account = 'detchar'
+new_certs = ['/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=detchar.ligo-wa.caltech.edu/CN=detchar_ligo-wa/CN=Alexander Urban/CN=UID:alexander.urban.robot',
+ '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=detchar.ligo.caltech.edu/CN=detchar_cit/CN=Alexander Urban/CN=UID:alexander.urban.robot',
+ '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=ldas-pcdev1.ligo-la.caltech.edu/CN=detchar_ldas-pcdev1_ligo-la/CN=Michael Thomas/CN=UID:michael.thomas.robot']
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Create new certificates
+ for cert in new_certs:
+ user.x509cert_set.create(subject=cert)
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Delete new certificates
+ for cert in new_certs:
+ cert = user.x509cert_set.get(subject=cert)
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0057_gstlalcbc_luigi_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
diff --git a/gracedb/ligoauth/migrations/0059_grb_exttrig_cert.py b/gracedb/ligoauth/migrations/0059_grb_exttrig_cert.py
new file mode 100644
index 0000000000000000000000000000000000000000..b27c286f278216e8b33805130f759e1b91341d76
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0059_grb_exttrig_cert.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# detchar is on a tear getting new certs, so I'm doing three
+# at once.
+
+gracedb_account = 'grb.exttrig'
+new_certs = ['/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=ldas-pcdev1.ligo.caltech.edu/CN=grb.exttrig/CN=Ryan Fisher/CN=UID:ryan.fisher.robot',]
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Create new certificates
+ for cert in new_certs:
+ user.x509cert_set.create(subject=cert)
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Delete new certificates
+ for cert in new_certs:
+ cert = user.x509cert_set.get(subject=cert)
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0058_add_more_detchar_certs'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
\ No newline at end of file
diff --git a/gracedb/ligoauth/migrations/0060_add_emfollow-test_cert.py b/gracedb/ligoauth/migrations/0060_add_emfollow-test_cert.py
new file mode 100644
index 0000000000000000000000000000000000000000..ca6b795a270784c7d21f08e961610fbfb0ea6cc2
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0060_add_emfollow-test_cert.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# detchar is on a tear getting new certs, so I'm doing three
+# at once.
+
+gracedb_account = 'emfollow'
+new_certs = ['/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=emfollow-test.ligo.caltech.edu/CN=emfollow-test/CN=Leo Singer/CN=UID:leo.singer.robot',]
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Create new certificates
+ for cert in new_certs:
+ user.x509cert_set.create(subject=cert)
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Delete new certificates
+ for cert in new_certs:
+ cert = user.x509cert_set.get(subject=cert)
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0059_grb_exttrig_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
\ No newline at end of file
diff --git a/gracedb/ligoauth/models.py b/gracedb/ligoauth/models.py
index 8dfad9e640ee5345e71acef3fcde6dbacce77782..238bb110f0d35da70fd0daeaa941ada81f9bc7f5 100644
--- a/gracedb/ligoauth/models.py
+++ b/gracedb/ligoauth/models.py
@@ -12,7 +12,7 @@ class LigoLdapUser(User):
def name(self):
# XXX I really don't freaking understand WHY THIS SEEMS NECESSARY.
- # print user.name() gives an idiotic ascii coding error otherwise. WHY!?
+ # print(user.name()) gives an idiotic ascii coding error otherwise. WHY!?
return u"{0} {1}".format(self.first_name, self.last_name).encode('utf-8')
diff --git a/gracedb/ligoauth/tests/test_backends.py b/gracedb/ligoauth/tests/test_backends.py
index 7a485526a60dfd46a6f13133e8ba23107e2af985..8505ee8b249f526088228d34a834463113cbe2be 100644
--- a/gracedb/ligoauth/tests/test_backends.py
+++ b/gracedb/ligoauth/tests/test_backends.py
@@ -48,7 +48,7 @@ class TestShibbolethRemoteUserBackend(GraceDbTestBase):
# Set up request and headers
request = self.factory.get(self.url)
for k,v in user_data.items():
- if settings.SHIB_ATTRIBUTE_MAP.has_key(k):
+ if k in settings.SHIB_ATTRIBUTE_MAP:
request.META[settings.SHIB_ATTRIBUTE_MAP[k]] = v
# Pass data to backend
@@ -71,7 +71,7 @@ class TestShibbolethRemoteUserBackend(GraceDbTestBase):
# Set up request and headers
request = self.factory.get(self.url)
for k,v in new_user_data.items():
- if settings.SHIB_ATTRIBUTE_MAP.has_key(k):
+ if k in settings.SHIB_ATTRIBUTE_MAP:
request.META[settings.SHIB_ATTRIBUTE_MAP[k]] = v
# Get initial user data
diff --git a/gracedb/migrations/auth/0016_create_access_and_superevent_groups.py b/gracedb/migrations/auth/0016_create_access_and_superevent_groups.py
index 461952d763f01fc5d220c6924ba73b4f2a14c251..03eae397012ed75928640e64410e4de05a1a1ed2 100644
--- a/gracedb/migrations/auth/0016_create_access_and_superevent_groups.py
+++ b/gracedb/migrations/auth/0016_create_access_and_superevent_groups.py
@@ -17,7 +17,7 @@ def add_groups(apps, schema_editor):
Group = apps.get_model('auth', 'Group')
User = apps.get_model('auth', 'User')
- for group_name, usernames in GROUPS.iteritems():
+ for group_name, usernames in GROUPS.items():
g, _ = Group.objects.get_or_create(name=group_name)
users = User.objects.filter(username__in=usernames)
g.user_set.add(*users)
diff --git a/gracedb/migrations/auth/0017_assign_permissions.py b/gracedb/migrations/auth/0017_assign_permissions.py
index 317d2727aa0cd976c2f28414cf11ae1e1917a452..2c8fd59c87d5e3fe39529f0f4e1e6fe1d5d1a598 100644
--- a/gracedb/migrations/auth/0017_assign_permissions.py
+++ b/gracedb/migrations/auth/0017_assign_permissions.py
@@ -8,7 +8,6 @@ from django.contrib.auth.management import create_permissions
# Group names
-LVC = settings.LVC_GROUP
EXECS = 'executives'
ACCESS = 'access_managers' # control external access
SUPEREVENTS = 'superevent_managers' # handle superevent creation/update
@@ -16,6 +15,9 @@ H1_CONTROL = 'h1_control_room'
L1_CONTROL = 'l1_control_room'
V1_CONTROL = 'v1_control_room'
EM_ADVOCATES = 'em_advocates'
+# Previously, this was teaken from settings.LVC_GROUP, but that value has
+# changed. So we have to hard-code it for past migrations
+LVC = 'Communities:LSCVirgoLIGOGroupMembers'
SUPEREVENT_PERMS = {
# Label permissions
@@ -74,7 +76,7 @@ def add_perms(apps, schema_editor):
Permission = apps.get_model('auth', 'Permission')
# Add superevent permissions to groups
- for codename, group_names in SUPEREVENT_PERMS.iteritems():
+ for codename, group_names in SUPEREVENT_PERMS.items():
p = Permission.objects.get(codename=codename,
content_type__app_label='superevents')
groups = Group.objects.filter(name__in=group_names)
@@ -86,7 +88,7 @@ def remove_perms(apps, schema_editor):
Permission = apps.get_model('auth', 'Permission')
# Add permissions to groups
- for codename, group_names in SUPEREVENT_PERMS.iteritems():
+ for codename, group_names in SUPEREVENT_PERMS.items():
p = Permission.objects.get(codename=codename,
content_type__app_label='superevents')
groups = Group.objects.filter(name__in=group_names)
diff --git a/gracedb/migrations/auth/0024_auto_20190919_1957.py b/gracedb/migrations/auth/0024_auto_20190919_1957.py
new file mode 100644
index 0000000000000000000000000000000000000000..f05eb14055a82c03b7dbb7320e57ea62951f1b84
--- /dev/null
+++ b/gracedb/migrations/auth/0024_auto_20190919_1957.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-09-19 19:57
+# This was auto-generated after moving to Python 3, with no changes to the
+# actual models. See the commit message for more details.
+from __future__ import unicode_literals
+
+import django.contrib.auth.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('auth', '0023_add_manage_pipeline_permissions'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='user',
+ name='username',
+ field=models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username'),
+ ),
+ ]
diff --git a/gracedb/migrations/guardian/0003_update_emfollow_accounts.py b/gracedb/migrations/guardian/0003_update_emfollow_accounts.py
index 3f799dbfb6641bdfe9aaaa12a3d36a04f185bf1e..68315728c6b6c744fc8e9822aeb5f5f35822c104 100644
--- a/gracedb/migrations/guardian/0003_update_emfollow_accounts.py
+++ b/gracedb/migrations/guardian/0003_update_emfollow_accounts.py
@@ -26,7 +26,7 @@ def update_perms(apps, schema_editor):
ctype = ContentType.objects.get_for_model(Pipeline)
# Remove UOPs from old accounts which allow pipeline population
- for username, pipelines in OLD_ACCOUNTS.iteritems():
+ for username, pipelines in OLD_ACCOUNTS.items():
user = LocalUser.objects.get(username=username)
for pipeline_name in pipelines:
pipeline = Pipeline.objects.get(name=pipeline_name)
@@ -53,7 +53,7 @@ def revert_perms(apps, schema_editor):
ctype = ContentType.objects.get_for_model(Pipeline)
# Re-add UOPs to old accounts which allow pipeline population
- for username, pipelines in OLD_ACCOUNTS.iteritems():
+ for username, pipelines in OLD_ACCOUNTS.items():
user = LocalUser.objects.get(username=username)
for pipeline_name in pipelines:
pipeline = Pipeline.objects.get(name=pipeline_name)
diff --git a/gracedb/search/constants.py b/gracedb/search/constants.py
index 8f55825883da4d607af1ccff851abcac442ac55f..d6d77fe35e341856a4444c4bbfbe014c9d2b9038 100644
--- a/gracedb/search/constants.py
+++ b/gracedb/search/constants.py
@@ -11,7 +11,7 @@ EXPR_OPERATORS = {
">": "__gt",
">=": "__gte",
}
-ExpressionOperator = Or(map(Literal, EXPR_OPERATORS.keys()))
+ExpressionOperator = Or(list(map(Literal, list(EXPR_OPERATORS))))
ExpressionOperator.setParseAction(lambda toks: EXPR_OPERATORS[toks[0]])
diff --git a/gracedb/search/fields.py b/gracedb/search/fields.py
index f456dc3f449f300396eb904ae3aa4f2ab909e360..eeb1f197b1632da7284b1a72d7cb6ac67188af66 100644
--- a/gracedb/search/fields.py
+++ b/gracedb/search/fields.py
@@ -29,14 +29,14 @@ class GraceQueryField(forms.CharField):
try:
qs = self.do_filtering(queryString)
return qs.distinct()
- except ParseException, e:
+ except ParseException as e:
err = "Error: " + escape(e.pstr[:e.loc]) + errorMarker + escape(e.pstr[e.loc:])
raise forms.ValidationError(mark_safe(err))
- except FieldError, e:
+ except FieldError as e:
# XXX error message can be more polished than this
err = "Error: " + str(e)
raise forms.ValidationError(mark_safe(err))
- except Exception, e:
+ except Exception as e:
# What could this be and how can we handle it better? XXX
raise forms.ValidationError(str(e)+str(type(e)))
diff --git a/gracedb/search/query/events.py b/gracedb/search/query/events.py
index 10c5e5b502d0cad6b64b7a53f84e4d51a5a7194e..602b6015198adc7a4a771825220786cb8a3dcd96 100644
--- a/gracedb/search/query/events.py
+++ b/gracedb/search/query/events.py
@@ -15,6 +15,10 @@ from pyparsing import Word, nums, Literal, CaselessLiteral, delimitedList, \
oneOf, stringStart, stringEnd, FollowedBy, ParseResults, ParseException, \
CaselessKeyword
import pytz
+try:
+ from functools import reduce
+except ImportError: # python < 3
+ pass
from django.db.models import Q
from django.db.models.query import QuerySet
@@ -41,7 +45,7 @@ gpsQ = Optional(Suppress(Keyword("gpstime:"))) + (gpstime^gpstimeRange)
gpsQ = gpsQ.setParseAction(maybeRange("gpstime"))
# run ids
-runid = Or(map(CaselessLiteral, RUN_MAP.keys())).setName("run id")
+runid = Or(list(map(CaselessLiteral, list(RUN_MAP)))).setName("run id")
#runidList = OneOrMore(runid).setName("run id list")
runQ = (Optional(Suppress(Keyword("runid:"))) + runid)
runQ = runQ.setParseAction(lambda toks: ("gpstime", Q(gpstime__range=
@@ -94,7 +98,7 @@ submitterQ = submitterQ.setParseAction(lambda toks: ("submitter", toks[0]))
nltimeRange = nltime + Suppress("..") + nltime
def doTime(tok):
- x = datetime.datetime(*(map(int, tok)))
+ x = datetime.datetime(*(list(map(int, tok))))
return pytz.utc.localize(x)
dash = Suppress('-')
@@ -253,7 +257,7 @@ def parseQuery(s):
# XXX Querying the database at module compile time is a bad idea!
# See: https://docs.djangoproject.com/en/1.8/topics/testing/overview/
groupNames = list(Group.objects.values_list('name', flat=True))
- group = Or(map(CaselessLiteral, groupNames)).setName("analysis group name")
+ group = Or(list(map(CaselessLiteral, groupNames))).setName("analysis group name")
#groupList = delimitedList(group, delim='|').setName("analysis group list")
groupList = OneOrMore(group).setName("analysis group list")
groupQ = (Optional(Suppress(Keyword("group:"))) + groupList)
@@ -262,7 +266,7 @@ def parseQuery(s):
# Pipeline
pipelineNames = list(Pipeline.objects.values_list('name', flat=True))
- pipeline = Or(map(CaselessLiteral, pipelineNames)).setName("pipeline name")
+ pipeline = Or(list(map(CaselessLiteral, pipelineNames))).setName("pipeline name")
pipelineList = OneOrMore(pipeline).setName("pipeline list")
pipelineQ = (Optional(Suppress(Keyword("pipeline:"))) + pipelineList)
pipelineQ = pipelineQ.setParseAction(lambda toks: ("pipeline",
@@ -270,7 +274,7 @@ def parseQuery(s):
# Search
searchNames = list(Search.objects.values_list('name', flat=True))
- search = Or(map(CaselessLiteral, searchNames)).setName("search name")
+ search = Or(list(map(CaselessLiteral, searchNames))).setName("search name")
# XXX Branson: The change below was made 2/17/15 to fix a bug in which
# searches like 'grbevent.ra > 0' failed due to the 'grb' being peeled off
# and assumed to be part of a 'Search' query. So we don't consume a token
@@ -335,4 +339,4 @@ def parseQuery(s):
if "id" in d and "hid" in d:
d["id"] = d["id"] | d["hid"]
del d["hid"]
- return reduce(Q.__and__, d.values(), Q())
+ return reduce(Q.__and__, list(d.values()), Q())
diff --git a/gracedb/search/query/superevents.py b/gracedb/search/query/superevents.py
index c5277c591ba696ddbba0f7ea0ddef034e710196d..909f68554492f246f3f6f3c91b8de9ad73ae863c 100644
--- a/gracedb/search/query/superevents.py
+++ b/gracedb/search/query/superevents.py
@@ -2,6 +2,10 @@ from __future__ import absolute_import
import datetime
import logging
import pytz
+try:
+ from functools import reduce
+except ImportError: # python < 3
+ pass
from django.conf import settings
from django.db.models import Q
@@ -70,7 +74,7 @@ superevent_preprefix = Optional(Or([CaselessLiteral(pref) for pref in
[Superevent.SUPEREVENT_CATEGORY_TEST, Superevent.SUPEREVENT_CATEGORY_MDC]])
).setResultsName('preprefix')
superevent_prefix = Or([CaselessLiteral(pref) for pref in
- Superevent.DEFAULT_ID_PREFIX, Superevent.GW_ID_PREFIX]).setResultsName('prefix')
+ (Superevent.DEFAULT_ID_PREFIX, Superevent.GW_ID_PREFIX)]).setResultsName('prefix')
superevent_date = Word(nums, exact=6).setResultsName('date')
superevent_suffix = Word(alphas).setResultsName('suffix')
superevent_expr = superevent_preprefix + superevent_prefix + \
@@ -92,7 +96,8 @@ parameter_dicts = {
'runid': {
'keyword': 'runid',
'keywordOptional': True,
- 'value': Or(map(CaselessLiteral, RUN_MAP.keys())).setName("run id"),
+ 'value': Or(list(map(CaselessLiteral, list(RUN_MAP)))).setName(
+ "run id"),
'doRange': False,
'parseAction': lambda toks: ("t_0", Q(t_0__range=RUN_MAP[toks[0]])),
},
@@ -190,7 +195,7 @@ parameter_dicts = {
Word(nums, exact=2) + Optional(Suppress(':') + \
Word(nums, exact=2)))).setParseAction(lambda toks:
pytz.timezone(settings.TIME_ZONE).localize(
- datetime.datetime(*map(int, toks)))),
+ datetime.datetime(*list(map(int, toks))))),
'parseAction': maybeRange("created"),
},
# test OR category: test
@@ -227,21 +232,21 @@ parameter_dicts = {
# Compile a list of expressions to try to match
expr_list = []
-for k,p in parameter_dicts.iteritems():
+for k,p in parameter_dicts.items():
# Define val and set name
val = p['value']
val.setName(k)
# Add range with format: parameter .. parameter
- if p.has_key('doRange') and p['doRange']:
+ if p.get('doRange'):
range_val = val + Suppress("..") + val
val ^= range_val
# Add keyword. Format is keyword: value
- if p.has_key('keyword'):
+ if 'keyword' in p:
if isinstance(p['keyword'], list):
- if p.has_key('keywordOptional') and p['keywordOptional']:
+ if p.get('keywordOptional'):
keyword_list = [Optional(Suppress(Keyword(k + ":"))) for k in
p['keyword']]
else:
@@ -249,7 +254,7 @@ for k,p in parameter_dicts.iteritems():
keyword = reduce(lambda x,y: x^y, keyword_list)
else:
keyword = Suppress(Keyword(p['keyword'] + ":"))
- if p.has_key('keywordOptional') and p['keywordOptional']:
+ if p.get('keywordOptional'):
keyword = Optional(keyword)
# Combine keyword and value into a single expression
diff --git a/gracedb/search/tests/test_access.py b/gracedb/search/tests/test_access.py
index 77f8317bf836a70d859112c676f68ccb4b36bacf..25de155a6824b4d27fdb711b9beff4cbc3838763 100644
--- a/gracedb/search/tests/test_access.py
+++ b/gracedb/search/tests/test_access.py
@@ -1,10 +1,9 @@
-import urllib
-
from django.conf import settings
from django.contrib.auth.models import Group as DjangoGroup, Permission
from django.contrib.contenttypes.models import ContentType
from django.test import TestCase
from django.urls import reverse
+from django.utils.http import urlencode
from core.tests.utils import GraceDbTestBase
from events.models import Event
@@ -34,7 +33,7 @@ class TestEventSearch(EventCreateMixin, GraceDbTestBase, SearchTestingBase):
'query_type': MainSearchForm.QUERY_TYPE_EVENT,
'results_format': MainSearchForm.FORMAT_CHOICE_STANDARD,
}
- cls.full_url = cls.url + '?' + urllib.urlencode(cls.query_dict)
+ cls.full_url = cls.url + '?' + urlencode(cls.query_dict)
@classmethod
def setUpTestData(cls):
@@ -104,7 +103,7 @@ class TestSupereventSearch(SupereventSetup, GraceDbTestBase,
'query_type': MainSearchForm.QUERY_TYPE_SUPEREVENT,
'results_format': MainSearchForm.FORMAT_CHOICE_STANDARD,
}
- cls.full_url = cls.url + '?' + urllib.urlencode(cls.query_dict)
+ cls.full_url = cls.url + '?' + urlencode(cls.query_dict)
def test_internal_user_search(self):
"""Internal user sees all superevents in search results"""
@@ -150,7 +149,7 @@ class TestEventLatest(EventCreateMixin, GraceDbTestBase):
'results_format': MainSearchForm.FORMAT_CHOICE_STANDARD,
}
cls.url = reverse('latest')
- cls.full_url = cls.url + '?' + urllib.urlencode(cls.query_dict)
+ cls.full_url = cls.url + '?' + urlencode(cls.query_dict)
@classmethod
def setUpTestData(cls):
@@ -183,7 +182,7 @@ class TestEventLatest(EventCreateMixin, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure all events are shown
- self.assertIn('events', response.context.keys())
+ self.assertIn('events', response.context)
for e in Event.objects.all():
self.assertIn(e, response.context['events'])
@@ -194,7 +193,7 @@ class TestEventLatest(EventCreateMixin, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure only exposed events are shown
- self.assertIn('events', response.context.keys())
+ self.assertIn('events', response.context)
self.assertIn(self.lvem_event, response.context['events'])
self.assertEqual(len(response.context['events']), 1)
@@ -207,7 +206,7 @@ class TestEventLatest(EventCreateMixin, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure only exposed events are shown
- self.assertIn('events', response.context.keys())
+ self.assertIn('events', response.context)
self.assertEqual(len(response.context['events']), 0)
@@ -224,7 +223,7 @@ class TestSupereventLatest(SupereventSetup, GraceDbTestBase):
'results_format': MainSearchForm.FORMAT_CHOICE_STANDARD,
}
cls.url = reverse('latest')
- cls.full_url = cls.url + '?' + urllib.urlencode(cls.query_dict)
+ cls.full_url = cls.url + '?' + urlencode(cls.query_dict)
def test_internal_user_latest(self):
"""Internal user sees all superevents on latest page"""
@@ -233,7 +232,7 @@ class TestSupereventLatest(SupereventSetup, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure all superevents are shown
- self.assertIn('superevents', response.context.keys())
+ self.assertIn('superevents', response.context)
for s in Superevent.objects.all():
self.assertIn(s, response.context['superevents'])
@@ -244,7 +243,7 @@ class TestSupereventLatest(SupereventSetup, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure only exposed superevents are shown
- self.assertIn('superevents', response.context.keys())
+ self.assertIn('superevents', response.context)
self.assertIn(self.lvem_superevent, response.context['superevents'])
self.assertIn(self.public_superevent, response.context['superevents'])
self.assertEqual(len(response.context['superevents']), 2)
@@ -255,6 +254,6 @@ class TestSupereventLatest(SupereventSetup, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure only exposed superevents are shown
- self.assertIn('superevents', response.context.keys())
+ self.assertIn('superevents', response.context)
self.assertIn(self.public_superevent, response.context['superevents'])
self.assertEqual(len(response.context['superevents']), 1)
diff --git a/gracedb/superevents/buildVOEvent.py b/gracedb/superevents/buildVOEvent.py
deleted file mode 100644
index 10bbdaaa65b0f3d66959706a4211ad0129077450..0000000000000000000000000000000000000000
--- a/gracedb/superevents/buildVOEvent.py
+++ /dev/null
@@ -1,456 +0,0 @@
-
-# Taken from VOEventLib example code, which is:
-# Copyright 2010 Roy D. Williams
-# then modified
-"""
-buildVOEvent: Creates a complex VOEvent with tables
-See the VOEvent specification for details
-http://www.ivoa.net/Documents/latest/VOEvent.html
-"""
-from scipy.constants import c, G, pi
-
-
-from VOEventLib.VOEvent import VOEvent, Who, Author, Param, How, What, Group
-from VOEventLib.VOEvent import Citations, EventIVORN
-from VOEventLib.Vutil import stringVOEvent
-
-from VOEventLib.VOEvent import AstroCoords, AstroCoordSystem
-from VOEventLib.VOEvent import ObservationLocation, ObservatoryLocation
-from VOEventLib.VOEvent import ObsDataLocation, WhereWhen
-from VOEventLib.VOEvent import Time, TimeInstant
-
-from core.urls import build_absolute_uri
-from core.time_utils import gpsToUtc
-from django.conf import settings
-from django.urls import reverse
-from django.utils import timezone
-from django.db.models import Min
-from events.models import CoincInspiralEvent, MultiBurstEvent, \
- LalInferenceBurstEvent
-from .models import VOEvent as GraceDBVOEvent
-
-import os
-import logging
-logger = logging.getLogger(__name__)
-
-VOEVENT_TYPE_DICT = dict(GraceDBVOEvent.VOEVENT_TYPE_CHOICES)
-
-class VOEventBuilderException(Exception):
- pass
-
-# Used to create the Packet_Type parameter block
-PACKET_TYPES = {
- GraceDBVOEvent.VOEVENT_TYPE_PRELIMINARY: (150, 'LVC_PRELIMINARY'),
- GraceDBVOEvent.VOEVENT_TYPE_INITIAL: (151, 'LVC_INITIAL'),
- GraceDBVOEvent.VOEVENT_TYPE_UPDATE: (152, 'LVC_UPDATE'),
- GraceDBVOEvent.VOEVENT_TYPE_RETRACTION: (164, 'LVC_RETRACTION'),
-}
-
-def get_voevent_type(short_name):
- for t in GraceDBVOEvent.VOEVENT_TYPE_CHOICES:
- if short_name in t:
- return t[1]
- return None
-
-
-def construct_voevent_file(superevent, voevent, request=None):
-
- # Set preferred_event as event to be used in most of this
- # Get the event subclass (CoincInspiralEvent, MultiBurstEvent, etc.) and
- # set that as the event
- event = superevent.preferred_event.get_subclass_or_self()
-
- # Let's convert that voevent_type to something nicer looking
- voevent_type = VOEVENT_TYPE_DICT[voevent.voevent_type]
-
- # Now build the IVORN.
- type_string = voevent_type.capitalize()
- voevent_id = '{s_id}-{N}-{type_str}'.format(type_str=type_string,
- s_id=superevent.default_superevent_id, N=voevent.N)
- ivorn = settings.IVORN_PREFIX + voevent_id
-
- ############ VOEvent header ############################
- v = VOEvent(version="2.0")
- v.set_ivorn(ivorn)
-
- if event.search and event.search.name == 'MDC':
- v.set_role("test")
- elif event.group.name == 'Test':
- v.set_role("test")
- else:
- v.set_role("observation")
- if voevent_type != 'retraction':
- v.set_Description(settings.SKYALERT_DESCRIPTION)
-
- ############ Who ############################
- w = Who()
- a = Author()
- a.add_contactName("LIGO Scientific Collaboration and Virgo Collaboration")
- #a.add_contactEmail("postmaster@ligo.org")
- w.set_Author(a)
- w.set_Date(timezone.now().strftime("%Y-%m-%dT%H:%M:%S"))
- v.set_Who(w)
-
- ############ How ############################
-
- if voevent_type != 'retraction':
- h = How()
- h.add_Description("Candidate gravitational wave event identified by low-latency analysis")
- instruments = event.instruments.split(',')
- if 'H1' in instruments:
- h.add_Description("H1: LIGO Hanford 4 km gravitational wave detector")
- if 'L1' in instruments:
- h.add_Description("L1: LIGO Livingston 4 km gravitational wave detector")
- if 'V1' in instruments:
- h.add_Description("V1: Virgo 3 km gravitational wave detector")
- if voevent.coinc_comment:
- h.add_Description("A gravitational wave trigger identified a possible counterpart GRB")
- v.set_How(h)
-
- ############ What ############################
- w = What()
-
- # UCD = Unified Content Descriptors
- # http://monet.uni-sw.gwdg.de/twiki/bin/view/VOEvent/UnifiedContentDescriptors
- # OR -- (from VOTable document, [21] below)
- # http://www.ivoa.net/twiki/bin/view/IVOA/IvoaUCD
- # http://cds.u-strasbg.fr/doc/UCD.htx
- #
- # which somehow gets you to: http://www.ivoa.net/Documents/REC/UCD/UCDlist-20070402.html
- # where you might find some actual information.
-
- # Unit / Section 4.3 of [21] which relies on [25]
- # [21] http://www.ivoa.net/Documents/latest/VOT.html
- # [25] http://vizier.u-strasbg.fr/doc/catstd-3.2.htx
- #
- # basically, a string that makes sense to humans about what units a value is. eg. "m/s"
-
- # Add Packet_Type for GCNs
- w.add_Param(Param(name="Packet_Type",
- value=PACKET_TYPES[voevent.voevent_type][0], dataType="int",
- Description=[("The Notice Type number is assigned/used within GCN, eg "
- "type={typenum} is an {typedesc} notice").format(
- typenum=PACKET_TYPES[voevent.voevent_type][0],
- typedesc=PACKET_TYPES[voevent.voevent_type][1])]))
-
- # Whether the alert is internal or not
- w.add_Param(Param(name="internal", value=int(voevent.internal),
- dataType="int", Description=['Indicates whether this event should be '
- 'distributed to LSC/Virgo members only']))
-
- # The serial number
- w.add_Param(Param(name="Pkt_Ser_Num", value=voevent.N,
- Description=["A number that increments by 1 each time a new revision "
- "is issued for this event"]))
-
- # The superevent ID
- w.add_Param(Param(name="GraceID",
- dataType="string",
- ucd="meta.id",
- value=superevent.default_superevent_id,
- Description=["Identifier in GraceDB"]))
-
- # Alert type parameter
- w.add_Param(Param(name="AlertType",
- dataType="string",
- ucd="meta.version",
- value = voevent_type.capitalize(),
- Description=["VOEvent alert type"]))
-
- # Whether the event is a hardware injection or not
- w.add_Param(Param(name="HardwareInj",
- dataType="int",
- ucd="meta.number",
- value=int(voevent.hardware_inj),
- Description=['Indicates that this event is a hardware injection if 1, no if 0']))
-
- w.add_Param(Param(name="OpenAlert",
- dataType="int",
- ucd="meta.number",
- value=int(voevent.open_alert),
- Description=['Indicates that this event is an open alert if 1, no if 0']))
-
- # Superevent page
- w.add_Param(Param(name="EventPage",
- ucd="meta.ref.url",
- value=build_absolute_uri(reverse("superevents:view",
- args=[superevent.default_superevent_id]), request),
- Description=["Web page for evolving status of this GW candidate"]))
-
- if voevent_type != 'retraction':
- # Instruments
- w.add_Param(Param(name="Instruments",
- dataType="string",
- ucd="meta.code",
- value=event.instruments,
- Description=["List of instruments used in analysis to identify this event"]))
-
- # False alarm rate
- if event.far:
- w.add_Param(Param(name="FAR",
- dataType="float",
- ucd="arith.rate;stat.falsealarm",
- unit="Hz",
- value=float(max(event.far, settings.VOEVENT_FAR_FLOOR)),
- Description=["False alarm rate for GW candidates with this strength or greater"]))
-
- # Group
- w.add_Param(Param(name="Group",
- dataType="string",
- ucd="meta.code",
- value=event.group.name,
- Description=["Data analysis working group"]))
-
- # Pipeline
- w.add_Param(Param(name="Pipeline",
- dataType="string",
- ucd="meta.code",
- value=event.pipeline.name,
- Description=["Low-latency data analysis pipeline"]))
-
- # Search
- if event.search:
- w.add_Param(Param(name="Search",
- ucd="meta.code",
- dataType="string",
- value=event.search.name,
- Description=["Specific low-latency search"]))
-
- # initial and update VOEvents must have a skymap.
- # new feature (10/24/2016): preliminary VOEvents can have a skymap,
- # but they don't have to.
- if (voevent_type in ["initial", "update"] or
- (voevent_type == "preliminary" and voevent.skymap_filename != None)):
-
- # Skymaps. Create group and set fits file name
- g = Group('GW_SKYMAP', voevent.skymap_type)
-
- fits_skymap_url = build_absolute_uri(reverse(
- "api:default:superevents:superevent-file-detail",
- args=[superevent.default_superevent_id, voevent.skymap_filename]),
- request)
-
- # Add parameters to the skymap group
- g.add_Param(Param(name="skymap_fits", dataType="string",
- ucd="meta.ref.url", value=fits_skymap_url,
- Description=["Sky Map FITS"]))
-
- w.add_Group(g)
-
- # Analysis specific attributes
- if voevent_type != 'retraction':
- classification_group = Group('Classification', Description=["Source "
- "classification: binary neutron star (BNS), neutron star-black "
- "hole (NSBH), binary black hole (BBH), MassGap, or terrestrial "
- "(noise)"])
- properties_group = Group('Properties', Description=["Qualitative "
- "properties of the source, conditioned on the assumption that the "
- "signal is an astrophysical compact binary merger"])
- if isinstance(event, CoincInspiralEvent) and voevent_type != 'retraction':
- # get mchirp and mass
- mchirp = float(event.mchirp)
- mass = float(event.mass)
- # calculate eta = (mchirp/total_mass)**(5/3)
- eta = pow((mchirp/mass),5.0/3.0)
-
- # EM-Bright mass classifier information for CBC event candidates
- if voevent.prob_bns is not None:
- classification_group.add_Param(Param(name="BNS",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_bns, Description=["Probability that "
- "the source is a binary neutron star merger (both objects "
- "lighter than 3 solar masses)"]))
-
- if voevent.prob_nsbh is not None:
- classification_group.add_Param(Param(name="NSBH",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_nsbh, Description=["Probability that "
- "the source is a neutron star-black hole merger (primary "
- "heavier than 5 solar masses, secondary lighter than 3 "
- "solar masses)"]))
-
- if voevent.prob_bbh is not None:
- classification_group.add_Param(Param(name="BBH",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_bbh, Description=["Probability that "
- "the source is a binary black hole merger (both objects "
- "heavier than 5 solar masses)"]))
-
- if voevent.prob_mass_gap is not None:
- classification_group.add_Param(Param(name="MassGap",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_mass_gap,
- Description=["Probability that the source has at least "
- "one object between 3 and 5 solar masses"]))
-
- if voevent.prob_terrestrial is not None:
- classification_group.add_Param(Param(name="Terrestrial",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_terrestrial, Description=["Probability "
- "that the source is terrestrial (i.e., a background noise "
- "fluctuation or a glitch)"]))
-
- # Add to source properties group
- if voevent.prob_has_ns is not None:
- properties_group.add_Param(Param(name="HasNS",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_has_ns,
- Description=["Probability that at least one object in the "
- "binary has a mass that is less than 3 solar masses"]))
-
- if voevent.prob_has_remnant is not None:
- properties_group.add_Param(Param(name="HasRemnant",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_has_remnant, Description=["Probability "
- "that a nonzero mass was ejected outside the central "
- "remnant object"]))
-
- # build up MaxDistance. event.singleinspiral_set.all()?
- # Each detector calculates an effective distance assuming the inspiral is
- # optimally oriented. It is the maximum distance at which a source of the
- # given parameters would've been seen by that particular detector. To get
- # an effective 'maximum distance', we just find the minumum over detectors
- max_distance = event.singleinspiral_set.all().aggregate(
- max_dist=Min('eff_distance'))
- max_distance = max_distance or float('inf')
-
- elif isinstance(event, MultiBurstEvent):
- w.add_Param(Param(name="CentralFreq",
- dataType="float",
- ucd="gw.frequency",
- unit="Hz",
- value=float(event.central_freq),
- Description=["Central frequency of GW burst signal"]))
- w.add_Param(Param(name="Duration",
- dataType="float",
- ucd="time.duration",
- unit="s",
- value=float(event.duration),
- Description=["Measured duration of GW burst signal"]))
-
- # XXX Calculate the fluence. Unfortunately, this requires parsing the trigger.txt
- # file for hrss values. These should probably be pulled into the database.
- # But there is no consensus on whether hrss or fluence is meaningful. So I will
- # put off changing the schema for now.
- try:
- # Go find the data file.
- log = event.eventlog_set.filter(comment__startswith="Original Data").all()[0]
- filename = log.filename
- filepath = os.path.join(event.datadir,filename)
- if os.path.isfile(filepath):
- datafile = open(filepath,"r")
- else:
- raise VOEventBuilderException("No file found.")
- # Now parse the datafile.
- # The line we want looks like:
- # hrss: 1.752741e-23 2.101590e-23 6.418900e-23
- for line in datafile:
- if line.startswith('hrss:'):
- hrss_values = [float(hrss) for hrss in line.split()[1:]]
- max_hrss = max(hrss_values)
- # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
- # Note that hrss here actually has units of s^(-1/2)
- fluence = pi * pow(c,3) * pow(event.central_freq,2)
- fluence = fluence * pow(max_hrss,2)
- fluence = fluence / (4.0*G)
-
- w.add_Param(Param(name="Fluence",
- dataType="float",
- ucd="gw.fluence",
- unit="erg/cm^2",
- value=fluence,
- Description=["Estimated fluence of GW burst signal"]))
- except Exception as e:
- logger.exception(e)
-
- elif isinstance(event, LalInferenceBurstEvent):
- w.add_Param(Param(name="frequency",
- dataType="float",
- ucd="gw.frequency",
- unit="Hz",
- value=float(event.frequency_mean),
- Description=["Mean frequency of GW burst signal"]))
-
- # Calculate the fluence.
- # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
- # Note that hrss here actually has units of s^(-1/2)
- # XXX obviously need to refactor here.
- try:
- fluence = pi * pow(c,3) * pow(event.frequency,2)
- fluence = fluence * pow(event.hrss,2)
- fluence = fluence / (4.0*G)
-
- w.add_Param(Param(name="Fluence",
- dataType="float",
- ucd="gw.fluence",
- unit="erg/cm^2",
- value=fluence,
- Description=["Estimated fluence of GW burst signal"]))
- except Exception as e:
- logger.exception(e)
-
- # Add Groups to What block
- w.add_Group(classification_group)
- w.add_Group(properties_group)
-
- v.set_What(w)
-
- ############ Wherewhen ############################
-# The old way of making the WhereWhen section led to a pointless position
-# location.
-# wwd = {'observatory': 'LIGO Virgo',
-# 'coord_system': 'UTC-FK5-GEO',
-# # XXX time format
-# 'time': str(gpsToUtc(event.gpstime).isoformat())[:-6], #'1918-11-11T11:11:11',
-# #'timeError': 1.0,
-# 'longitude': 0.0,
-# 'latitude': 0.0,
-# 'positionalError': 180.0,
-# }
-#
-# ww = makeWhereWhen(wwd)
-# if ww: v.set_WhereWhen(ww)
-
- coord_system_id = 'UTC-FK5-GEO'
- event_time = str(gpsToUtc(event.gpstime).isoformat())[:-6]
- observatory_id = 'LIGO Virgo'
- ac = AstroCoords(coord_system_id=coord_system_id)
- acs = AstroCoordSystem(id=coord_system_id)
- ac.set_Time(Time(TimeInstant = TimeInstant(event_time)))
-
- onl = ObservationLocation(acs, ac)
- oyl = ObservatoryLocation(id=observatory_id)
- odl = ObsDataLocation(oyl, onl)
- ww = WhereWhen()
- ww.set_ObsDataLocation(odl)
- v.set_WhereWhen(ww)
-
- ############ Citation ############################
- if superevent.voevent_set.count() > 1:
- c = Citations()
- for ve in superevent.voevent_set.all():
- # Oh, actually we need to exclude *this* voevent.
- if ve.N == voevent.N:
- continue
- if voevent_type == 'initial':
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Initial localization is now available')
- elif voevent_type == 'update':
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Updated localization is now available')
- elif voevent_type == 'retraction':
- ei = EventIVORN('retraction', ve.ivorn)
- c.set_Description('Determined to not be a viable GW event candidate')
- elif voevent_type == 'preliminary':
- # For cases when an additional preliminary VOEvent is sent
- # in order to add a preliminary skymap.
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Initial localization is now available (preliminary)')
- c.add_EventIVORN(ei)
-
- v.set_Citations(c)
-
- ############ output the event ############################
- xml = stringVOEvent(v)
- #schemaURL = "http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd")
- return xml, ivorn
diff --git a/gracedb/superevents/migrations/0004_populate_voevent_fields.py b/gracedb/superevents/migrations/0004_populate_voevent_fields.py
index a693ff8355d620ae3e55323cad02a360ecbc3907..2adc34c7f5af9f49d9f9aad4205c8b286f39eee1 100644
--- a/gracedb/superevents/migrations/0004_populate_voevent_fields.py
+++ b/gracedb/superevents/migrations/0004_populate_voevent_fields.py
@@ -1,7 +1,10 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.20 on 2019-06-10 16:58
from __future__ import unicode_literals
-from cStringIO import StringIO
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
from hashlib import sha1
import os
from lxml import etree
@@ -62,7 +65,7 @@ def get_datadir(event_or_superevent):
hash_input = str(event_or_superevent.id)
if event_or_superevent.__class__.__name__.lower() == 'superevent':
hash_input = 'superevent' + hash_input
- hdf = StringIO(sha1(hash_input).hexdigest())
+ hdf = StringIO(sha1(hash_input.encode()).hexdigest())
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
diff --git a/gracedb/superevents/migrations/0005_auto_20190919_1957.py b/gracedb/superevents/migrations/0005_auto_20190919_1957.py
new file mode 100644
index 0000000000000000000000000000000000000000..42d87f724f40c0cb677460ca9473f12f1ea3e849
--- /dev/null
+++ b/gracedb/superevents/migrations/0005_auto_20190919_1957.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-09-19 19:57
+# Note from Tanner:
+# This was auto-generated after moving to Python 3, with no changes to the
+# actual models. See the commit message for more details.
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('superevents', '0004_populate_voevent_fields'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='log',
+ name='filename',
+ field=models.CharField(blank=True, default='', max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='instrument',
+ field=models.CharField(blank=True, choices=[('H1', 'LHO'), ('L1', 'LLO'), ('V1', 'Virgo')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='signoff_type',
+ field=models.CharField(choices=[('OP', 'operator'), ('ADV', 'advocate')], max_length=3),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='status',
+ field=models.CharField(choices=[('OK', 'OKAY'), ('NO', 'NOT OKAY')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='superevent',
+ name='category',
+ field=models.CharField(choices=[('P', 'Production'), ('T', 'Test'), ('M', 'MDC')], default='P', max_length=1),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='filename',
+ field=models.CharField(blank=True, default='', editable=False, max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=200),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='voevent_type',
+ field=models.CharField(choices=[('PR', 'preliminary'), ('IN', 'initial'), ('UP', 'update'), ('RE', 'retraction')], max_length=2),
+ ),
+ ]
diff --git a/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
new file mode 100644
index 0000000000000000000000000000000000000000..695631b34a2f0330d6df1b5be2956ff1632e7338
--- /dev/null
+++ b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-11-19 17:39
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('superevents', '0005_auto_20190919_1957'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='superevent',
+ name='coinc_far',
+ field=models.FloatField(blank=True, null=True),
+ ),
+ migrations.AddField(
+ model_name='superevent',
+ name='em_type',
+ field=models.CharField(blank=True, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='combined_skymap_filename',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='delta_t',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(-1000), django.core.validators.MaxValueValidator(1000)]),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_gcn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_pipeline',
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_search',
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='raven_coinc',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='space_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='time_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=300),
+ ),
+ ]
diff --git a/gracedb/superevents/mixins.py b/gracedb/superevents/mixins.py
index 7d62075b373bfd136824fd9cba2e1a8f13e6bfa8..db645c46784e6447a658b1b7174f337297f40a09 100644
--- a/gracedb/superevents/mixins.py
+++ b/gracedb/superevents/mixins.py
@@ -178,18 +178,23 @@ class ExposeHideMixin(ContextMixin):
# Object is hidden and user can expose
can_modify_permissions = True
button_text = 'Make this superevent publicly visible'
+ confirmation_text = 'Warning: You are attempting to make this \
+ event publicly visible. Continue?'
action = 'expose'
elif (self.request.user.has_perm(self.hide_perm_name) and
self.object.is_exposed):
# Object is visible and user can hide
can_modify_permissions = True
button_text = 'Make this superevent internal-only'
+ confirmation_text = 'Warning: You are attempting to make this \
+ event internal only. Continue?'
action = 'hide'
# Update context
context['can_modify_permissions'] = can_modify_permissions
if can_modify_permissions:
context['permissions_form_button_text'] = button_text
+ context['confirmation_dialog_text'] = confirmation_text
context['permissions_action'] = action
return context
@@ -220,7 +225,7 @@ class ConfirmGwFormMixin(ContextMixin):
'is_test': 'superevents.confirm_gw_test_superevent',
'is_mdc': 'superevents.confirm_gw_mdc_superevent',
}
- for method_name, perm_name in method_perm_pairs.iteritems():
+ for method_name, perm_name in method_perm_pairs.items():
is_category = getattr(self.object, method_name)
if (is_category() and self.request.user.has_perm(perm_name)):
context['show_gw_status_form'] = True
diff --git a/gracedb/superevents/models.py b/gracedb/superevents/models.py
index 72548cff95f506b84b07e143a60827f251f20e24..ca3bdff0cc8f094b9fea4a1aaceec64036c7baa6 100644
--- a/gracedb/superevents/models.py
+++ b/gracedb/superevents/models.py
@@ -1,4 +1,7 @@
-from cStringIO import StringIO
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
import datetime
from hashlib import sha1
import logging
@@ -14,6 +17,8 @@ from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.db import models, IntegrityError
from django.urls import reverse
+from django.utils import six
+from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase
@@ -35,6 +40,7 @@ SUPEREVENT_DATE_START = datetime.datetime(1980, 1, 1, 0, 0, 0, 0, pytz.utc)
SUPEREVENT_DATE_END = datetime.datetime(2080, 1, 1, 0, 0, 0, 0, pytz.utc)
+@python_2_unicode_compatible
class Superevent(CleanSaveModel, AutoIncrementModel):
"""
Superevent date-based IDs:
@@ -112,6 +118,10 @@ class Superevent(CleanSaveModel, AutoIncrementModel):
# superevent, we are going to use a database field to track it.
is_exposed = models.BooleanField(default=False)
+ # New O3b fields for RAVEN:
+ coinc_far = models.FloatField(null=True, blank=True)
+ em_type = models.CharField(blank=True, null=True, max_length=100)
+
# Meta class --------------------------------------------------------------
class Meta:
ordering = ["-id"]
@@ -359,7 +369,7 @@ class Superevent(CleanSaveModel, AutoIncrementModel):
# object's primary key. We prepend 'superevent' so as to not
# have collisions with Event files
hash_input = 'superevent' + str(self.id)
- hdf = StringIO(sha1(hash_input).hexdigest())
+ hdf = StringIO(sha1(hash_input.encode()).hexdigest())
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
@@ -436,8 +446,8 @@ class Superevent(CleanSaveModel, AutoIncrementModel):
raise NotImplemented
#return reverse('')
- def __unicode__(self):
- return self.superevent_id
+ def __str__(self):
+ return six.text_type(self.superevent_id)
class DateIdError(Exception):
# To be raised when the superevent date ID is in a bad format; i.e.,
@@ -505,6 +515,7 @@ class LogUserObjectPermission(UserObjectPermissionBase):
content_object = models.ForeignKey(Log, on_delete=models.CASCADE)
+@python_2_unicode_compatible
class Labelling(m2mThroughBase):
"""
Model which provides the 'through' relationship between Superevents and
@@ -527,11 +538,16 @@ class Labelling(m2mThroughBase):
related_name='%(app_label)s_%(class)s_set',
on_delete=models.CASCADE)
- def __unicode__(self):
- return "{superevent_id} | {label}".format(superevent_id=
- self.superevent.superevent_id, label=self.label.name)
+ def __str__(self):
+ return six.text_type(
+ "{superevent_id} | {label}".format(
+ superevent_id=self.superevent.superevent_id,
+ label=self.label.name
+ )
+ )
+@python_2_unicode_compatible
class Signoff(CleanSaveModel, SignoffBase):
"""Class for superevent signoffs"""
superevent = models.ForeignKey(Superevent, null=False,
@@ -547,10 +563,14 @@ class Signoff(CleanSaveModel, SignoffBase):
('do_adv_signoff', 'Can interact with advocate signoffs'),
)
- def __unicode__(self):
- return "{superevent_id} | {instrument} | {status}".format(
- superevent_id=self.superevent.superevent_id,
- instrument=self.instrument, status=self.status)
+ def __str__(self):
+ return six.text_type(
+ "{superevent_id} | {instrument} | {status}".format(
+ superevent_id=self.superevent.superevent_id,
+ instrument=self.instrument,
+ status=self.status
+ )
+ )
class VOEvent(VOEventBase, AutoIncrementModel):
@@ -568,6 +588,7 @@ class VOEvent(VOEventBase, AutoIncrementModel):
super(Log, self).fileurl()
+@python_2_unicode_compatible
class EMObservation(CleanSaveModel, EMObservationBase, AutoIncrementModel):
"""EMObservation class for superevents"""
AUTO_FIELD = 'N'
@@ -578,10 +599,14 @@ class EMObservation(CleanSaveModel, EMObservationBase, AutoIncrementModel):
class Meta(EMObservationBase.Meta):
unique_together = (('superevent', 'N'),)
- def __unicode__(self):
- return "{superevent_id} | {group} | {N}".format(
- superevent_id=self.superevent.superevent_id,
- group=self.group.name, N=self.N)
+ def __str__(self):
+ return six.text_type(
+ "{superevent_id} | {group} | {N}".format(
+ superevent_id=self.superevent.superevent_id,
+ group=self.group.name,
+ N=self.N
+ )
+ )
def calculateCoveringRegion(self):
footprints = self.emfootprint_set.all()
diff --git a/gracedb/superevents/tests/test_access.py b/gracedb/superevents/tests/test_access.py
index ddbdca090a1b68385d098f180cc855aaa719065d..be35272af20b20d275acd4344cc2475e223b2edb 100644
--- a/gracedb/superevents/tests/test_access.py
+++ b/gracedb/superevents/tests/test_access.py
@@ -199,8 +199,8 @@ class TestSupereventFileListView(SupereventSetup, GraceDbTestBase):
super(TestSupereventFileListView, cls).setUpTestData()
# Create files for internal and exposed superevents
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_superevent, filename=cls.file1['filename'],
diff --git a/gracedb/superevents/utils.py b/gracedb/superevents/utils.py
index cec67f8203e27e6e6614734aaafec7a997faff0e..277a7003f1d9472b9e69ac07061b0c2e081bf12c 100644
--- a/gracedb/superevents/utils.py
+++ b/gracedb/superevents/utils.py
@@ -6,7 +6,6 @@ from django.http import Http404
from django.shortcuts import get_object_or_404
from django.contrib.auth.models import Group as DjangoGroup
-from .buildVOEvent import construct_voevent_file
from .models import Superevent, Log, Labelling, EMObservation, EMFootprint, \
VOEvent, Signoff
from .shortcuts import is_superevent
@@ -15,6 +14,7 @@ from alerts.issuers.superevents import SupereventAlertIssuer, \
SupereventLogAlertIssuer, SupereventLabelAlertIssuer, \
SupereventVOEventAlertIssuer, SupereventEMObservationAlertIssuer, \
SupereventSignoffAlertIssuer, SupereventPermissionsAlertIssuer
+from annotations.voevent_utils import construct_voevent_file
from core.permissions import expose_log_to_lvem, expose_log_to_public, \
hide_log_from_lvem, hide_log_from_public, assign_perms_to_obj, \
remove_perms_from_obj
@@ -125,18 +125,20 @@ def update_superevent(superevent, updater, add_log_message=True,
issue_alert=True, **kwargs):
"""
kwargs which are used as superevent parameters:
- t_start, t_0, t_end, preferred_event
+ t_start, t_0, t_end, preferred_event,
+ em_type, coinc_far
"""
# Extract "updatable" superevent params from kwargs
- param_names = ['t_start', 't_0', 't_end', 'preferred_event']
- new_params = {k: v for k,v in kwargs.iteritems() if k in param_names}
+ param_names = ['t_start', 't_0', 't_end', 'preferred_event',
+ 'em_type','coinc_far']
+ new_params = {k: v for k,v in kwargs.items() if k in param_names}
# Get old parameters
- old_params = {k: getattr(superevent, k) for k in new_params.keys()}
+ old_params = {k: getattr(superevent, k) for k in new_params}
# Update superevent object
- for k,v in new_params.iteritems():
+ for k,v in new_params.items():
setattr(superevent, k, v)
superevent.save()
@@ -145,7 +147,7 @@ def update_superevent(superevent, updater, add_log_message=True,
superevent_alert_kwargs = {}
if add_log_message:
updates = ["{name}: {old} -> {new}".format(name=k, old=old_params[k],
- new=new_params[k]) for k in new_params.keys()
+ new=new_params[k]) for k in new_params
if old_params[k] != new_params[k]]
update_comment = "Updated superevent parameters: {0}".format(
", ".join(updates))
@@ -153,7 +155,7 @@ def update_superevent(superevent, updater, add_log_message=True,
issue_alert=False)
# If preferred event changed, do a few things
- if new_params.has_key('preferred_event') and \
+ if 'preferred_event' in new_params and \
(old_params['preferred_event'] != new_params['preferred_event']):
# Write log for old preferred event
old_msg = ("Removed as preferred event for superevent: "
@@ -592,7 +594,8 @@ def create_voevent_for_superevent(superevent, issuer, voevent_type,
skymap_type=None, skymap_filename=None, internal=True, open_alert=False,
hardware_inj=False, CoincComment=False, ProbHasNS=None,
ProbHasRemnant=None, BNS=None, NSBH=None, BBH=None, Terrestrial=None,
- MassGap=None, add_log_message=True, issue_alert=True):
+ MassGap=None, add_log_message=True, issue_alert=True,
+ combined_skymap_filename=None, raven_coinc=False):
# Instantiate VOEvent object
voevent = VOEvent.objects.create(superevent=superevent, issuer=issuer,
@@ -601,7 +604,9 @@ def create_voevent_for_superevent(superevent, issuer, voevent_type,
open_alert=open_alert, hardware_inj=hardware_inj,
coinc_comment=CoincComment, prob_has_ns=ProbHasNS,
prob_has_remnant=ProbHasRemnant, prob_bns=BNS, prob_nsbh=NSBH,
- prob_bbh=BBH, prob_terrestrial=Terrestrial, prob_mass_gap=MassGap)
+ prob_bbh=BBH, prob_terrestrial=Terrestrial, prob_mass_gap=MassGap,
+ combined_skymap_filename=combined_skymap_filename,
+ raven_coinc=raven_coinc)
# Construct VOEvent file text
voevent_text, ivorn = construct_voevent_file(superevent, voevent)
@@ -709,7 +714,7 @@ def update_signoff(signoff, user, status, comment, add_log_message=True,
superevent = signoff.superevent
# Update signoff values
- for k,v in updated_attributes.iteritems():
+ for k,v in updated_attributes.items():
setattr(signoff, k, v)
signoff.save(update_fields=list(updated_attributes))
diff --git a/gracedb/superevents/views.py b/gracedb/superevents/views.py
index 0c8a45f647865e0d2cc7b6ba4300225df89687d2..7015c862346e7a601d0f7a0b4505df838b7d023d 100644
--- a/gracedb/superevents/views.py
+++ b/gracedb/superevents/views.py
@@ -233,7 +233,7 @@ class SupereventPublic(DisplayFarMixin, ListView):
("BBH", voe.prob_bbh),
("Terrestrial", voe.prob_terrestrial),
("MassGap", voe.prob_mass_gap)]
- pastro_values.sort(reverse=True, key=lambda (a,b): b)
+ pastro_values.sort(reverse=True, key=lambda p_a: p_a[1])
sourcelist = []
for key, value in pastro_values:
if value > 0.01:
diff --git a/gracedb/templates/gracedb/event_detail.html b/gracedb/templates/gracedb/event_detail.html
index aeb55bdab19eda8bf0e3fcdaa1c2a1ffba12985c..363a9a81b45b194d953c1fee016714fa60eee906 100644
--- a/gracedb/templates/gracedb/event_detail.html
+++ b/gracedb/templates/gracedb/event_detail.html
@@ -192,6 +192,7 @@
</th>
<th>FAR (Hz)</th>
<th>FAR (yr<sup>-1</sup>)</th>
+ <th> Latency (s) </th>
<th>Links</th>
<th>
<div id="basic_info_created_ts"></div>
@@ -219,6 +220,7 @@
{# NOTE: XXX Using event_far so it can be floored for external users. #}
<td>{% if far_is_upper_limit %} < {% endif %}{{ display_far|scientific }}</td>
<td>{% if far_is_upper_limit %} < {% endif %}{{ display_far_yr }}</td>
+ <td>{{ object.reportingLatency }} </td>
<td><a href="{{ object.weburl }}">Data</a></td>
<td>{{ object.created|multiTime:"created" }}</td>
{% if object.superevent %}
diff --git a/gracedb/templates/gracedb/performance.html b/gracedb/templates/gracedb/performance.html
index 2cf558bd611caf37579b94db5644795efeb934c8..fdaf479097a331b4ad7c72d0842b0f30307406b8 100644
--- a/gracedb/templates/gracedb/performance.html
+++ b/gracedb/templates/gracedb/performance.html
@@ -25,7 +25,7 @@
<h4>Details</h4>
<table style="border-spacing:0px;">
<tr> <th>Status</th> <th>Number of responses</th> </tr>
- {% for key, value in totals_by_status.create.iteritems %}
+ {% for key, value in totals_by_status.create.items %}
<tr> <td> {{ key }} </td> <td> {{ value }} </td> </tr>
{% endfor %}
</table>
@@ -45,7 +45,7 @@
<h4>Details</h4>
<table style="border-spacing:0px;">
<tr> <th>Status</th> <th>Number of responses</th> </tr>
- {% for key, value in totals_by_status.annotate.iteritems %}
+ {% for key, value in totals_by_status.annotate.items %}
<tr> <td> {{ key }} </td> <td> {{ value }} </td> </tr>
{% endfor %}
</table>
diff --git a/gracedb/templates/superevents/detail.html b/gracedb/templates/superevents/detail.html
index e529337d7d6129e327a7da698ab5ec291baca495..8da74e0d1fe7a2c28df66ff681fcdd5b931b402a 100644
--- a/gracedb/templates/superevents/detail.html
+++ b/gracedb/templates/superevents/detail.html
@@ -35,6 +35,12 @@
{% include "superevents/superevent_detail_script.js" %}
</script>
+<script>
+ function clickConfirm() {
+ return confirm(" {{confirmation_dialog_text}} ");
+}
+</script>
+
{% endblock %}
{% block content %}
@@ -68,9 +74,11 @@
{#-- XXX This next bit is super hacky. #}
{% if can_modify_permissions %}
<div class="content-area">
-<form action="{% url "legacy_apiweb:default:superevents:superevent-permission-modify" superevent.superevent_id %}" method="POST" id="permissions_form">
+<form action="{% url "legacy_apiweb:default:superevents:superevent-permission-modify" superevent.superevent_id %}"
+ method="POST"
+ id="permissions_form">
<input type="hidden" name="action" value="{{ permissions_action }}">
- <input type="submit" value="{{ permissions_form_button_text }}" class="permButtonClass" disabled>
+ <input type="submit" value="{{ permissions_form_button_text }}" class="permButtonClass" id="permissions_submit_link" onclick="return clickConfirm()">
</form>
</div>
{% endif %}
diff --git a/manage.py b/manage.py
index 1755bbdbd5c6ea7ce6ff9318dbccf652b19efb42..bd91b43b948923117b15ef970709f5d035e2a1a7 100755
--- a/manage.py
+++ b/manage.py
@@ -18,7 +18,10 @@ if __name__ == '__main__':
if (exists(VENV_PATH) and 'VIRTUAL_ENV' not in os.environ):
VIRTUALENV_ACTIVATOR = abspath(join(VENV_PATH, 'bin',
'activate_this.py'))
- execfile(VIRTUALENV_ACTIVATOR, dict(__file__=VIRTUALENV_ACTIVATOR))
+ exec(
+ open(VIRTUALENV_ACTIVATOR).read(),
+ {'__file__': VIRTUALENV_ACTIVATOR}
+ )
# Set DJANGO_SETTINGS_MODULE environment variable
os.environ.setdefault('DJANGO_SETTINGS_MODULE', DEFAULT_SETTINGS_MODULE)
diff --git a/requirements.txt b/requirements.txt
index f47d82f1216fab22c4f7248ed97e740529dcfe68..98331bc6a37a9a69ecd88b175b214ff54346a654 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,17 +12,21 @@ djangorestframework==3.9.0
djangorestframework-guardian==0.1.1
dnspython==1.15.0
flake8==3.5.0
+gunicorn[gthread]==19.9.0
html5lib==1.0.1
ipdb==0.10.2
ipython==5.5.0
lalsuite==6.53 # undocumented requirement for lscsoft-glue
ligo-lvalert==1.5.6
-ligo-lvalert-overseer==0.1.3
+# Temporary pin of lvalert-overseer for Python 3 update
+git+https://git.ligo.org/lscsoft/lvalert-overseer.git@python3-overseer#egg=ligo-lvalert-overseer
+#ligo-lvalert-overseer==0.1.3
lscsoft-glue==1.60.0
lxml==4.2.0
matplotlib==2.0.0
mock==2.0.0
mysqlclient==1.4.2
+numpy==1.17.2
packaging==17.1
phonenumbers==8.8.11
python-ldap==3.1.0
@@ -33,11 +37,14 @@ service_identity==17.0.0
simplejson==3.15.0
Sphinx==1.7.0
twilio==6.10.3
+voevent-parse==1.0.3
# Do NOT upgrade pyparsing beyond 2.3.0 (even to 2.3.1) without carefully
# testing and modifying the search query code. There were a number of
# problematic API changes in 2.3.1 and beyond that will need to be handled
# appropriately.
pyparsing==2.3.0
+pytest<5; python_version < '3'
+pytest==5.1.2; python_version >= '3'
pytest-cov==2.6.1
pytest-django==3.4.8
pytz==2018.9
@@ -46,7 +53,5 @@ pytz==2018.9
# https://github.com/etingof/pyasn1/issues/112
pyasn1==0.3.6
pyasn1-modules==0.1.5
-# Installing future for gunicorn gthreads:
-future==0.17.1
-futures==3.2.0
-gunicorn[gthread]==19.9.0
+# Installing futures for gunicorn gthreads (Python 2 only):
+futures==3.2.0; python_version < '3'