From 2a4c1e3b2b716c0c8d3e278fc5ca2bbead13380b Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 20 Sep 2019 12:07:01 -0500
Subject: [PATCH 001/106] Bugfix migration which assigns permissions to groups
---
gracedb/migrations/auth/0017_assign_permissions.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/gracedb/migrations/auth/0017_assign_permissions.py b/gracedb/migrations/auth/0017_assign_permissions.py
index 317d2727a..3aa26471c 100644
--- a/gracedb/migrations/auth/0017_assign_permissions.py
+++ b/gracedb/migrations/auth/0017_assign_permissions.py
@@ -8,7 +8,6 @@ from django.contrib.auth.management import create_permissions
# Group names
-LVC = settings.LVC_GROUP
EXECS = 'executives'
ACCESS = 'access_managers' # control external access
SUPEREVENTS = 'superevent_managers' # handle superevent creation/update
@@ -16,6 +15,9 @@ H1_CONTROL = 'h1_control_room'
L1_CONTROL = 'l1_control_room'
V1_CONTROL = 'v1_control_room'
EM_ADVOCATES = 'em_advocates'
+# Previously, this was teaken from settings.LVC_GROUP, but that value has
+# changed. So we have to hard-code it for past migrations
+LVC = 'Communities:LSCVirgoLIGOGroupMembers'
SUPEREVENT_PERMS = {
# Label permissions
--
GitLab
From 8461c20727bf2c8d5cb82f57b5a402500d15c88b Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Thu, 25 Apr 2019 20:48:33 +0100
Subject: [PATCH 002/106] gracedb.search: updated syntax for python3
---
gracedb/search/query/superevents.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/search/query/superevents.py b/gracedb/search/query/superevents.py
index c5277c591..b6c34aa11 100644
--- a/gracedb/search/query/superevents.py
+++ b/gracedb/search/query/superevents.py
@@ -70,7 +70,7 @@ superevent_preprefix = Optional(Or([CaselessLiteral(pref) for pref in
[Superevent.SUPEREVENT_CATEGORY_TEST, Superevent.SUPEREVENT_CATEGORY_MDC]])
).setResultsName('preprefix')
superevent_prefix = Or([CaselessLiteral(pref) for pref in
- Superevent.DEFAULT_ID_PREFIX, Superevent.GW_ID_PREFIX]).setResultsName('prefix')
+ (Superevent.DEFAULT_ID_PREFIX, Superevent.GW_ID_PREFIX)]).setResultsName('prefix')
superevent_date = Word(nums, exact=6).setResultsName('date')
superevent_suffix = Word(alphas).setResultsName('suffix')
superevent_expr = superevent_preprefix + superevent_prefix + \
--
GitLab
From 94a5d812c9980d4a60cea360c86b33103955e6d2 Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Thu, 25 Apr 2019 20:47:11 +0100
Subject: [PATCH 003/106] gracedb: updated dict usage for python3
---
gracedb/alerts/views.py | 6 ++----
gracedb/api/v1/events/views.py | 4 ++--
gracedb/api/v1/superevents/url_templates.py | 6 +++---
gracedb/events/tests/test_label_search.py | 2 +-
gracedb/ligoauth/middleware.py | 2 +-
gracedb/search/query/superevents.py | 10 +++++-----
gracedb/superevents/mixins.py | 2 +-
gracedb/superevents/utils.py | 6 +++---
gracedb/templates/gracedb/performance.html | 4 ++--
9 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/gracedb/alerts/views.py b/gracedb/alerts/views.py
index 0ea11cce0..df94588a2 100644
--- a/gracedb/alerts/views.py
+++ b/gracedb/alerts/views.py
@@ -65,8 +65,7 @@ class CreateNotificationView(MultipleFormView):
return kw
def form_valid(self, form):
- if form.cleaned_data.has_key('key_field'):
- form.cleaned_data.pop('key_field')
+ form.cleaned_data.pop('key_field', None)
# Add user (from request) and category (stored on form class) to
# the form instance, then save
@@ -169,8 +168,7 @@ class CreateContactView(MultipleFormView):
def form_valid(self, form):
# Remove key_field, add user, and save form
- if form.cleaned_data.has_key('key_field'):
- form.cleaned_data.pop('key_field')
+ form.cleaned_data.pop('key_field', None)
form.instance.user = self.request.user
form.save()
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index bfbd37e8c..c3a4641c3 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -489,7 +489,7 @@ class EventList(InheritPermissionsAPIView):
# TP (21 Nov 2017): Hack to allow basic event submission without
# labels to function with versions of gracedb-client before 1.26.
# Should be removed eventually.
- if (request.data.has_key('labels') and request.data['labels'] == ''):
+ if (request.data.get('labels') == ''):
request.data.pop('labels', None)
form = CreateEventForm(request.data, request.data)
@@ -746,7 +746,7 @@ class EventNeighbors(InheritPermissionsAPIView):
# and TSV renderers.
@event_and_auth_required
def get(self, request, event):
- if request.query_params.has_key('neighborhood'):
+ if 'neighborhood' in request.query_params:
delta = request.query_params['neighborhood']
try:
if delta.find(',') < 0:
diff --git a/gracedb/api/v1/superevents/url_templates.py b/gracedb/api/v1/superevents/url_templates.py
index 49433e0bb..9085b757a 100644
--- a/gracedb/api/v1/superevents/url_templates.py
+++ b/gracedb/api/v1/superevents/url_templates.py
@@ -60,12 +60,12 @@ def construct_url_templates(request=None):
# keys are like '{view_name}-template'
# values are URLs with placeholder parameters
templates = {view_name + '-template': sr(view_name, args=args)
- for view_name, args in views.iteritems()}
+ for view_name, args in views.items()}
# Replace URL placeholder parameters with string formatting placeholders
# Ex: replace 'G1234' with '{graceid}'
- for k,v in templates.iteritems():
- for pattern,placeholder in PH.iteritems():
+ for k,v in templates.items():
+ for pattern,placeholder in PH.items():
if placeholder in v:
v = v.replace(placeholder, "{{{0}}}".format(pattern))
templates[k] = v
diff --git a/gracedb/events/tests/test_label_search.py b/gracedb/events/tests/test_label_search.py
index 3219c5653..14063ae50 100644
--- a/gracedb/events/tests/test_label_search.py
+++ b/gracedb/events/tests/test_label_search.py
@@ -53,7 +53,7 @@ class LabelSearchTestCase(TestCase):
Labelling.objects.create(event=e, label=label, creator=submitter)
def test_all_queries(self):
- for key, d in QUERY_CASES.iteritems():
+ for key, d in QUERY_CASES.items():
print "Checking %s ... " % key
# Explicitly search for test events
query = 'Test ' + d['query']
diff --git a/gracedb/ligoauth/middleware.py b/gracedb/ligoauth/middleware.py
index a4b879fbf..e08bd5f64 100644
--- a/gracedb/ligoauth/middleware.py
+++ b/gracedb/ligoauth/middleware.py
@@ -125,7 +125,7 @@ class ControlRoomMiddleware(object):
user_ip = self.get_client_ip(request)
# Add user to control room group(s)
- for ifo, ip in settings.CONTROL_ROOM_IPS.iteritems():
+ for ifo, ip in settings.CONTROL_ROOM_IPS.items():
if (ip == user_ip):
control_room_group = DjangoGroup.objects.get(name=
(ifo.lower() + self.control_room_group_suffix))
diff --git a/gracedb/search/query/superevents.py b/gracedb/search/query/superevents.py
index b6c34aa11..fae5156e7 100644
--- a/gracedb/search/query/superevents.py
+++ b/gracedb/search/query/superevents.py
@@ -227,21 +227,21 @@ parameter_dicts = {
# Compile a list of expressions to try to match
expr_list = []
-for k,p in parameter_dicts.iteritems():
+for k,p in parameter_dicts.items():
# Define val and set name
val = p['value']
val.setName(k)
# Add range with format: parameter .. parameter
- if p.has_key('doRange') and p['doRange']:
+ if p.get('doRange'):
range_val = val + Suppress("..") + val
val ^= range_val
# Add keyword. Format is keyword: value
- if p.has_key('keyword'):
+ if 'keyword' in p:
if isinstance(p['keyword'], list):
- if p.has_key('keywordOptional') and p['keywordOptional']:
+ if p.get('keywordOptional'):
keyword_list = [Optional(Suppress(Keyword(k + ":"))) for k in
p['keyword']]
else:
@@ -249,7 +249,7 @@ for k,p in parameter_dicts.iteritems():
keyword = reduce(lambda x,y: x^y, keyword_list)
else:
keyword = Suppress(Keyword(p['keyword'] + ":"))
- if p.has_key('keywordOptional') and p['keywordOptional']:
+ if p.get('keywordOptional'):
keyword = Optional(keyword)
# Combine keyword and value into a single expression
diff --git a/gracedb/superevents/mixins.py b/gracedb/superevents/mixins.py
index 7d62075b3..a7cf26adb 100644
--- a/gracedb/superevents/mixins.py
+++ b/gracedb/superevents/mixins.py
@@ -220,7 +220,7 @@ class ConfirmGwFormMixin(ContextMixin):
'is_test': 'superevents.confirm_gw_test_superevent',
'is_mdc': 'superevents.confirm_gw_mdc_superevent',
}
- for method_name, perm_name in method_perm_pairs.iteritems():
+ for method_name, perm_name in method_perm_pairs.items():
is_category = getattr(self.object, method_name)
if (is_category() and self.request.user.has_perm(perm_name)):
context['show_gw_status_form'] = True
diff --git a/gracedb/superevents/utils.py b/gracedb/superevents/utils.py
index cec67f820..6422150ce 100644
--- a/gracedb/superevents/utils.py
+++ b/gracedb/superevents/utils.py
@@ -130,13 +130,13 @@ def update_superevent(superevent, updater, add_log_message=True,
# Extract "updatable" superevent params from kwargs
param_names = ['t_start', 't_0', 't_end', 'preferred_event']
- new_params = {k: v for k,v in kwargs.iteritems() if k in param_names}
+ new_params = {k: v for k,v in kwargs.items() if k in param_names}
# Get old parameters
old_params = {k: getattr(superevent, k) for k in new_params.keys()}
# Update superevent object
- for k,v in new_params.iteritems():
+ for k,v in new_params.items():
setattr(superevent, k, v)
superevent.save()
@@ -709,7 +709,7 @@ def update_signoff(signoff, user, status, comment, add_log_message=True,
superevent = signoff.superevent
# Update signoff values
- for k,v in updated_attributes.iteritems():
+ for k,v in updated_attributes.items():
setattr(signoff, k, v)
signoff.save(update_fields=list(updated_attributes))
diff --git a/gracedb/templates/gracedb/performance.html b/gracedb/templates/gracedb/performance.html
index 2cf558bd6..fdaf47909 100644
--- a/gracedb/templates/gracedb/performance.html
+++ b/gracedb/templates/gracedb/performance.html
@@ -25,7 +25,7 @@
<h4>Details</h4>
<table style="border-spacing:0px;">
<tr> <th>Status</th> <th>Number of responses</th> </tr>
- {% for key, value in totals_by_status.create.iteritems %}
+ {% for key, value in totals_by_status.create.items %}
<tr> <td> {{ key }} </td> <td> {{ value }} </td> </tr>
{% endfor %}
</table>
@@ -45,7 +45,7 @@
<h4>Details</h4>
<table style="border-spacing:0px;">
<tr> <th>Status</th> <th>Number of responses</th> </tr>
- {% for key, value in totals_by_status.annotate.iteritems %}
+ {% for key, value in totals_by_status.annotate.items %}
<tr> <td> {{ key }} </td> <td> {{ value }} </td> </tr>
{% endfor %}
</table>
--
GitLab
From 8e86f3ad823702d79d8ab13b9b038f67130baf8c Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Wed, 24 Apr 2019 18:03:34 +0100
Subject: [PATCH 004/106] updated exception syntax for python3
---
docs/user_docs/source/auth.rst | 2 +-
docs/user_docs/source/lvem.rst | 2 +-
docs/user_docs/source/rest.rst | 2 +-
gracedb/api/v1/events/views.py | 37 ++++++++++++++++++++--------------
gracedb/api/v1/main/views.py | 2 +-
gracedb/core/vfile.py | 4 ++--
gracedb/events/serialize.py | 8 ++++----
gracedb/events/translator.py | 14 ++++++-------
gracedb/events/view_logic.py | 10 ++++-----
gracedb/events/views.py | 12 +++++------
gracedb/search/fields.py | 6 +++---
11 files changed, 53 insertions(+), 46 deletions(-)
diff --git a/docs/user_docs/source/auth.rst b/docs/user_docs/source/auth.rst
index 216f05f6f..c0d9ad385 100644
--- a/docs/user_docs/source/auth.rst
+++ b/docs/user_docs/source/auth.rst
@@ -75,7 +75,7 @@ Shibbolized client as follows::
try:
r = client.ping()
- except HTTPError, e:
+ except HTTPError as e:
print e.message
print "Response code: %d" % r.status
diff --git a/docs/user_docs/source/lvem.rst b/docs/user_docs/source/lvem.rst
index 488f6e7c6..344f580cb 100644
--- a/docs/user_docs/source/lvem.rst
+++ b/docs/user_docs/source/lvem.rst
@@ -84,7 +84,7 @@ For example, you can use the GraceDB Python client::
try:
r = client.ping()
- except HTTPError, e:
+ except HTTPError as e:
print e.message
print "Response code: %d" % r.status
diff --git a/docs/user_docs/source/rest.rst b/docs/user_docs/source/rest.rst
index e5a579087..149588fe3 100644
--- a/docs/user_docs/source/rest.rst
+++ b/docs/user_docs/source/rest.rst
@@ -78,7 +78,7 @@ If you have reason to believe that your request may be throttled, you can wrap i
try:
r = gracedb.writeLog(graceid, "Hello, this is a log message.")
success = True
- except HTTPError, e:
+ except HTTPError as e:
try:
rdict = json.loads(e.message)
if 'retry-after' in rdict.keys():
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index c3a4641c3..dd60160dd 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -381,7 +381,7 @@ class EventList(InheritPermissionsAPIView):
except ParseException:
d = {'error': 'Invalid query' }
return Response(d,status=status.HTTP_400_BAD_REQUEST)
- except Exception, e:
+ except Exception as e:
d = {'error': str(e) }
return Response(d,status=status.HTTP_500_INTERNAL_SERVER_ERROR)
form = SimpleSearchForm(request.GET)
@@ -447,7 +447,7 @@ class EventList(InheritPermissionsAPIView):
# the function we are presently inside.
response = self.finalize_response(request, response, *args, **kwargs)
response.render()
- except Exception, e:
+ except Exception as e:
try:
status_code = e.status_code
except:
@@ -567,7 +567,7 @@ class EventDetail(InheritPermissionsAPIView):
# the function we are presently inside.
response = self.finalize_response(request, response)
response.render()
- except Exception, e:
+ except Exception as e:
try:
status_code = e.status_code
except:
@@ -585,7 +585,7 @@ class EventDetail(InheritPermissionsAPIView):
if request.user != event.submitter:
msg = "You (%s) Them (%s)" % (request.user, event.submitter)
return HttpResponseForbidden("You did not create this event. %s" %msg)
- except Exception, e:
+ except Exception as e:
return Response(str(e))
# Compile far and nscand for alerts
@@ -901,7 +901,7 @@ class EventLogList(InheritPermissionsAPIView):
fdest.close()
# Ascertain the version assigned to this particular file.
file_version = fdest.version
- except Exception, e:
+ except Exception as e:
# XXX This needs some thought.
response = Response(str(e), status=status.HTTP_400_BAD_REQUEST)
@@ -1013,12 +1013,12 @@ class EMBBEventLogList(InheritPermissionsAPIView):
try:
# Alert is issued in this code
eel = create_eel(request.data, event, request.user)
- except ValueError, e:
+ except ValueError as e:
return Response("%s" % str(e), status=status.HTTP_400_BAD_REQUEST)
- except IntegrityError, e:
+ except IntegrityError as e:
return Response("Failed to save EMBB entry: %s" % str(e),
status=status.HTTP_503_SERVICE_UNAVAILABLE)
- except Exception, e:
+ except Exception as e:
return Response("Problem creating EEL: %s" % str(e),
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -1097,12 +1097,12 @@ class EMObservationList(InheritPermissionsAPIView):
try:
# Create EMObservation - alert is issued inside this code
emo = create_emobservation(request, event)
- except ValueError, e:
+ except ValueError as e:
return Response("%s" % str(e), status=status.HTTP_400_BAD_REQUEST)
- except IntegrityError, e:
+ except IntegrityError as e:
return Response("Failed to save EMBB observation record: %s" % str(e),
status=status.HTTP_503_SERVICE_UNAVAILABLE)
- except Exception, e:
+ except Exception as e:
return Response("Problem creating EMBB Observation: %s" % str(e),
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -1462,7 +1462,7 @@ class GroupEventPermissionDetail(InheritPermissionsAPIView):
permission=underlying_permission)
underlying_event.refresh_perms()
- except Exception, e:
+ except Exception as e:
# We're gonna blame the user here.
return Response("Problem creating permission: %" % str(e),
status=status.HTTP_400_BAD_REQUEST)
@@ -1528,7 +1528,7 @@ class GroupEventPermissionDetail(InheritPermissionsAPIView):
except GroupObjectPermission.DoesNotExist:
return Response("GroupObjectPermission not found.",
status=status.HTTP_404_NOT_FOUND)
- except Exception, e:
+ except Exception as e:
return Response("Problem deleting permission: %s" % str(e),
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -1628,7 +1628,7 @@ class Files(InheritPermissionsAPIView):
rv['permalink'] = api_reverse(
"events:files", args=[event.graceid, shortname], request=request)
response = Response(rv, status=status.HTTP_201_CREATED)
- except Exception, e:
+ except Exception as e:
# XXX This needs some thought.
response = Response(str(e), status=status.HTTP_400_BAD_REQUEST)
# XXX Uhm, we don't to try creating a log message for this, right?
@@ -1740,7 +1740,14 @@ class VOEventList(InheritPermissionsAPIView):
# Now, you need to actually build the VOEvent.
try:
voevent_text, ivorn = buildVOEvent(event, voevent, request=request)
- except VOEventBuilderException, e:
+ except VOEventBuilderException as e:
+ voevent_text, ivorn = buildVOEvent(event, voevent.N, voevent_type, request,
+ skymap_filename = skymap_filename, skymap_type = skymap_type,
+ internal = internal, open_alert=open_alert,
+ hardware_inj=hardware_inj, CoincComment=CoincComment,
+ ProbHasNS=ProbHasNS, ProbHasRemnant=ProbHasRemnant, BNS=BNS,
+ NSBH=NSBH, BBH=BBH, Terrestrial=Terrestrial, MassGap=MassGap)
+
msg = "Problem building VOEvent: %s" % str(e)
return Response({'error': msg}, status = status.HTTP_400_BAD_REQUEST)
diff --git a/gracedb/api/v1/main/views.py b/gracedb/api/v1/main/views.py
index 2cef09eb6..8387bc657 100644
--- a/gracedb/api/v1/main/views.py
+++ b/gracedb/api/v1/main/views.py
@@ -185,7 +185,7 @@ class PerformanceInfo(InheritDefaultPermissionsMixin, APIView):
try:
performance_info = get_performance_info()
- except Exception, e:
+ except Exception as e:
return Response(str(e),
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
diff --git a/gracedb/core/vfile.py b/gracedb/core/vfile.py
index 4ff97d3cd..aaa1f9d52 100644
--- a/gracedb/core/vfile.py
+++ b/gracedb/core/vfile.py
@@ -128,7 +128,7 @@ class VersionedFile(file):
# lose fd we used to ensure file creation.
os.close(fd)
break
- except OSError, e:
+ except OSError as e:
if e.errno != errno.EEXIST:
raise
version += 1
@@ -183,7 +183,7 @@ class VersionedFile(file):
try:
# XXX Another race condition. File will not exist for a very brief time.
os.unlink(self.fullname)
- except OSError, e:
+ except OSError as e:
# Do not care if file does not exist, otherwise raise exception.
if e.errno != errno.ENOENT:
raise
diff --git a/gracedb/events/serialize.py b/gracedb/events/serialize.py
index e36d9a126..56831a079 100644
--- a/gracedb/events/serialize.py
+++ b/gracedb/events/serialize.py
@@ -132,7 +132,7 @@ def populate_omega_tables(datafile, set_keys = Omega_set_keys):
for line in f.readlines():
if not line.strip(): continue # ignore blank lines
elif '#' in line.strip()[0]: continue # ignore comments
- elif '=' not in line: raise ValueError, "Improperly formatted line"
+ elif '=' not in line: raise ValueError("Improperly formatted line")
else:
omega_list.extend([dat.strip() for dat in line.split('=',1)])
f.close()
@@ -140,7 +140,7 @@ def populate_omega_tables(datafile, set_keys = Omega_set_keys):
# basic error checking
# for key in omega_data:
# if not (key in omega_vars):
-# raise ValueError, "Unknown variable"
+# raise ValueError("Unknown variable")
#create the content for the event.log file
log_data = '\nLog File created '\
@@ -255,7 +255,7 @@ def populate_coinc_tables(xmldoc, coinc_event_id, event_id_dict,\
elif 'burst' in CoincDef.search:
row.nevents = 1
else:
- raise ValueError, "Unrecognize CoincDef.search"
+ raise ValueError("Unrecognize CoincDef.search")
row.likelihood = likelihood
coinc_table.append(row)
@@ -270,7 +270,7 @@ def populate_coinc_tables(xmldoc, coinc_event_id, event_id_dict,\
elif 'burst' in CoincDef.search:
row.table_name = lsctables.MultiBurstTable.tableName.split(':')[0]
else:
- raise ValueError, "Unrecognize CoincDef.search"
+ raise ValueError("Unrecognize CoincDef.search")
if event_id_dict:
row.event_id = event_id_dict[ifo]
coinc_map_table.append(row)
diff --git a/gracedb/events/translator.py b/gracedb/events/translator.py
index 977a6420b..be6ccc4da 100644
--- a/gracedb/events/translator.py
+++ b/gracedb/events/translator.py
@@ -76,7 +76,7 @@ def handle_uploaded_data(event, datafilename,
try:
xmldoc = load_filename(datafilename, contenthandler = LIGOLWContentHandler)
- except Exception, e:
+ except Exception as e:
message = "Could not read data (%s)" % str(e)
EventLog(event=event, issuer=event.submitter, comment=message).save()
return
@@ -84,7 +84,7 @@ def handle_uploaded_data(event, datafilename,
# Try reading the CoincInspiralTable
try:
coinc_table = CoincInspiralTable.get_table(xmldoc)[0]
- except Exception, e:
+ except Exception as e:
warnings += "Could not extract coinc inspiral table."
return temp_data_loc, warnings
@@ -124,7 +124,7 @@ def handle_uploaded_data(event, datafilename,
log_data.append("FAR: %0.3e" % far)
else:
log_data.append("FAR: ---")
- except Exception, e:
+ except Exception as e:
log_comment = "Problem Creating Log File"
log_data = ["Cannot create log file", "error was:", str(e)]
@@ -157,7 +157,7 @@ def handle_uploaded_data(event, datafilename,
# Try to get the coinc_event_table
try:
coinc_event_table = CoincTable.get_table(xmldoc)[0]
- except Exception, e:
+ except Exception as e:
warnings += "Could not extract coinc event table."
return temp_data_loc, warnings
event.nevents = coinc_event_table.nevents
@@ -220,7 +220,7 @@ def handle_uploaded_data(event, datafilename,
# log_data.append("Component 1 Spin: (%f, %f, %f)" % spin1)
# log_data.append("Component 2 Spin: (%f, %f, %f)" % spin2)
# log_data.append("Geocentric End Time: %d.%09d" % end_time)
-# except Exception, e:
+# except Exception as e:
# log_comment = "Problem Creating Log File"
# log_data = ["Cannot create log file", "error was:", str(e)]
# log_data = "\n".join(log_data)
@@ -281,7 +281,7 @@ def handle_uploaded_data(event, datafilename,
warnings = []
try:
coinc_table = CoincInspiralTable.get_table(xmldoc)[0]
- except Exception, e:
+ except Exception as e:
warnings += "Could not extract coinc inspiral table."
return temp_data_loc, warnings
@@ -345,7 +345,7 @@ def handle_uploaded_data(event, datafilename,
try:
#event.gpstime = getGpsFromVOEvent(datafilename)
populateGrbEventFromVOEventFile(datafilename, event)
- except Exception, e:
+ except Exception as e:
error = "Problem parsing VOEvent: %s" % e.__repr__()
event.save()
if error is not None:
diff --git a/gracedb/events/view_logic.py b/gracedb/events/view_logic.py
index 8987f65a5..2bfe74d73 100644
--- a/gracedb/events/view_logic.py
+++ b/gracedb/events/view_logic.py
@@ -128,7 +128,7 @@ def _createEventFromForm(request, form):
# XXX This reverse will give the web-interface URL, not the REST URL.
# This could be a problem if anybody ever tries to use it.
EventAlertIssuer(event, alert_type='new').issue_alerts()
- except Exception, e:
+ except Exception as e:
message = "Problem issuing an alert (%s)" % e
logger.warning(message)
warnings += [message]
@@ -145,12 +145,12 @@ def _createEventFromForm(request, form):
create_label(event, request, label.name,
can_add_protected=False)
- except Exception, e:
+ except Exception as e:
message = "Problem scanning data. No alert issued (%s)" % e
logger.warning(message)
warnings += [message]
#return HttpResponseRedirect(reverse(view, args=[event.graceid]))
- except Exception, e:
+ except Exception as e:
# something went wrong.
# XXX We need to make sure we clean up EVERYTHING.
# We don't. Wiki page and data directories remain.
@@ -491,7 +491,7 @@ def create_emobservation(request, event):
startTimeList = d.get('start_time_list')
durationList = d.get('duration_list')
- except Exception, e:
+ except Exception as e:
raise ValueError('Lacking input: %s' % str(e))
# Handle case where comma-separated strings are submitted rather than lists
@@ -576,7 +576,7 @@ def create_emobservation(request, event):
group=emo.group)
EventEMObservationAlertIssuer(emo, alert_type='emobservation') \
.issue_alerts()
- except Exception, e:
+ except Exception as e:
# XXX Should probably send back warnings, as in the other cases.
logger.error('error sending alert for emobservation: {0}'.format(e))
diff --git a/gracedb/events/views.py b/gracedb/events/views.py
index e05949917..7a981ddef 100644
--- a/gracedb/events/views.py
+++ b/gracedb/events/views.py
@@ -137,7 +137,7 @@ def index(request):
if request.user and not is_external(request.user) and settings.SHOW_RECENT_EVENTS_ON_HOME:
try:
recent_events = get_recent_events_string(request)
- except Exception, e:
+ except Exception as e:
pass
context['recent_events'] = recent_events
@@ -221,7 +221,7 @@ def logentry(request, event, num=None):
fdest.close()
# Ascertain the version assigned to this particular file.
file_version = fdest.version
- except Exception, e:
+ except Exception as e:
return HttpResponseServerError(str(e))
elog.filename = filename
@@ -294,7 +294,7 @@ def logentry(request, event, num=None):
return HttpResponseForbidden("Forbidden")
try:
elog = event.eventlog_set.filter(N=num)[0]
- except Exception, e:
+ except Exception as e:
raise Http404
# Check authorization for this log message
@@ -589,7 +589,7 @@ def performance(request):
try:
context = get_performance_info()
- except Exception, e:
+ except Exception as e:
return HttpResponseServerError(str(e))
return render(request, 'gracedb/performance.html', context=context)
@@ -745,9 +745,9 @@ def emobservation_entry(request, event, num=None):
try:
# Alert is issued in this function
create_emobservation(request, event)
- except ValueError, e:
+ except ValueError as e:
return HttpResponseBadRequest(str(e))
- except Exception, e:
+ except Exception as e:
return HttpResponseServerError(str(e))
return HttpResponseRedirect(reverse('view', args=[event.graceid]))
diff --git a/gracedb/search/fields.py b/gracedb/search/fields.py
index f456dc3f4..eeb1f197b 100644
--- a/gracedb/search/fields.py
+++ b/gracedb/search/fields.py
@@ -29,14 +29,14 @@ class GraceQueryField(forms.CharField):
try:
qs = self.do_filtering(queryString)
return qs.distinct()
- except ParseException, e:
+ except ParseException as e:
err = "Error: " + escape(e.pstr[:e.loc]) + errorMarker + escape(e.pstr[e.loc:])
raise forms.ValidationError(mark_safe(err))
- except FieldError, e:
+ except FieldError as e:
# XXX error message can be more polished than this
err = "Error: " + str(e)
raise forms.ValidationError(mark_safe(err))
- except Exception, e:
+ except Exception as e:
# What could this be and how can we handle it better? XXX
raise forms.ValidationError(str(e)+str(type(e)))
--
GitLab
From 6e3d8492b5e29fef11741df308edb16e1250d72a Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Wed, 24 Apr 2019 18:10:18 +0100
Subject: [PATCH 005/106] updated urllib imports for python3
---
docker/check_shibboleth_status | 12 +++++++++---
docs/admin_docs/source/public_gracedb.rst | 2 +-
gracedb/api/v1/events/views.py | 11 +++++++----
gracedb/api/v1/paginators.py | 7 +++++--
gracedb/events/tests/test_perms.py | 5 ++++-
gracedb/search/tests/test_access.py | 13 ++++++++-----
6 files changed, 34 insertions(+), 16 deletions(-)
diff --git a/docker/check_shibboleth_status b/docker/check_shibboleth_status
index b0ba42bde..f515a2d91 100644
--- a/docker/check_shibboleth_status
+++ b/docker/check_shibboleth_status
@@ -8,8 +8,14 @@ Run ./check_shibboleth_status -h for help.
'''
# Imports
-import argparse, urllib2, sys
+import argparse
+import sys
import xml.etree.ElementTree as ET
+try:
+ from urllib.request import urlopen
+ from urllib.error import URLError
+except ImportError: # python < 3
+ from urllib2 import (urlopen, URLError)
# Parameters - may need to be modified in the future
# if Shibboleth status pages change or new metadata
@@ -48,8 +54,8 @@ metadata_feeds = args.feeds.split(",")
# Get XML data from URL.
host_url = host + "/" + urlpath
try:
- response = urllib2.urlopen(host_url, timeout=timeout)
-except urllib2.URLError:
+ response = urlopen(host_url, timeout=timeout)
+except URLError:
print "Error opening Shibboleth status page (" + host_url + ")."
sys.exit(2)
except:
diff --git a/docs/admin_docs/source/public_gracedb.rst b/docs/admin_docs/source/public_gracedb.rst
index 5b563dec9..9fdcd4bff 100644
--- a/docs/admin_docs/source/public_gracedb.rst
+++ b/docs/admin_docs/source/public_gracedb.rst
@@ -114,7 +114,7 @@ example, here is how to grant ``view`` permissions to ``public``::
group_name = 'public'
perm_shortname = 'view'
- url = g.service_url + urllib.quote('events/%s/%s/%s' % (graceid, group_name, perm_codename))
+ url = g.service_url + urllib.parse.quote('events/%s/%s/%s' % (graceid, group_name, perm_codename))
r = g.put(url)
Templates
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index dd60160dd..2a347decc 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -5,7 +5,10 @@ import logging
import os
import shutil
import StringIO
-import urllib
+try:
+ from urllib.parse import urlencode
+except ImportError: # python < 3
+ from urllib import urlencode
from django.conf import settings
from django.contrib.auth.models import User, Permission, Group as DjangoGroup
@@ -416,14 +419,14 @@ class EventList(InheritPermissionsAPIView):
d = { 'start' : 0, "count": count, "sort": sort }
if query: d['query'] = query
- links['first'] = baseuri + "?" + urllib.urlencode(d)
+ links['first'] = baseuri + "?" + urlencode(d)
d['start'] = last
- links['last'] = baseuri + "?" + urllib.urlencode(d)
+ links['last'] = baseuri + "?" + urlencode(d)
if start != last:
d['start'] = start+count
- links['next'] = baseuri + "?" + urllib.urlencode(d)
+ links['next'] = baseuri + "?" + urlencode(d)
rv['numRows'] = events.count()
response = Response(rv)
diff --git a/gracedb/api/v1/paginators.py b/gracedb/api/v1/paginators.py
index 6fa5e6777..f03eaa9c7 100644
--- a/gracedb/api/v1/paginators.py
+++ b/gracedb/api/v1/paginators.py
@@ -1,6 +1,9 @@
from collections import OrderedDict
import logging
-import urllib
+try:
+ from urllib.parse import urlencode
+except ImportError: # python < 3
+ from urllib import urlencode
from rest_framework import pagination
from rest_framework.response import Response
@@ -80,7 +83,7 @@ class CustomSupereventPagination(pagination.LimitOffsetPagination):
'start': last,
self.limit_query_param: self.limit,
}
- last_uri = base_uri + '?' + urllib.urlencode(param_dict)
+ last_uri = base_uri + '?' + urlencode(param_dict)
output = OrderedDict([
('numRows', numRows),
diff --git a/gracedb/events/tests/test_perms.py b/gracedb/events/tests/test_perms.py
index a2b0148f7..36ab3923e 100644
--- a/gracedb/events/tests/test_perms.py
+++ b/gracedb/events/tests/test_perms.py
@@ -16,7 +16,10 @@ from events.permission_utils import assign_default_event_perms
import json
import os
import shutil
-from urllib import urlencode
+try:
+ from urllib.parse import urlencode
+except ImportError: # python < 3
+ from urllib import urlencode
#------------------------------------------------------------------------------
#------------------------------------------------------------------------------
diff --git a/gracedb/search/tests/test_access.py b/gracedb/search/tests/test_access.py
index 77f8317bf..45592ae8b 100644
--- a/gracedb/search/tests/test_access.py
+++ b/gracedb/search/tests/test_access.py
@@ -1,4 +1,7 @@
-import urllib
+try:
+ from urllib.parse import urlencode
+except ImportError: # python < 3
+ from urllib import urlencode
from django.conf import settings
from django.contrib.auth.models import Group as DjangoGroup, Permission
@@ -34,7 +37,7 @@ class TestEventSearch(EventCreateMixin, GraceDbTestBase, SearchTestingBase):
'query_type': MainSearchForm.QUERY_TYPE_EVENT,
'results_format': MainSearchForm.FORMAT_CHOICE_STANDARD,
}
- cls.full_url = cls.url + '?' + urllib.urlencode(cls.query_dict)
+ cls.full_url = cls.url + '?' + urlencode(cls.query_dict)
@classmethod
def setUpTestData(cls):
@@ -104,7 +107,7 @@ class TestSupereventSearch(SupereventSetup, GraceDbTestBase,
'query_type': MainSearchForm.QUERY_TYPE_SUPEREVENT,
'results_format': MainSearchForm.FORMAT_CHOICE_STANDARD,
}
- cls.full_url = cls.url + '?' + urllib.urlencode(cls.query_dict)
+ cls.full_url = cls.url + '?' + urlencode(cls.query_dict)
def test_internal_user_search(self):
"""Internal user sees all superevents in search results"""
@@ -150,7 +153,7 @@ class TestEventLatest(EventCreateMixin, GraceDbTestBase):
'results_format': MainSearchForm.FORMAT_CHOICE_STANDARD,
}
cls.url = reverse('latest')
- cls.full_url = cls.url + '?' + urllib.urlencode(cls.query_dict)
+ cls.full_url = cls.url + '?' + urlencode(cls.query_dict)
@classmethod
def setUpTestData(cls):
@@ -224,7 +227,7 @@ class TestSupereventLatest(SupereventSetup, GraceDbTestBase):
'results_format': MainSearchForm.FORMAT_CHOICE_STANDARD,
}
cls.url = reverse('latest')
- cls.full_url = cls.url + '?' + urllib.urlencode(cls.query_dict)
+ cls.full_url = cls.url + '?' + urlencode(cls.query_dict)
def test_internal_user_latest(self):
"""Internal user sees all superevents on latest page"""
--
GitLab
From 7841312f95e6503c4c823cb9e7e988cab4b105c2 Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Thu, 25 Apr 2019 20:46:19 +0100
Subject: [PATCH 006/106] gracedb: updated mock imports for python3
---
gracedb/alerts/tests/test_access.py | 5 ++++-
gracedb/alerts/tests/test_email.py | 5 ++++-
gracedb/alerts/tests/test_phone.py | 5 ++++-
gracedb/alerts/tests/test_views.py | 5 ++++-
gracedb/api/tests/test_authentication.py | 5 ++++-
gracedb/api/tests/test_throttling.py | 5 ++++-
gracedb/api/tests/utils.py | 5 ++++-
7 files changed, 28 insertions(+), 7 deletions(-)
diff --git a/gracedb/alerts/tests/test_access.py b/gracedb/alerts/tests/test_access.py
index cbd068b1f..fb6da4767 100644
--- a/gracedb/alerts/tests/test_access.py
+++ b/gracedb/alerts/tests/test_access.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.contrib.auth.models import Group as AuthGroup
diff --git a/gracedb/alerts/tests/test_email.py b/gracedb/alerts/tests/test_email.py
index ea2a65559..238317505 100644
--- a/gracedb/alerts/tests/test_email.py
+++ b/gracedb/alerts/tests/test_email.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.test import override_settings
diff --git a/gracedb/alerts/tests/test_phone.py b/gracedb/alerts/tests/test_phone.py
index 163664776..96b458e4d 100644
--- a/gracedb/alerts/tests/test_phone.py
+++ b/gracedb/alerts/tests/test_phone.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.test import override_settings
diff --git a/gracedb/alerts/tests/test_views.py b/gracedb/alerts/tests/test_views.py
index aeb6e6897..488734d29 100644
--- a/gracedb/alerts/tests/test_views.py
+++ b/gracedb/alerts/tests/test_views.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
import pytest
from django.conf import settings
diff --git a/gracedb/api/tests/test_authentication.py b/gracedb/api/tests/test_authentication.py
index d1f0c9f7e..a43591594 100644
--- a/gracedb/api/tests/test_authentication.py
+++ b/gracedb/api/tests/test_authentication.py
@@ -1,5 +1,8 @@
from base64 import b64encode
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.urls import reverse
diff --git a/gracedb/api/tests/test_throttling.py b/gracedb/api/tests/test_throttling.py
index 6f2d6e593..f917f12bf 100644
--- a/gracedb/api/tests/test_throttling.py
+++ b/gracedb/api/tests/test_throttling.py
@@ -1,4 +1,7 @@
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.core.cache import caches
diff --git a/gracedb/api/tests/utils.py b/gracedb/api/tests/utils.py
index bef02a5db..872a333e0 100644
--- a/gracedb/api/tests/utils.py
+++ b/gracedb/api/tests/utils.py
@@ -1,5 +1,8 @@
from copy import deepcopy
-import mock
+try:
+ from unittest import mock
+except ImportError: # python < 3
+ import mock
from django.conf import settings
from django.core.cache import caches
--
GitLab
From 9bdb45fb8609171c5ab89df4ee206e536246b568 Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Thu, 25 Apr 2019 20:48:22 +0100
Subject: [PATCH 007/106] gracedb.search: import reduce in python3
---
gracedb/search/query/superevents.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/gracedb/search/query/superevents.py b/gracedb/search/query/superevents.py
index fae5156e7..2ccaffc3a 100644
--- a/gracedb/search/query/superevents.py
+++ b/gracedb/search/query/superevents.py
@@ -2,6 +2,10 @@ from __future__ import absolute_import
import datetime
import logging
import pytz
+try:
+ from functools import reduce
+except ImportError: # python < 3
+ pass
from django.conf import settings
from django.db.models import Q
--
GitLab
From 8b4c5f8b6e7cb91dc103e17354400235c343fc51 Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Wed, 24 Apr 2019 18:13:49 +0100
Subject: [PATCH 008/106] updated imports of StringIO for python3
---
gracedb/api/v1/events/views.py | 5 +++--
gracedb/core/middleware/profiling.py | 11 ++++++++---
gracedb/events/models.py | 6 +++++-
gracedb/events/reports.py | 7 +++++--
gracedb/events/translator.py | 7 +++++--
gracedb/superevents/models.py | 5 ++++-
6 files changed, 30 insertions(+), 11 deletions(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index 2a347decc..11f64fcad 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -4,10 +4,11 @@ import json
import logging
import os
import shutil
-import StringIO
try:
+ from io import StringIO
from urllib.parse import urlencode
except ImportError: # python < 3
+ from StringIO import StringIO
from urllib import urlencode
from django.conf import settings
@@ -277,7 +278,7 @@ class LigoLwRenderer(BaseRenderer):
xmldoc = assembleLigoLw(data)
# XXX Aaargh! Just give me the contents of the xml doc. Annoying.
- output = StringIO.StringIO()
+ output = StringIO()
xmldoc.write(output)
return output.getvalue()
diff --git a/gracedb/core/middleware/profiling.py b/gracedb/core/middleware/profiling.py
index 811c41aaf..6e6b0254d 100644
--- a/gracedb/core/middleware/profiling.py
+++ b/gracedb/core/middleware/profiling.py
@@ -5,9 +5,14 @@
import sys
import os
import re
-import hotshot, hotshot.stats
import tempfile
-import StringIO
+try:
+ from io import StringIO
+except ImportError: # python < 3
+ from StringIO import StringIO
+
+import hotshot, hotshot.stats
+
from django.utils.deprecation import MiddlewareMixin
from django.conf import settings
@@ -91,7 +96,7 @@ class ProfileMiddleware(MiddlewareMixin):
if (settings.DEBUG or request.user.is_superuser) and 'prof' in request.GET:
self.prof.close()
- out = StringIO.StringIO()
+ out = StringIO()
old_stdout = sys.stdout
sys.stdout = out
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index 2b4f11ed0..2e3ed5b11 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -33,7 +33,11 @@ from django.conf import settings
import pytz
import calendar
-from cStringIO import StringIO
+try:
+ from io import StringIO
+except ImportError: # python < 3
+ from cStringIO import StringIO
+
from hashlib import sha1
import shutil
diff --git a/gracedb/events/reports.py b/gracedb/events/reports.py
index 4edb24979..b6d70420e 100644
--- a/gracedb/events/reports.py
+++ b/gracedb/events/reports.py
@@ -21,7 +21,6 @@ import matplotlib
matplotlib.use('Agg')
import numpy
import matplotlib.pyplot as plot
-import StringIO
import base64
import sys
import calendar
@@ -30,6 +29,10 @@ from core.time_utils import posixToGpsTime
from django.utils import timezone
import pytz
import json
+try:
+ from io import StringIO
+except ImportError: # python < 3
+ from StringIO import StringIO
@internal_user_required
def histo(request):
@@ -124,7 +127,7 @@ def cluster(events):
return [e for e in events if not quieter(e)]
def to_png_image(out = sys.stdout):
- f = StringIO.StringIO()
+ f = StringIO()
plot.savefig(f, format="png")
return base64.b64encode(f.getvalue())
diff --git a/gracedb/events/translator.py b/gracedb/events/translator.py
index be6ccc4da..0877df9c2 100644
--- a/gracedb/events/translator.py
+++ b/gracedb/events/translator.py
@@ -18,7 +18,10 @@ from core.time_utils import isoToGps, isoToGpsFloat
from core.vfile import VersionedFile
import json
-import StringIO
+try:
+ from io import StringIO
+except ImportError: # python < 3
+ from StringIO import StringIO
from math import sqrt
@@ -192,7 +195,7 @@ def handle_uploaded_data(event, datafilename,
if datafilename:
xmldoc = load_filename(datafilename, contenthandler=LIGOLWContentHandler)
elif file_contents:
- f = StringIO.StringIO(file_contents)
+ f = StringIO(file_contents)
xmldoc, digest = load_fileobj(f, contenthandler=LIGOLWContentHandler)
else:
msg = "If you wanna make an injection event, I'm gonna need a filepath or filecontents."
diff --git a/gracedb/superevents/models.py b/gracedb/superevents/models.py
index 72548cff9..815a4bebf 100644
--- a/gracedb/superevents/models.py
+++ b/gracedb/superevents/models.py
@@ -1,4 +1,7 @@
-from cStringIO import StringIO
+try:
+ from io import StringIO
+except ImportError: # python < 3
+ from StringIO import StringIO
import datetime
from hashlib import sha1
import logging
--
GitLab
From 05c02f838df807e3fa30ba87ba54a10de5ea4cf5 Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Wed, 1 May 2019 23:16:14 +0100
Subject: [PATCH 009/106] gracedb.superevents: decode input to StringIO
only required on python2, but this code will break on python3 anyway
---
gracedb/superevents/models.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/superevents/models.py b/gracedb/superevents/models.py
index 815a4bebf..b91e3e28f 100644
--- a/gracedb/superevents/models.py
+++ b/gracedb/superevents/models.py
@@ -362,7 +362,7 @@ class Superevent(CleanSaveModel, AutoIncrementModel):
# object's primary key. We prepend 'superevent' so as to not
# have collisions with Event files
hash_input = 'superevent' + str(self.id)
- hdf = StringIO(sha1(hash_input).hexdigest())
+ hdf = StringIO(sha1(hash_input).hexdigest().decode('utf-8'))
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
--
GitLab
From 8d571b10a97b316e63f9eb658bba0d0daffb7a04 Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Thu, 2 May 2019 07:30:28 +0100
Subject: [PATCH 010/106] gracedb.events: updated Event.datadir for python3
we need to dance between str (unicode) and bytes to talk to hashlib and StringIO at the same time
---
gracedb/events/models.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index 2e3ed5b11..65e4af1c3 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -33,10 +33,7 @@ from django.conf import settings
import pytz
import calendar
-try:
- from io import StringIO
-except ImportError: # python < 3
- from cStringIO import StringIO
+from io import StringIO
from hashlib import sha1
import shutil
@@ -246,7 +243,11 @@ class Event(models.Model):
@property
def datadir(self):
# Create a file-like object which is the SHA-1 hexdigest of the Event's primary key
- hdf = StringIO(sha1(str(self.id)).hexdigest())
+ hid = sha1(str(self.id).encode("utf-8")).hexdigest()
+ try:
+ hdf = StringIO(hid.decode("utf-8"))
+ except AttributeError: # python < 3
+ hdf = StringIO(hid)
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
--
GitLab
From de66251a0d0bf35460c90c4b08a4e53115ceda8b Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Thu, 25 Apr 2019 20:47:53 +0100
Subject: [PATCH 011/106] gracedb.core: updated octal for python3
---
gracedb/core/vfile.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/core/vfile.py b/gracedb/core/vfile.py
index aaa1f9d52..c451e6b7f 100644
--- a/gracedb/core/vfile.py
+++ b/gracedb/core/vfile.py
@@ -122,7 +122,7 @@ class VersionedFile(file):
# os.O_EXCL causes the open to fail if the file already exists.
fd = os.open(actual_name,
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
- 0644)
+ 0o644)
# re-open
file.__init__(self, actual_name, *args, **kwargs)
# lose fd we used to ensure file creation.
--
GitLab
From 99b2ccdc503bd013a208c5cef2abba819e3cca8d Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Sat, 18 May 2019 13:27:07 -0500
Subject: [PATCH 012/106] Fix urlencode imports
Import version of urlencode which is packaged with Django in order
to facilitate Python 2-3 compatibility.
---
gracedb/api/v1/events/views.py | 3 +--
gracedb/api/v1/paginators.py | 6 ++----
gracedb/api/v1/superevents/paginators.py | 5 +++--
gracedb/events/tests/test_perms.py | 7 ++-----
gracedb/search/tests/test_access.py | 6 +-----
5 files changed, 9 insertions(+), 18 deletions(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index 11f64fcad..b6c6b8a39 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -6,10 +6,8 @@ import os
import shutil
try:
from io import StringIO
- from urllib.parse import urlencode
except ImportError: # python < 3
from StringIO import StringIO
- from urllib import urlencode
from django.conf import settings
from django.contrib.auth.models import User, Permission, Group as DjangoGroup
@@ -20,6 +18,7 @@ from django.http import HttpResponse, HttpResponseForbidden, \
HttpResponseNotFound, HttpResponseServerError, HttpResponseBadRequest
from django.http.request import QueryDict
from django.utils.functional import wraps
+from django.utils.http import urlencode
# Stuff for the LigoLwRenderer
from glue.ligolw import ligolw
diff --git a/gracedb/api/v1/paginators.py b/gracedb/api/v1/paginators.py
index f03eaa9c7..d099a4b0d 100644
--- a/gracedb/api/v1/paginators.py
+++ b/gracedb/api/v1/paginators.py
@@ -1,9 +1,7 @@
from collections import OrderedDict
import logging
-try:
- from urllib.parse import urlencode
-except ImportError: # python < 3
- from urllib import urlencode
+
+from django.utils.http import urlencode
from rest_framework import pagination
from rest_framework.response import Response
diff --git a/gracedb/api/v1/superevents/paginators.py b/gracedb/api/v1/superevents/paginators.py
index 599682a17..8cc967292 100644
--- a/gracedb/api/v1/superevents/paginators.py
+++ b/gracedb/api/v1/superevents/paginators.py
@@ -1,6 +1,7 @@
from collections import OrderedDict
import logging
-import urllib
+
+from django.utils.http import urlencode
from rest_framework import pagination
from rest_framework.response import Response
@@ -26,7 +27,7 @@ class CustomSupereventPagination(pagination.LimitOffsetPagination):
'start': last,
self.limit_query_param: self.limit,
}
- last_uri = base_uri + '?' + urllib.urlencode(param_dict)
+ last_uri = base_uri + '?' + urlencode(param_dict)
output = OrderedDict([
('numRows', numRows),
diff --git a/gracedb/events/tests/test_perms.py b/gracedb/events/tests/test_perms.py
index 36ab3923e..61e22e16b 100644
--- a/gracedb/events/tests/test_perms.py
+++ b/gracedb/events/tests/test_perms.py
@@ -4,6 +4,7 @@ from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import Permission, Group, User
from django.conf import settings
from django.urls import reverse
+from django.utils.http import urlencode
from guardian.models import GroupObjectPermission, UserObjectPermission
from guardian.shortcuts import assign_perm
@@ -16,11 +17,7 @@ from events.permission_utils import assign_default_event_perms
import json
import os
import shutil
-try:
- from urllib.parse import urlencode
-except ImportError: # python < 3
- from urllib import urlencode
-
+
#------------------------------------------------------------------------------
#------------------------------------------------------------------------------
# Some utilities
diff --git a/gracedb/search/tests/test_access.py b/gracedb/search/tests/test_access.py
index 45592ae8b..94625c3bb 100644
--- a/gracedb/search/tests/test_access.py
+++ b/gracedb/search/tests/test_access.py
@@ -1,13 +1,9 @@
-try:
- from urllib.parse import urlencode
-except ImportError: # python < 3
- from urllib import urlencode
-
from django.conf import settings
from django.contrib.auth.models import Group as DjangoGroup, Permission
from django.contrib.contenttypes.models import ContentType
from django.test import TestCase
from django.urls import reverse
+from django.utils.http import urlencode
from core.tests.utils import GraceDbTestBase
from events.models import Event
--
GitLab
From d520373e35f60bb27eecdc4bcd6fdf301902d8d5 Mon Sep 17 00:00:00 2001
From: Duncan Macleod <duncan.macleod@ligo.org>
Date: Wed, 24 Apr 2019 17:53:04 +0100
Subject: [PATCH 013/106] update print statements to function calls
---
docker/check_shibboleth_status | 18 +++++++++---------
docs/admin_docs/source/user_permissions.rst | 8 ++++----
docs/user_docs/source/auth.rst | 6 +++---
docs/user_docs/source/lvem.rst | 10 +++++-----
.../user_docs/source/responding_to_lvalert.rst | 11 +++++------
.../migrations/0005_initial_label_data.py | 4 ++--
gracedb/events/nltime.py | 8 ++++----
gracedb/events/tests/test_label_search.py | 2 +-
gracedb/ligoauth/models.py | 2 +-
9 files changed, 34 insertions(+), 35 deletions(-)
diff --git a/docker/check_shibboleth_status b/docker/check_shibboleth_status
index f515a2d91..0e30bfda2 100644
--- a/docker/check_shibboleth_status
+++ b/docker/check_shibboleth_status
@@ -56,10 +56,10 @@ host_url = host + "/" + urlpath
try:
response = urlopen(host_url, timeout=timeout)
except URLError:
- print "Error opening Shibboleth status page (" + host_url + ")."
+ print("Error opening Shibboleth status page (" + host_url + ").")
sys.exit(2)
except:
- print "Unknown error opening Shibboleth status page (" + host_url + ")."
+ print("Unknown error opening Shibboleth status page (" + host_url + ").")
sys.exit(3)
# Convert from string to ElementTree
@@ -67,11 +67,11 @@ try:
status_tree = ET.fromstring(response.read())
except ET.ParseError:
# Error parsing response.
- print "Error parsing response from server - not in XML format."
+ print("Error parsing response from server - not in XML format.")
sys.exit(2)
except:
# Error that is not ParseError.
- print "Unknown error occurred when parsing response from server."
+ print("Unknown error occurred when parsing response from server.")
sys.exit(3)
response.close()
@@ -81,12 +81,12 @@ response.close()
for tag in tags_to_check:
status_tag = status_tree.find(tag)
if (status_tag is None):
- print "Error: tag \'" + tag + "\' not found."
+ print("Error: tag \'" + tag + "\' not found.")
sys.exit(2)
else:
status_OK = status_tag.find('OK')
if (status_OK is None):
- print "Error: tag \'" + tag + "\' is not OK."
+ print("Error: tag \'" + tag + "\' is not OK.")
sys.exit(2)
# Check 2: make sure metadata feeds that we expect
@@ -96,12 +96,12 @@ srcs = [element.attrib['source'] for element in metaprov_tags]
for feed in metadata_feeds:
feed_found = [src.lower().find(feed) >= 0 for src in srcs]
if (sum(feed_found) < 1):
- print "MetadataProvider " + feed + " not found."
+ print("MetadataProvider " + feed + " not found.")
sys.exit(2)
elif (sum(feed_found) < 1):
- print "MetadataProvider " + feed + "found in multiple elements."
+ print("MetadataProvider " + feed + "found in multiple elements.")
sys.exit(2)
# If we make it to this point, everything is OK.
-print "All MetadataProviders found. Status and SessionCache are OK."
+print("All MetadataProviders found. Status and SessionCache are OK.")
sys.exit(0)
diff --git a/docs/admin_docs/source/user_permissions.rst b/docs/admin_docs/source/user_permissions.rst
index 781969a56..e5ddbd924 100644
--- a/docs/admin_docs/source/user_permissions.rst
+++ b/docs/admin_docs/source/user_permissions.rst
@@ -62,7 +62,7 @@ in real life (from inside the Django shell)::
>>> u = User.objects.get(username='albert.einstein@LIGO.ORG')
>>> if p in u.user_permissions.all():
- ...: print "Albert can add events!"
+ ...: print("Albert can add events!")
The Django ``User`` class has a convenience function ``has_perm`` to
make this easier::
@@ -72,7 +72,7 @@ make this easier::
>>> u = User.objects.get(username='albert.einstein@LIGO.ORG')
>>> if u.has_perm('events.add_event'):
- ...: print "Albert can add events!"
+ ...: print("Albert can add events!")
Again, notice that the ``has_perm`` function needs the codename to be scoped by
the app to which the model belongs. Both are required to fully specify the model.
@@ -124,7 +124,7 @@ event data. Thus, we have added a custom ``view`` permission for the event model
>>> perms = Permission.objects.filter(codename__startswith='view')
>>> for p in perms:
- ...: print p.codename
+ ...: print(p.codename)
...:
view_coincinspiralevent
view_event
@@ -314,7 +314,7 @@ can be done by adding the permission by hand::
>>> u = User.objects.get(username='albert.einstein@LIGO.ORG')
>>> u.user_permissions.add(p):
- ...: print "Albert can add events!"
+ ...: print("Albert can add events!")
Granting permission to populate a pipeline
------------------------------------------
diff --git a/docs/user_docs/source/auth.rst b/docs/user_docs/source/auth.rst
index c0d9ad385..21d751974 100644
--- a/docs/user_docs/source/auth.rst
+++ b/docs/user_docs/source/auth.rst
@@ -76,8 +76,8 @@ Shibbolized client as follows::
try:
r = client.ping()
except HTTPError as e:
- print e.message
+ print(e.message)
- print "Response code: %d" % r.status
- print "Response content: %s" % r.json()
+ print("Response code: %d" % r.status)
+ print("Response content: %s" % r.json())
diff --git a/docs/user_docs/source/lvem.rst b/docs/user_docs/source/lvem.rst
index 344f580cb..5234a94e8 100644
--- a/docs/user_docs/source/lvem.rst
+++ b/docs/user_docs/source/lvem.rst
@@ -85,10 +85,10 @@ For example, you can use the GraceDB Python client::
try:
r = client.ping()
except HTTPError as e:
- print e.message
-
- print "Response code: %d" % r.status
- print "Response content: %s" % r.json()
+ print(e.message)
+
+ print("Response code: %d" % r.status)
+ print("Response content: %s" % r.json())
If you're not comfortable using Python for scripted access to GraceDB, it is
also possible to use ``curl`` to directly make requests to the server with the
@@ -153,7 +153,7 @@ observation record consisting of three separate footprints::
decList, decWidthList, startTimeList, durationList, comment)
if r.status == 201: # 201 means 'Created'
- print 'Success!'
+ print('Success!')
Note that the start times are always assumed to be in UTC. For users not
familiar with Python, there are several other options available for uploading
diff --git a/docs/user_docs/source/responding_to_lvalert.rst b/docs/user_docs/source/responding_to_lvalert.rst
index a2a7db67c..764060924 100644
--- a/docs/user_docs/source/responding_to_lvalert.rst
+++ b/docs/user_docs/source/responding_to_lvalert.rst
@@ -329,7 +329,7 @@ Create a Python executable ``iReact.py`` and fill it with the following::
import sys
alert = json.loads(sys.stdin.read())
- print 'uid : ' + alert['uid']
+ print('uid : ' + alert['uid'])
Don't forget to give this executable permissions with::
@@ -377,7 +377,7 @@ Open ``iReact.py`` and modify it so it reads::
from ligo.gracedb.rest import GraceDb
alert = json.loads(sys.stdin.read())
- print 'uid : ' + alert['uid']
+ print('uid : ' + alert['uid'])
gdb = GraceDb() ### instantiate a GraceDB object which connects to the default server
@@ -419,15 +419,14 @@ more example for what ``iReact.py`` might look like::
FarThr = float(sys.argv[1])
alert = json.loads(sys.stdin.read())
- print 'uid : '+alert['uid']
+ print('uid : '+alert['uid'])
gdb = GraceDb() ### instantiate a GraceDB object which connects to the default server
if alert['alert_type'] == 'new': ### the event was just created and this is the first announcment
if alert['far'] < FarThr:
- file_obj = open("iReact.txt", "w")
- print >> file_obj, "wow! this was a rare event! It had FAR = %.3e < %.3e, which was my threshold"%(alert['far'], FarThr)
- file_obj.close()
+ with open("iReact.txt", "w") as file_obj:
+ print("wow! this was a rare event! It had FAR = %.3e < %.3e, which was my threshold"%(alert['far'], FarThr), file=file_obj)
gdb.writeLog( alert['uid'], message="user.name heard an alert about this new event!", filename="iReact.txt", tagname=["data_quality"] )
Try to figure out exactly what this version does. If you can
diff --git a/gracedb/events/migrations/0005_initial_label_data.py b/gracedb/events/migrations/0005_initial_label_data.py
index c1e5e92c2..1b4a15200 100644
--- a/gracedb/events/migrations/0005_initial_label_data.py
+++ b/gracedb/events/migrations/0005_initial_label_data.py
@@ -53,8 +53,8 @@ def remove_labels(apps, schema_editor):
try:
l = Label.objects.get(name=label_dict['name'])
except Label.DoesNotExist:
- print('Label {0} not found to be deleted, skipping.' \
- .format(label_dict['name']))
+ print(('Label {0} not found to be deleted, skipping.' \
+ .format(label_dict['name'])))
break
l.delete()
diff --git a/gracedb/events/nltime.py b/gracedb/events/nltime.py
index 0cbcc7338..fc2eaf59b 100755
--- a/gracedb/events/nltime.py
+++ b/gracedb/events/nltime.py
@@ -187,11 +187,11 @@ if __name__ == "__main__":
2009/12/22 12:13:14""".splitlines()
for t in tests:
- print t, "(relative to %s)" % timezone.now()
+ print(t, "(relative to %s)" % timezone.now())
res = nlTimeExpression.parseString(t)
if "calculatedTime" in res:
- print res.calculatedTime
+ print(res.calculatedTime)
else:
- print "???"
- print
+ print("???")
+ print()
diff --git a/gracedb/events/tests/test_label_search.py b/gracedb/events/tests/test_label_search.py
index 14063ae50..75cdd7310 100644
--- a/gracedb/events/tests/test_label_search.py
+++ b/gracedb/events/tests/test_label_search.py
@@ -54,7 +54,7 @@ class LabelSearchTestCase(TestCase):
def test_all_queries(self):
for key, d in QUERY_CASES.items():
- print "Checking %s ... " % key
+ print("Checking %s ... " % key)
# Explicitly search for test events
query = 'Test ' + d['query']
self.assertEqual(set(get_pks_for_query(query)), set(d['pk_list']))
diff --git a/gracedb/ligoauth/models.py b/gracedb/ligoauth/models.py
index 8dfad9e64..238bb110f 100644
--- a/gracedb/ligoauth/models.py
+++ b/gracedb/ligoauth/models.py
@@ -12,7 +12,7 @@ class LigoLdapUser(User):
def name(self):
# XXX I really don't freaking understand WHY THIS SEEMS NECESSARY.
- # print user.name() gives an idiotic ascii coding error otherwise. WHY!?
+ # print(user.name()) gives an idiotic ascii coding error otherwise. WHY!?
return u"{0} {1}".format(self.first_name, self.last_name).encode('utf-8')
--
GitLab
From 41446b75e72dad3d44b9e048696031414b838f7b Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Wed, 17 Jul 2019 13:48:47 -0500
Subject: [PATCH 014/106] WIP: get lvalert-overseer from gitlab for python3
compatibility
---
requirements.txt | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index f47d82f12..d8dea6d06 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,7 +17,9 @@ ipdb==0.10.2
ipython==5.5.0
lalsuite==6.53 # undocumented requirement for lscsoft-glue
ligo-lvalert==1.5.6
-ligo-lvalert-overseer==0.1.3
+# Temporary pin of lvalert-overseer for Python 3 update
+git+https://git.ligo.org/lscsoft/lvalert-overseer.git@python3-overseer#egg=ligo-lvalert-overseer
+#ligo-lvalert-overseer==0.1.3
lscsoft-glue==1.60.0
lxml==4.2.0
matplotlib==2.0.0
--
GitLab
From 44185f23886932fc5d95e5a5b50b4e40635949e7 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Wed, 17 Jul 2019 14:31:02 -0500
Subject: [PATCH 015/106] core: rework VersionedFile
Previously, core.vfile.VersionedFile inherited from file,
which is problematic for several reasons, but primarily because
file doesn't exist in Python 3. So we rework it to just be a
normal class with a file attribute, and add/change certain
methods accordingly.
---
gracedb/api/v1/events/views.py | 2 +-
gracedb/api/v1/superevents/views.py | 2 +-
gracedb/core/vfile.py | 21 ++++++++++++++-------
gracedb/events/serialize.py | 2 +-
4 files changed, 17 insertions(+), 10 deletions(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index b6c6b8a39..f2fab2b52 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -617,7 +617,7 @@ class EventDetail(InheritPermissionsAPIView):
#for chunk in f.chunks():
# fdest.write(chunk)
#fdest.close()
- shutil.copyfileobj(f, fdest)
+ shutil.copyfileobj(f, fdest.file)
fdest.close()
# Extract Info from uploaded data
diff --git a/gracedb/api/v1/superevents/views.py b/gracedb/api/v1/superevents/views.py
index 2575d53af..be85aa3ad 100644
--- a/gracedb/api/v1/superevents/views.py
+++ b/gracedb/api/v1/superevents/views.py
@@ -15,7 +15,7 @@ from rest_framework.views import APIView
from core.file_utils import get_file_list
from core.http import check_and_serve_file
-from core.vfile import VersionedFile, FileVersionError, FileVersionNameError
+from core.vfile import FileVersionError, FileVersionNameError
from events.models import Event, Label
from events.view_utils import reverse as gracedb_reverse
from ligoauth.utils import is_internal
diff --git a/gracedb/core/vfile.py b/gracedb/core/vfile.py
index c451e6b7f..5a8786722 100644
--- a/gracedb/core/vfile.py
+++ b/gracedb/core/vfile.py
@@ -24,7 +24,7 @@ class FileVersionNameError(Exception):
pass
-class VersionedFile(file):
+class VersionedFile(object):
"""
Open a versioned file.
@@ -74,7 +74,7 @@ class VersionedFile(file):
# one scoped inside of this __init__). But I'm reluctant to mess with
# Brian's code too much.
self.version = version
- file.__init__(self, actual_name, *args, **kwargs)
+ self.file = open(actual_name, *args, **kwargs)
# Otherwise...
@@ -124,7 +124,7 @@ class VersionedFile(file):
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o644)
# re-open
- file.__init__(self, actual_name, *args, **kwargs)
+ self.file = open(actual_name, *args, **kwargs)
# lose fd we used to ensure file creation.
os.close(fd)
break
@@ -208,6 +208,13 @@ class VersionedFile(file):
return [int(f.split(',')[1])
for f in os.listdir(d) if f.startswith(name + ',')]
+ def write(self, s):
+ self.file.write(s)
+
+ @property
+ def closed(self):
+ return self.file.closed
+
def close(self):
if self.writing:
# no need to update symlink if we were only reading.
@@ -215,13 +222,13 @@ class VersionedFile(file):
# file -- trying to discover the lastest version fails
# painfully. (max(known_versions()) => max([]))
self._repoint_symlink()
- if not self.closed:
- file.close(self)
+ if not self.file.closed:
+ self.file.close()
def __del__(self):
# XXX file does not have a __del__ method. Should we?
- if not self.closed:
- self.close()
+ if not self.file.closed:
+ self.file.close()
@staticmethod
def guess_mimetype(filename):
diff --git a/gracedb/events/serialize.py b/gracedb/events/serialize.py
index 56831a079..9551517ce 100644
--- a/gracedb/events/serialize.py
+++ b/gracedb/events/serialize.py
@@ -89,7 +89,7 @@ def write_output_files(root_dir, xmldoc, log_content, \
"""
f = VersionedFile(root_dir+'/'+xml_fname,'w')
- xmldoc.write(f)
+ xmldoc.write(f.file)
f.close()
f = VersionedFile(root_dir+'/'+log_fname,'w')
--
GitLab
From 853524602b04e2f0eb25af564826976595fdd71d Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 10:12:31 -0500
Subject: [PATCH 016/106] events: python 3 compatibility for encoding utils
---
gracedb/events/templatetags/logtags.py | 1 -
gracedb/events/templatetags/scientific.py | 7 +++----
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/gracedb/events/templatetags/logtags.py b/gracedb/events/templatetags/logtags.py
index c19075255..0e38cfac6 100644
--- a/gracedb/events/templatetags/logtags.py
+++ b/gracedb/events/templatetags/logtags.py
@@ -1,6 +1,5 @@
from django import template
-from django.utils.encoding import force_unicode
from django.utils.safestring import mark_safe
from ..models import Tag, EventLog
diff --git a/gracedb/events/templatetags/scientific.py b/gracedb/events/templatetags/scientific.py
index 0cdfafad2..0c3df5d80 100644
--- a/gracedb/events/templatetags/scientific.py
+++ b/gracedb/events/templatetags/scientific.py
@@ -1,7 +1,6 @@
-
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
from django import template
-from django.utils.encoding import force_unicode
+from django.utils.encoding import force_text
from django.utils.safestring import mark_safe
from django.utils import formats
register = template.Library()
@@ -38,7 +37,7 @@ def scientificformat(text):
"""
try:
- input_val = force_unicode(text)
+ input_val = force_text(text)
d = Decimal(input_val)
except UnicodeEncodeError:
return u''
@@ -46,7 +45,7 @@ def scientificformat(text):
if input_val in special_floats:
return input_val
try:
- d = Decimal(force_unicode(float(text)))
+ d = Decimal(force_text(float(text)))
except (ValueError, InvalidOperation, TypeError, UnicodeEncodeError):
return u''
--
GitLab
From 3b80b108771a79544d56f2c7d1f401d3180512e0 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 10:36:34 -0500
Subject: [PATCH 017/106] python 3 compatibility: dict usage
Remove usage of .keys(), .has_key(), and .iteritems(), and
account for the fact that .values() produces an iterator in Python 3.
---
docs/user_docs/source/rest.rst | 2 +-
gracedb/api/exceptions.py | 3 ++-
gracedb/api/v1/events/views.py | 13 +++++++------
gracedb/api/v1/fields.py | 4 ++--
gracedb/api/v1/main/tests/test_views.py | 2 +-
gracedb/api/v1/superevents/filters.py | 2 +-
.../api/v1/superevents/tests/test_serializers.py | 12 ++++++------
gracedb/core/forms.py | 4 ++--
gracedb/core/middleware/proxy.py | 2 +-
gracedb/events/buildVOEvent.py | 2 +-
.../migrations/0011_add_O2VirgoTest_search.py | 2 +-
gracedb/events/models.py | 4 ++--
gracedb/events/serialize.py | 6 +++---
gracedb/events/tests/test_perms.py | 2 +-
gracedb/events/view_logic.py | 10 +++++-----
.../migrations/0005_update_emfollow_accounts.py | 4 ++--
.../ligoauth/migrations/0019_update_idq_certs.py | 4 ++--
gracedb/ligoauth/tests/test_backends.py | 4 ++--
.../0016_create_access_and_superevent_groups.py | 2 +-
gracedb/migrations/auth/0017_assign_permissions.py | 4 ++--
.../guardian/0003_update_emfollow_accounts.py | 4 ++--
gracedb/search/constants.py | 2 +-
gracedb/search/query/events.py | 4 ++--
gracedb/search/query/superevents.py | 2 +-
gracedb/search/tests/test_access.py | 12 ++++++------
gracedb/superevents/utils.py | 6 +++---
26 files changed, 60 insertions(+), 58 deletions(-)
diff --git a/docs/user_docs/source/rest.rst b/docs/user_docs/source/rest.rst
index 149588fe3..3f4157fec 100644
--- a/docs/user_docs/source/rest.rst
+++ b/docs/user_docs/source/rest.rst
@@ -81,7 +81,7 @@ If you have reason to believe that your request may be throttled, you can wrap i
except HTTPError as e:
try:
rdict = json.loads(e.message)
- if 'retry-after' in rdict.keys():
+ if 'retry-after' in rdict:
time.sleep(int(rdict['retry-after']))
continue
else:
diff --git a/gracedb/api/exceptions.py b/gracedb/api/exceptions.py
index 07c86d032..2716683b8 100644
--- a/gracedb/api/exceptions.py
+++ b/gracedb/api/exceptions.py
@@ -13,7 +13,8 @@ def gracedb_exception_handler(exc, context):
if hasattr(exc, 'detail') and hasattr(exc.detail, 'values'):
# Combine values into one list
- exc_out = [item for sublist in exc.detail.values() for item in sublist]
+ exc_out = [item for sublist in list(exc.detail.values())
+ for item in sublist]
# For only one exception, just print it rather than the list
if len(exc_out) == 1:
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index f2fab2b52..58fc41311 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -246,7 +246,8 @@ class CoincAccess(Exception):
return repr(self.detail)
def assembleLigoLw(data):
- if 'events' in data.keys():
+ # data is a dict
+ if 'events' in data:
eventDictList = data['events']
else:
# There is only one event.
@@ -272,7 +273,7 @@ class LigoLwRenderer(BaseRenderer):
# XXX If there was an error, we will return the error message
# in plain text, effectively ignoring the accepts header.
# Somewhat irregular?
- if 'error' in data.keys():
+ if 'error' in data:
return data['error']
xmldoc = assembleLigoLw(data)
@@ -286,12 +287,12 @@ class TSVRenderer(BaseRenderer):
format = 'tsv'
def render(self, data, media_type=None, renderer_context=None):
- if 'error' in data.keys():
+ if 'error' in data:
return data['error']
accessFun = {
"labels" : lambda e: \
- ",".join(e['labels'].keys()),
+ ",".join(list(e['labels'])),
"dataurl" : lambda e: e['links']['files'],
}
def defaultAccess(e,a):
@@ -957,7 +958,7 @@ class EventLogList(InheritPermissionsAPIView):
rv = eventLogToDict(logentry, request=request)
response = Response(rv, status=status.HTTP_201_CREATED)
response['Location'] = rv['self']
- #if 'tagWarning' in tw_dict.keys():
+ #if 'tagWarning' in tw_dict:
# response['tagWarning'] = tw_dict['tagWarning']
# Issue alert.
@@ -1064,7 +1065,7 @@ class EMObservationList(InheritPermissionsAPIView):
# XXX Note the following hack.
# If this JSON information is requested for skymapViewer, use a different
# representation for backwards compatibility.
- if 'skymapViewer' in request.query_params.keys():
+ if 'skymapViewer' in request.query_params:
emo = [ skymapViewerEMObservationToDict(emo, request)
for emo in emo_set.iterator() ]
diff --git a/gracedb/api/v1/fields.py b/gracedb/api/v1/fields.py
index ebc3ec316..7fc167bc5 100644
--- a/gracedb/api/v1/fields.py
+++ b/gracedb/api/v1/fields.py
@@ -125,8 +125,8 @@ class GenericField(fields.Field):
err_msg = self.get_does_not_exist_error(data)
else:
err_msg = '{model} with {lf}={data} does not exist'.format(
- model=self.model.__name__, lf=model_dict.keys()[0],
- data=model_dict.values()[0]
+ model=self.model.__name__, lf=list(model_dict)[0],
+ data=list(model_dict.values())[0]
)
raise exceptions.ValidationError(err_msg)
diff --git a/gracedb/api/v1/main/tests/test_views.py b/gracedb/api/v1/main/tests/test_views.py
index 74d8c9810..728d34526 100644
--- a/gracedb/api/v1/main/tests/test_views.py
+++ b/gracedb/api/v1/main/tests/test_views.py
@@ -46,5 +46,5 @@ class TestUserInfoView(GraceDbApiTestBase):
self.assertEqual(response.status_code, 200)
# Test information
- self.assertEqual(response.data.keys(), ['username'])
+ self.assertEqual(list(response.data), ['username'])
self.assertEqual(response.data['username'], 'AnonymousUser')
diff --git a/gracedb/api/v1/superevents/filters.py b/gracedb/api/v1/superevents/filters.py
index cb6a15306..8ff62a004 100644
--- a/gracedb/api/v1/superevents/filters.py
+++ b/gracedb/api/v1/superevents/filters.py
@@ -58,7 +58,7 @@ class SupereventOrderingFilter(filters.OrderingFilter):
for f in fields:
prefix = '-' if f.startswith('-') else ''
f_s = f.lstrip('-')
- if f_s in self.field_map.keys():
+ if f_s in self.field_map:
mapped_fields = self.field_map[f_s]
if not isinstance(mapped_fields, list):
mapped_fields = [mapped_fields]
diff --git a/gracedb/api/v1/superevents/tests/test_serializers.py b/gracedb/api/v1/superevents/tests/test_serializers.py
index 809d3ea24..939bf64e0 100644
--- a/gracedb/api/v1/superevents/tests/test_serializers.py
+++ b/gracedb/api/v1/superevents/tests/test_serializers.py
@@ -45,12 +45,12 @@ class TestSupereventSerializerViaWeb(SupereventSetup, GraceDbApiTestBase):
response = self.request_as_user(url, "GET", self.internal_user)
self.assertEqual(response.status_code, 200)
# Check data on page
- response_keys = response.json().keys()
+ response_keys = list(response.json())
response_links = response.json()['links']
self.assertIn('preferred_event', response_keys)
self.assertIn('gw_events', response_keys)
self.assertIn('em_events', response_keys)
- self.assertIn('events', response_links.keys())
+ self.assertIn('events', response_links)
def test_lvem_user_get_superevent_detail(self):
"""LV-EM user sees events link and all event graceids"""
@@ -61,12 +61,12 @@ class TestSupereventSerializerViaWeb(SupereventSetup, GraceDbApiTestBase):
response = self.request_as_user(url, "GET", self.lvem_user)
self.assertEqual(response.status_code, 200)
# Check data on page
- response_keys = response.json().keys()
+ response_keys = list(response.json())
response_links = response.json()['links']
self.assertIn('preferred_event', response_keys)
self.assertIn('gw_events', response_keys)
self.assertIn('em_events', response_keys)
- self.assertIn('events', response_links.keys())
+ self.assertIn('events', response_links)
def test_public_user_get_superevent_detail(self):
"""Public user does not see events link or all event graceids"""
@@ -77,9 +77,9 @@ class TestSupereventSerializerViaWeb(SupereventSetup, GraceDbApiTestBase):
response = self.request_as_user(url, "GET")
self.assertEqual(response.status_code, 200)
# Check data on page
- response_keys = response.json().keys()
+ response_keys = list(response.json())
response_links = response.json()['links']
self.assertNotIn('preferred_event', response_keys)
self.assertNotIn('gw_events', response_keys)
self.assertNotIn('em_events', response_keys)
- self.assertNotIn('events', response_links.keys())
+ self.assertNotIn('events', response_links)
diff --git a/gracedb/core/forms.py b/gracedb/core/forms.py
index 45a5e8462..4e49be5bd 100644
--- a/gracedb/core/forms.py
+++ b/gracedb/core/forms.py
@@ -37,8 +37,8 @@ class ModelFormUpdateMixin(forms.ModelForm):
# Insert instance data for missing fields only
instance_data = self.get_instance_data()
- for key in self.fields.keys():
- if not self.data.has_key(key) and instance_data[key]:
+ for key in self.fields:
+ if not key in self.data and instance_data[key]:
self.data[key] = instance_data[key]
diff --git a/gracedb/core/middleware/proxy.py b/gracedb/core/middleware/proxy.py
index 0f989656a..93c05a066 100644
--- a/gracedb/core/middleware/proxy.py
+++ b/gracedb/core/middleware/proxy.py
@@ -11,7 +11,7 @@ class XForwardedForMiddleware(object):
def __call__(self, request):
# Process request -----------------------------------------------------
- if request.META.has_key('HTTP_X_FORWARDED_FOR'):
+ if 'HTTP_X_FORWARDED_FOR' in request.META:
request.META['REMOTE_ADDR'] = \
request.META['HTTP_X_FORWARDED_FOR'].split(",")[0].strip()
diff --git a/gracedb/events/buildVOEvent.py b/gracedb/events/buildVOEvent.py
index 1ef4f27e3..67b159665 100644
--- a/gracedb/events/buildVOEvent.py
+++ b/gracedb/events/buildVOEvent.py
@@ -59,7 +59,7 @@ def buildVOEvent(event, voevent, request=None):
if not event.gpstime:
raise VOEventBuilderException("Cannot build a VOEvent because event has no gpstime.")
- if not voevent.voevent_type in VOEVENT_TYPE_DICT.keys():
+ if not voevent.voevent_type in VOEVENT_TYPE_DICT:
raise VOEventBuilderException("voevent_type must be preliminary, initial, update, or retraction")
# Let's convert that voevent_type to something nicer looking
diff --git a/gracedb/events/migrations/0011_add_O2VirgoTest_search.py b/gracedb/events/migrations/0011_add_O2VirgoTest_search.py
index 8c625f349..3bb7f6cb3 100644
--- a/gracedb/events/migrations/0011_add_O2VirgoTest_search.py
+++ b/gracedb/events/migrations/0011_add_O2VirgoTest_search.py
@@ -18,7 +18,7 @@ def add_search(apps, schema_editor):
# Create search
new_search, created = Search.objects.get_or_create(name=SEARCH['name'])
if created:
- for key in SEARCH.keys():
+ for key in SEARCH:
setattr(new_search, key, SEARCH[key])
new_search.save()
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index 65e4af1c3..d1f4fa4a5 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -833,8 +833,8 @@ class SingleInspiral(models.Model):
return cls._field_names
except AttributeError: pass
model_field_names = set([ x.name for x in cls._meta.get_fields(include_parents=False) ])
- ligolw_field_names = set(
- glue.ligolw.lsctables.SnglInspiralTable.validcolumns.keys())
+ ligolw_field_names = set(list(
+ glue.ligolw.lsctables.SnglInspiralTable.validcolumns))
cls._field_names = model_field_names.intersection(ligolw_field_names)
return cls._field_names
diff --git a/gracedb/events/serialize.py b/gracedb/events/serialize.py
index 9551517ce..1a40f7cae 100644
--- a/gracedb/events/serialize.py
+++ b/gracedb/events/serialize.py
@@ -39,7 +39,7 @@ H1_detlist = ['H1']
L1_detlist = ['L1']
V1_detlist = ['V1']
-#this is the subset of SnglInspiralTable.validcolumn.keys() that
+#this is the subset of keys in SnglInspiralTable.validcolumn that
#are assigned from MBTA coinc triggers
MBTA_set_keys = ['ifo', 'search', 'end_time', 'end_time_ns', 'mass1', 'mass2',\
'mchirp', 'mtotal', 'eta', 'snr', 'eff_distance', 'event_id',\
@@ -166,7 +166,7 @@ def populate_omega_tables(datafile, set_keys = Omega_set_keys):
row.confidence = -log(float(omega_data['probGlitch']))
cid = lsctables.CoincTable.get_next_id()
row.coinc_event_id = cid
- for key in mb_table.validcolumns.keys():
+ for key in mb_table.validcolumns:
if key not in set_keys:
setattr(row,key,None)
mb_table.append(row)
@@ -215,7 +215,7 @@ def populate_cwb_tables(datafile, set_keys=CWB_set_keys):
row.start_time_ns = st.nanoseconds
cid = lsctables.CoincTable.get_next_id()
row.coinc_event_id = cid
- for key in mb_table.validcolumns.keys():
+ for key in mb_table.validcolumns:
if key not in set_keys:
setattr(row,key,None)
mb_table.append(row)
diff --git a/gracedb/events/tests/test_perms.py b/gracedb/events/tests/test_perms.py
index 61e22e16b..72d14684e 100644
--- a/gracedb/events/tests/test_perms.py
+++ b/gracedb/events/tests/test_perms.py
@@ -50,7 +50,7 @@ def extra_args(user):
# Need to handle reverse proxy case where headers are used
# instead of Apache environment variables.
if settings.USE_X_FORWARDED_HOST:
- for k in AUTH_DICT.keys():
+ for k in AUTH_DICT:
AUTH_DICT['HTTP_' + k.upper()] = AUTH_DICT.pop(k)
return AUTH_DICT
diff --git a/gracedb/events/view_logic.py b/gracedb/events/view_logic.py
index 2bfe74d73..8e54e94a4 100644
--- a/gracedb/events/view_logic.py
+++ b/gracedb/events/view_logic.py
@@ -299,28 +299,28 @@ def get_performance_info():
# Localize so we can compare with aware datetimes
dt = SERVER_TZ.localize(dt)
if dt > dt_min:
- if method not in totals_by_method.keys():
+ if method not in totals_by_method:
totals_by_method[method] = 1
totals_by_status[method] = {status: 1}
else:
totals_by_method[method] += 1
- if status not in totals_by_status[method].keys():
+ if status not in totals_by_status[method]:
totals_by_status[method][status] = 1
else:
totals_by_status[method][status] += 1
# Calculate summary information:
summaries = {}
- for method in totals_by_method.keys():
+ for method in totals_by_method:
summaries[method] = {'gt_500': 0, 'btw_300_500': 0}
- for key in totals_by_status[method].keys():
+ for key in totals_by_status[method]:
if key >= 500:
summaries[method]['gt_500'] += totals_by_status[method][key]
elif key >= 300:
summaries[method]['btw_300_500'] += totals_by_status[method][key]
# Normalize
if totals_by_method[method] > 0:
- for key in summaries[method].keys():
+ for key in summaries[method]:
summaries[method][key] = float(summaries[method][key])/totals_by_method[method]
context = {
diff --git a/gracedb/ligoauth/migrations/0005_update_emfollow_accounts.py b/gracedb/ligoauth/migrations/0005_update_emfollow_accounts.py
index f88071fce..ce155144e 100644
--- a/gracedb/ligoauth/migrations/0005_update_emfollow_accounts.py
+++ b/gracedb/ligoauth/migrations/0005_update_emfollow_accounts.py
@@ -49,7 +49,7 @@ def deactivate_old_and_add_new_accounts(apps, schema_editor):
Group = apps.get_model('auth', 'Group')
# Mark old users as inactive and delete their certificates
- for username in OLD_ACCOUNTS.keys():
+ for username in OLD_ACCOUNTS:
user = LocalUser.objects.get(username=username)
for subject in OLD_ACCOUNTS[username]['certs']:
cert = user.x509cert_set.get(subject=subject)
@@ -73,7 +73,7 @@ def activate_old_and_remove_new_accounts(apps, schema_editor):
X509Cert = apps.get_model('ligoauth', 'X509Cert')
# Activate old accounts and add their X509 certificates
- for username in OLD_ACCOUNTS.keys():
+ for username in OLD_ACCOUNTS:
user = LocalUser.objects.get(username=username)
for subject in OLD_ACCOUNTS[username]['certs']:
cert = user.x509cert_set.create(subject=subject)
diff --git a/gracedb/ligoauth/migrations/0019_update_idq_certs.py b/gracedb/ligoauth/migrations/0019_update_idq_certs.py
index 66a636670..0bb25d3e2 100644
--- a/gracedb/ligoauth/migrations/0019_update_idq_certs.py
+++ b/gracedb/ligoauth/migrations/0019_update_idq_certs.py
@@ -19,7 +19,7 @@ ACCOUNTS = {
def update_certs(apps, schema_editor):
User = apps.get_model('auth', 'User')
- for user, certs in ACCOUNTS.iteritems():
+ for user, certs in ACCOUNTS.items():
# Get user
user = User.objects.get(username=user)
@@ -35,7 +35,7 @@ def revert_certs(apps, schema_editor):
User = apps.get_model('auth', 'User')
X509Cert = apps.get_model('ligoauth', 'X509Cert')
- for user, certs in ACCOUNTS.iteritems():
+ for user, certs in ACCOUNTS.items():
# Get user
user = User.objects.get(username=user)
diff --git a/gracedb/ligoauth/tests/test_backends.py b/gracedb/ligoauth/tests/test_backends.py
index 7a485526a..8505ee8b2 100644
--- a/gracedb/ligoauth/tests/test_backends.py
+++ b/gracedb/ligoauth/tests/test_backends.py
@@ -48,7 +48,7 @@ class TestShibbolethRemoteUserBackend(GraceDbTestBase):
# Set up request and headers
request = self.factory.get(self.url)
for k,v in user_data.items():
- if settings.SHIB_ATTRIBUTE_MAP.has_key(k):
+ if k in settings.SHIB_ATTRIBUTE_MAP:
request.META[settings.SHIB_ATTRIBUTE_MAP[k]] = v
# Pass data to backend
@@ -71,7 +71,7 @@ class TestShibbolethRemoteUserBackend(GraceDbTestBase):
# Set up request and headers
request = self.factory.get(self.url)
for k,v in new_user_data.items():
- if settings.SHIB_ATTRIBUTE_MAP.has_key(k):
+ if k in settings.SHIB_ATTRIBUTE_MAP:
request.META[settings.SHIB_ATTRIBUTE_MAP[k]] = v
# Get initial user data
diff --git a/gracedb/migrations/auth/0016_create_access_and_superevent_groups.py b/gracedb/migrations/auth/0016_create_access_and_superevent_groups.py
index 461952d76..03eae3970 100644
--- a/gracedb/migrations/auth/0016_create_access_and_superevent_groups.py
+++ b/gracedb/migrations/auth/0016_create_access_and_superevent_groups.py
@@ -17,7 +17,7 @@ def add_groups(apps, schema_editor):
Group = apps.get_model('auth', 'Group')
User = apps.get_model('auth', 'User')
- for group_name, usernames in GROUPS.iteritems():
+ for group_name, usernames in GROUPS.items():
g, _ = Group.objects.get_or_create(name=group_name)
users = User.objects.filter(username__in=usernames)
g.user_set.add(*users)
diff --git a/gracedb/migrations/auth/0017_assign_permissions.py b/gracedb/migrations/auth/0017_assign_permissions.py
index 3aa26471c..2c8fd59c8 100644
--- a/gracedb/migrations/auth/0017_assign_permissions.py
+++ b/gracedb/migrations/auth/0017_assign_permissions.py
@@ -76,7 +76,7 @@ def add_perms(apps, schema_editor):
Permission = apps.get_model('auth', 'Permission')
# Add superevent permissions to groups
- for codename, group_names in SUPEREVENT_PERMS.iteritems():
+ for codename, group_names in SUPEREVENT_PERMS.items():
p = Permission.objects.get(codename=codename,
content_type__app_label='superevents')
groups = Group.objects.filter(name__in=group_names)
@@ -88,7 +88,7 @@ def remove_perms(apps, schema_editor):
Permission = apps.get_model('auth', 'Permission')
# Add permissions to groups
- for codename, group_names in SUPEREVENT_PERMS.iteritems():
+ for codename, group_names in SUPEREVENT_PERMS.items():
p = Permission.objects.get(codename=codename,
content_type__app_label='superevents')
groups = Group.objects.filter(name__in=group_names)
diff --git a/gracedb/migrations/guardian/0003_update_emfollow_accounts.py b/gracedb/migrations/guardian/0003_update_emfollow_accounts.py
index 3f799dbfb..68315728c 100644
--- a/gracedb/migrations/guardian/0003_update_emfollow_accounts.py
+++ b/gracedb/migrations/guardian/0003_update_emfollow_accounts.py
@@ -26,7 +26,7 @@ def update_perms(apps, schema_editor):
ctype = ContentType.objects.get_for_model(Pipeline)
# Remove UOPs from old accounts which allow pipeline population
- for username, pipelines in OLD_ACCOUNTS.iteritems():
+ for username, pipelines in OLD_ACCOUNTS.items():
user = LocalUser.objects.get(username=username)
for pipeline_name in pipelines:
pipeline = Pipeline.objects.get(name=pipeline_name)
@@ -53,7 +53,7 @@ def revert_perms(apps, schema_editor):
ctype = ContentType.objects.get_for_model(Pipeline)
# Re-add UOPs to old accounts which allow pipeline population
- for username, pipelines in OLD_ACCOUNTS.iteritems():
+ for username, pipelines in OLD_ACCOUNTS.items():
user = LocalUser.objects.get(username=username)
for pipeline_name in pipelines:
pipeline = Pipeline.objects.get(name=pipeline_name)
diff --git a/gracedb/search/constants.py b/gracedb/search/constants.py
index 8f5582588..5e0c74c96 100644
--- a/gracedb/search/constants.py
+++ b/gracedb/search/constants.py
@@ -11,7 +11,7 @@ EXPR_OPERATORS = {
">": "__gt",
">=": "__gte",
}
-ExpressionOperator = Or(map(Literal, EXPR_OPERATORS.keys()))
+ExpressionOperator = Or(map(Literal, list(EXPR_OPERATORS)))
ExpressionOperator.setParseAction(lambda toks: EXPR_OPERATORS[toks[0]])
diff --git a/gracedb/search/query/events.py b/gracedb/search/query/events.py
index 10c5e5b50..6644bb4ca 100644
--- a/gracedb/search/query/events.py
+++ b/gracedb/search/query/events.py
@@ -41,7 +41,7 @@ gpsQ = Optional(Suppress(Keyword("gpstime:"))) + (gpstime^gpstimeRange)
gpsQ = gpsQ.setParseAction(maybeRange("gpstime"))
# run ids
-runid = Or(map(CaselessLiteral, RUN_MAP.keys())).setName("run id")
+runid = Or(map(CaselessLiteral, list(RUN_MAP))).setName("run id")
#runidList = OneOrMore(runid).setName("run id list")
runQ = (Optional(Suppress(Keyword("runid:"))) + runid)
runQ = runQ.setParseAction(lambda toks: ("gpstime", Q(gpstime__range=
@@ -335,4 +335,4 @@ def parseQuery(s):
if "id" in d and "hid" in d:
d["id"] = d["id"] | d["hid"]
del d["hid"]
- return reduce(Q.__and__, d.values(), Q())
+ return reduce(Q.__and__, list(d.values()), Q())
diff --git a/gracedb/search/query/superevents.py b/gracedb/search/query/superevents.py
index 2ccaffc3a..d2440ef7e 100644
--- a/gracedb/search/query/superevents.py
+++ b/gracedb/search/query/superevents.py
@@ -96,7 +96,7 @@ parameter_dicts = {
'runid': {
'keyword': 'runid',
'keywordOptional': True,
- 'value': Or(map(CaselessLiteral, RUN_MAP.keys())).setName("run id"),
+ 'value': Or(map(CaselessLiteral, list(RUN_MAP))).setName("run id"),
'doRange': False,
'parseAction': lambda toks: ("t_0", Q(t_0__range=RUN_MAP[toks[0]])),
},
diff --git a/gracedb/search/tests/test_access.py b/gracedb/search/tests/test_access.py
index 94625c3bb..25de155a6 100644
--- a/gracedb/search/tests/test_access.py
+++ b/gracedb/search/tests/test_access.py
@@ -182,7 +182,7 @@ class TestEventLatest(EventCreateMixin, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure all events are shown
- self.assertIn('events', response.context.keys())
+ self.assertIn('events', response.context)
for e in Event.objects.all():
self.assertIn(e, response.context['events'])
@@ -193,7 +193,7 @@ class TestEventLatest(EventCreateMixin, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure only exposed events are shown
- self.assertIn('events', response.context.keys())
+ self.assertIn('events', response.context)
self.assertIn(self.lvem_event, response.context['events'])
self.assertEqual(len(response.context['events']), 1)
@@ -206,7 +206,7 @@ class TestEventLatest(EventCreateMixin, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure only exposed events are shown
- self.assertIn('events', response.context.keys())
+ self.assertIn('events', response.context)
self.assertEqual(len(response.context['events']), 0)
@@ -232,7 +232,7 @@ class TestSupereventLatest(SupereventSetup, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure all superevents are shown
- self.assertIn('superevents', response.context.keys())
+ self.assertIn('superevents', response.context)
for s in Superevent.objects.all():
self.assertIn(s, response.context['superevents'])
@@ -243,7 +243,7 @@ class TestSupereventLatest(SupereventSetup, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure only exposed superevents are shown
- self.assertIn('superevents', response.context.keys())
+ self.assertIn('superevents', response.context)
self.assertIn(self.lvem_superevent, response.context['superevents'])
self.assertIn(self.public_superevent, response.context['superevents'])
self.assertEqual(len(response.context['superevents']), 2)
@@ -254,6 +254,6 @@ class TestSupereventLatest(SupereventSetup, GraceDbTestBase):
# Response status
self.assertEqual(response.status_code, 200)
# Make sure only exposed superevents are shown
- self.assertIn('superevents', response.context.keys())
+ self.assertIn('superevents', response.context)
self.assertIn(self.public_superevent, response.context['superevents'])
self.assertEqual(len(response.context['superevents']), 1)
diff --git a/gracedb/superevents/utils.py b/gracedb/superevents/utils.py
index 6422150ce..57a73d836 100644
--- a/gracedb/superevents/utils.py
+++ b/gracedb/superevents/utils.py
@@ -133,7 +133,7 @@ def update_superevent(superevent, updater, add_log_message=True,
new_params = {k: v for k,v in kwargs.items() if k in param_names}
# Get old parameters
- old_params = {k: getattr(superevent, k) for k in new_params.keys()}
+ old_params = {k: getattr(superevent, k) for k in new_params}
# Update superevent object
for k,v in new_params.items():
@@ -145,7 +145,7 @@ def update_superevent(superevent, updater, add_log_message=True,
superevent_alert_kwargs = {}
if add_log_message:
updates = ["{name}: {old} -> {new}".format(name=k, old=old_params[k],
- new=new_params[k]) for k in new_params.keys()
+ new=new_params[k]) for k in new_params
if old_params[k] != new_params[k]]
update_comment = "Updated superevent parameters: {0}".format(
", ".join(updates))
@@ -153,7 +153,7 @@ def update_superevent(superevent, updater, add_log_message=True,
issue_alert=False)
# If preferred event changed, do a few things
- if new_params.has_key('preferred_event') and \
+ if 'preferred_event' in new_params and \
(old_params['preferred_event'] != new_params['preferred_event']):
# Write log for old preferred event
old_msg = ("Removed as preferred event for superevent: "
--
GitLab
From 5236e2f541e22acabfd0fdab53fa280866c919b9 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 11:08:03 -0500
Subject: [PATCH 018/106] python 3: rework __unicode__ -> __str__ methods
---
gracedb/alerts/models.py | 15 +++--
gracedb/events/models.py | 103 +++++++++++++++++++++-------------
gracedb/superevents/models.py | 44 ++++++++++-----
3 files changed, 105 insertions(+), 57 deletions(-)
diff --git a/gracedb/alerts/models.py b/gracedb/alerts/models.py
index a21988b75..ab4bc8858 100644
--- a/gracedb/alerts/models.py
+++ b/gracedb/alerts/models.py
@@ -8,7 +8,8 @@ from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
from django.core.mail import EmailMessage
from django.db import models
-from django.utils import timezone
+from django.utils import six, timezone
+from django.utils.encoding import python_2_unicode_compatible
from django.utils.http import urlencode
from django_twilio.client import twilio_client
@@ -55,7 +56,6 @@ class Contact(CleanSaveModel):
updated = models.DateTimeField(auto_now=True)
verified_time = models.DateTimeField(null=True, blank=True, editable=False)
-
def __str__(self):
return "{0}: {1}".format(self.user.username, self.description)
@@ -168,6 +168,7 @@ class Contact(CleanSaveModel):
###############################################################################
# Notifications ###############################################################
###############################################################################
+@python_2_unicode_compatible
class Notification(models.Model):
# Notification categories
NOTIFICATION_CATEGORY_EVENT = 'E'
@@ -196,10 +197,12 @@ class Notification(models.Model):
pipelines = models.ManyToManyField('events.pipeline', blank=True)
searches = models.ManyToManyField('events.search', blank=True)
- def __unicode__(self):
- return (u"%s: %s") % (
- self.user.username,
- self.display()
+ def __str__(self):
+ return six.text_type(
+ "{username}: {display}".format(
+ username=self.user.username,
+ display=self.display()
+ )
)
def display(self):
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index d1f4fa4a5..c46b9992a 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -4,6 +4,8 @@ import numbers
from django.db import models, IntegrityError
from django.urls import reverse
from django.core.exceptions import ValidationError
+from django.utils import six
+from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from model_utils.managers import InheritanceManager
@@ -51,25 +53,16 @@ SERVER_TZ = pytz.timezone(settings.TIME_ZONE)
#
schema_version = "1.1"
-#class User(models.Model):
- #name = models.CharField(max_length=100)
- #email = models.EmailField()
- #principal = models.CharField(max_length=100)
- #dn = models.CharField(max_length=100)
- #unixid = models.CharField(max_length=25)
-
- #class Meta:
- #ordering = ["name"]
-
- #def __unicode__(self):
- #return self.name
+@python_2_unicode_compatible
class Group(models.Model):
name = models.CharField(max_length=20)
- def __unicode__(self):
- return self.name
+
+ def __str__(self):
+ return six.text_type(self.name)
+@python_2_unicode_compatible
class Pipeline(models.Model):
PIPELINE_TYPE_EXTERNAL = 'E'
PIPELINE_TYPE_OTHER = 'O'
@@ -98,8 +91,8 @@ class Pipeline(models.Model):
('manage_pipeline', 'Can enable or disable pipeline'),
)
- def __unicode__(self):
- return self.name
+ def __str__(self):
+ return six.text_type(self.name)
class PipelineLog(models.Model):
@@ -116,16 +109,20 @@ class PipelineLog(models.Model):
choices=PIPELINE_LOG_ACTION_CHOICES)
+@python_2_unicode_compatible
class Search(models.Model):
name = models.CharField(max_length=100)
description = models.TextField(blank=True)
# XXX Need any additional fields? Like a PI email? Or perhaps even fk?
- def __unicode__(self):
- return self.name
+
+ def __str__(self):
+ return six.text_type(self.name)
+
# Label color will be used in CSS, see
# https://www.w3schools.com/colors/colors_names.asp for
# allowed color choices
+@python_2_unicode_compatible
class Label(models.Model):
name = models.CharField(max_length=20, unique=True)
# XXX really, does this belong here? probably not.
@@ -138,8 +135,8 @@ class Label(models.Model):
# signoffs, for examples.
protected = models.BooleanField(default=False)
- def __unicode__(self):
- return self.name
+ def __str__(self):
+ return six.text_type(self.name)
class ProtectedLabelError(Exception):
# To be raised when an attempt is made to apply or remove a
@@ -154,6 +151,7 @@ class Label(models.Model):
pass
+@python_2_unicode_compatible
class Event(models.Model):
objects = InheritanceManager() # Queries can return subclasses, if available.
@@ -341,8 +339,8 @@ class Event(models.Model):
return e
raise cls.DoesNotExist("Event matching query does not exist")
- def __unicode__(self):
- return self.graceid
+ def __str__(self):
+ return six.text_type(self.graceid)
# Return a list of distinct tags associated with the log messages of this
# event.
@@ -487,6 +485,7 @@ class EventLog(CleanSaveModel, LogBase, AutoIncrementModel):
return None
+@python_2_unicode_compatible
class EMGroup(models.Model):
name = models.CharField(max_length=50, unique=True)
@@ -497,8 +496,8 @@ class EMGroup(models.Model):
# purposes.
#liasons = models.ManyToManyField(UserModel)
- def __unicode__(self):
- return self.name
+ def __str__(self):
+ return six.text_type(self.name)
class EMObservationBase(models.Model):
@@ -566,6 +565,7 @@ class EMObservationBase(models.Model):
self.decWidth = decmax-decmin
+@python_2_unicode_compatible
class EMObservation(EMObservationBase, AutoIncrementModel):
"""EMObservation class for events"""
AUTO_FIELD = 'N'
@@ -575,9 +575,14 @@ class EMObservation(EMObservationBase, AutoIncrementModel):
class Meta(EMObservationBase.Meta):
unique_together = (('event', 'N'),)
- def __unicode__(self):
- return "{event_id} | {group} | {N}".format(
- event_id=self.event.graceid, group=self.group.name, N=self.N)
+ def __str__(self):
+ return six.text_type(
+ "{event_id} | {group} | {N}".format(
+ event_id=self.event.graceid,
+ group=self.group.name,
+ N=self.N
+ )
+ )
def calculateCoveringRegion(self):
footprints = self.emfootprint_set.all()
@@ -625,6 +630,7 @@ class EMFootprint(EMFootprintBase, AutoIncrementModel):
unique_together = (('observation', 'N'),)
+@python_2_unicode_compatible
class Labelling(m2mThroughBase):
"""
Model which provides the "through" relationship between Events and Labels.
@@ -632,9 +638,13 @@ class Labelling(m2mThroughBase):
event = models.ForeignKey(Event)
label = models.ForeignKey(Label)
- def __unicode__(self):
- return "{graceid} | {label}".format(graceid=self.event.graceid,
- label=self.label.name)
+ def __str__(self):
+ return six.text_type(
+ "{graceid} | {label}".format(
+ graceid=self.event.graceid,
+ label=self.label.name
+ )
+ )
## Analysis Specific Attributes.
@@ -910,6 +920,7 @@ class SimInspiralEvent(Event):
return cls._field_names
# Tags (user-defined log message attributes)
+@python_2_unicode_compatible
class Tag(CleanSaveModel):
"""
Model for tags attached to EventLogs.
@@ -931,8 +942,10 @@ class Tag(CleanSaveModel):
])
displayName = models.CharField(max_length=200, null=True, blank=True)
- def __unicode__(self):
- return self.displayName if self.displayName else self.name
+ def __str__(self):
+ return six.text_type(
+ self.displayName if self.displayName else self.name
+ )
class VOEventBase(CleanSaveModel):
@@ -1109,7 +1122,7 @@ class SignoffBase(models.Model):
return 'ADV' + self.opposite_status
-
+@python_2_unicode_compatible
class Signoff(SignoffBase):
"""Class for Event signoffs"""
@@ -1118,9 +1131,15 @@ class Signoff(SignoffBase):
class Meta:
unique_together = ('event', 'instrument')
- def __unicode__(self):
- return "%s | %s | %s" % (self.event.graceid, self.instrument,
- self.status)
+ def __str__(self):
+ return six.text_type(
+ "{gid} | {instrument} | {status}".format(
+ self.event.graceid,
+ self.instrument,
+ self.status
+ )
+ )
+
EMSPECTRUM = (
('em.gamma', 'Gamma rays part of the spectrum'),
@@ -1175,8 +1194,10 @@ EMSPECTRUM = (
('em.radio.20-100MHz', 'Radio between 20 and 100 MHz'),
)
+
# TP (2 Apr 2018): pretty sure this class is deprecated - most recent
# production use is T137114 = April 2015.
+@python_2_unicode_compatible
class EMBBEventLog(AutoIncrementModel):
"""EMBB EventLog: A multi-purpose annotation for EM followup.
@@ -1187,8 +1208,14 @@ class EMBBEventLog(AutoIncrementModel):
ordering = ['-created', '-N']
unique_together = ("event","N")
- def __unicode__(self):
- return "%s-%s-%d" % (self.event.graceid, self.group.name, self.N)
+ def __str__(self):
+ return six.text_type(
+ "{gid}-{name}-{N}".format(
+ self.event.graceid,
+ self.group.name,
+ self.N
+ )
+ )
# A counter for Eels associated with a given event. This is
# important for addressibility.
diff --git a/gracedb/superevents/models.py b/gracedb/superevents/models.py
index b91e3e28f..815603832 100644
--- a/gracedb/superevents/models.py
+++ b/gracedb/superevents/models.py
@@ -17,6 +17,8 @@ from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.db import models, IntegrityError
from django.urls import reverse
+from django.utils import six
+from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase
@@ -38,6 +40,7 @@ SUPEREVENT_DATE_START = datetime.datetime(1980, 1, 1, 0, 0, 0, 0, pytz.utc)
SUPEREVENT_DATE_END = datetime.datetime(2080, 1, 1, 0, 0, 0, 0, pytz.utc)
+@python_2_unicode_compatible
class Superevent(CleanSaveModel, AutoIncrementModel):
"""
Superevent date-based IDs:
@@ -439,8 +442,8 @@ class Superevent(CleanSaveModel, AutoIncrementModel):
raise NotImplemented
#return reverse('')
- def __unicode__(self):
- return self.superevent_id
+ def __str__(self):
+ return six.text_type(self.superevent_id)
class DateIdError(Exception):
# To be raised when the superevent date ID is in a bad format; i.e.,
@@ -508,6 +511,7 @@ class LogUserObjectPermission(UserObjectPermissionBase):
content_object = models.ForeignKey(Log, on_delete=models.CASCADE)
+@python_2_unicode_compatible
class Labelling(m2mThroughBase):
"""
Model which provides the 'through' relationship between Superevents and
@@ -530,11 +534,16 @@ class Labelling(m2mThroughBase):
related_name='%(app_label)s_%(class)s_set',
on_delete=models.CASCADE)
- def __unicode__(self):
- return "{superevent_id} | {label}".format(superevent_id=
- self.superevent.superevent_id, label=self.label.name)
+ def __str__(self):
+ return six.text_type(
+ "{superevent_id} | {label}".format(
+ superevent_id=self.superevent.superevent_id,
+ label=self.label.name
+ )
+ )
+@python_2_unicode_compatible
class Signoff(CleanSaveModel, SignoffBase):
"""Class for superevent signoffs"""
superevent = models.ForeignKey(Superevent, null=False,
@@ -550,10 +559,14 @@ class Signoff(CleanSaveModel, SignoffBase):
('do_adv_signoff', 'Can interact with advocate signoffs'),
)
- def __unicode__(self):
- return "{superevent_id} | {instrument} | {status}".format(
- superevent_id=self.superevent.superevent_id,
- instrument=self.instrument, status=self.status)
+ def __str__(self):
+ return six.text_type(
+ "{superevent_id} | {instrument} | {status}".format(
+ superevent_id=self.superevent.superevent_id,
+ instrument=self.instrument,
+ status=self.status
+ )
+ )
class VOEvent(VOEventBase, AutoIncrementModel):
@@ -571,6 +584,7 @@ class VOEvent(VOEventBase, AutoIncrementModel):
super(Log, self).fileurl()
+@python_2_unicode_compatible
class EMObservation(CleanSaveModel, EMObservationBase, AutoIncrementModel):
"""EMObservation class for superevents"""
AUTO_FIELD = 'N'
@@ -581,10 +595,14 @@ class EMObservation(CleanSaveModel, EMObservationBase, AutoIncrementModel):
class Meta(EMObservationBase.Meta):
unique_together = (('superevent', 'N'),)
- def __unicode__(self):
- return "{superevent_id} | {group} | {N}".format(
- superevent_id=self.superevent.superevent_id,
- group=self.group.name, N=self.N)
+ def __str__(self):
+ return six.text_type(
+ "{superevent_id} | {group} | {N}".format(
+ superevent_id=self.superevent.superevent_id,
+ group=self.group.name,
+ N=self.N
+ )
+ )
def calculateCoveringRegion(self):
footprints = self.emfootprint_set.all()
--
GitLab
From 6a99c94b47c26d8932b1d6ee4593d124867eaacd Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 11:47:21 -0500
Subject: [PATCH 019/106] Python 3: remove use of unicode()
---
gracedb/api/v1/events/views.py | 2 +-
gracedb/events/templatetags/logtags.py | 3 ---
gracedb/events/templatetags/scientific.py | 5 +++--
.../commands/update_user_accounts_from_ligo_ldap.py | 7 ++++---
4 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index 58fc41311..9b7f81633 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -1281,7 +1281,7 @@ class EventLogTagDetail(InheritPermissionsAPIView):
# client that the creation was sucessful when, in fact, the database
# was unchanged.
tag = eventlog.tags.filter(name=tagname)[0]
- msg = "Log already has tag %s" % unicode(tag)
+ msg = "Log already has tag {0}".format(tag.name)
return Response(msg,status=status.HTTP_409_CONFLICT)
except:
# Check authorization
diff --git a/gracedb/events/templatetags/logtags.py b/gracedb/events/templatetags/logtags.py
index 0e38cfac6..d07c6188e 100644
--- a/gracedb/events/templatetags/logtags.py
+++ b/gracedb/events/templatetags/logtags.py
@@ -19,9 +19,6 @@ def getLogsForTag(event,name=None):
# In either case, we want the template to just ignore it.
return None
-@register.filter("tagUnicode")
-def tagUnicode(tag):
- return unicode(tag);
@register.filter("logsForTagHaveImage")
def logsForTagAllHaveImages(event,name=None):
diff --git a/gracedb/events/templatetags/scientific.py b/gracedb/events/templatetags/scientific.py
index 0c3df5d80..ce1281520 100644
--- a/gracedb/events/templatetags/scientific.py
+++ b/gracedb/events/templatetags/scientific.py
@@ -1,3 +1,4 @@
+from builtins import str
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
from django import template
from django.utils.encoding import force_text
@@ -61,7 +62,7 @@ def scientificformat(text):
elif ( (d > Decimal('-100') and d < Decimal('-0.1')) or ( d > Decimal('0.1') and d < Decimal('100')) ) :
# this is what the original floatformat() function does
sign, digits, exponent = d.quantize(Decimal('.01'), ROUND_HALF_UP).as_tuple()
- digits = [unicode(digit) for digit in reversed(digits)]
+ digits = [str(digit) for digit in reversed(digits)]
while len(digits) <= abs(exponent):
digits.append(u'0')
digits.insert(-exponent, u'.')
@@ -72,7 +73,7 @@ def scientificformat(text):
# for very small and very large numbers
sign, digits, exponent = d.as_tuple()
exponent = d.adjusted()
- digits = [unicode(digit) for digit in digits][:3] # limit to 2 decimal places
+ digits = [str(digit) for digit in digits][:3] # limit to 2 decimal places
while len(digits) < 3:
digits.append(u'0')
digits.insert(1, u'.')
diff --git a/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py b/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py
index 38fd05eda..2330e71a5 100644
--- a/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py
+++ b/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py
@@ -1,3 +1,4 @@
+from builtins import str
import datetime
import ldap
@@ -33,8 +34,8 @@ class LdapPersonResultProcessor(object):
if self.ldap_connection is None:
raise RuntimeError('LDAP connection not configured')
self.user_data = {
- 'first_name': unicode(self.ldap_result['givenName'][0], 'utf-8'),
- 'last_name': unicode(self.ldap_result['sn'][0], 'utf-8'),
+ 'first_name': str(self.ldap_result['givenName'][0], 'utf-8'),
+ 'last_name': str(self.ldap_result['sn'][0], 'utf-8'),
'email': self.ldap_result['mail'][0],
'is_active': self.ldap_connection.lvc_group.ldap_name in
self.ldap_result.get('isMemberOf', []),
@@ -239,7 +240,7 @@ class LdapRobotResultProcessor(LdapPersonResultProcessor):
if self.ldap_connection is None:
raise RuntimeError('LDAP connection not configured')
self.user_data = {
- 'last_name': unicode(self.ldap_result['x-LIGO-TWikiName'][0],
+ 'last_name': str(self.ldap_result['x-LIGO-TWikiName'][0],
'utf-8'),
'email': self.ldap_result['mail'][0],
'is_active': self.ldap_connection.groups.get(
--
GitLab
From 8a0785da49d996cf631e9181b1a174846d0299fd Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 12:32:30 -0500
Subject: [PATCH 020/106] Python 3: remove unneeded import of 'exceptions'
module
---
gracedb/api/v1/events/views.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index 9b7f81633..1321a43ef 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -1,5 +1,4 @@
from __future__ import absolute_import
-import exceptions
import json
import logging
import os
--
GitLab
From 28e4d210e65c6895acc0792d54a8c9ff2aa28816 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 12:32:51 -0500
Subject: [PATCH 021/106] Python 3: fix lambda syntax
---
gracedb/superevents/views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/superevents/views.py b/gracedb/superevents/views.py
index 0c8a45f64..7b7f8ac31 100644
--- a/gracedb/superevents/views.py
+++ b/gracedb/superevents/views.py
@@ -233,7 +233,7 @@ class SupereventPublic(DisplayFarMixin, ListView):
("BBH", voe.prob_bbh),
("Terrestrial", voe.prob_terrestrial),
("MassGap", voe.prob_mass_gap)]
- pastro_values.sort(reverse=True, key=lambda (a,b): b)
+ pastro_values.sort(reverse=True, key=lambda a, b: b)
sourcelist = []
for key, value in pastro_values:
if value > 0.01:
--
GitLab
From 4fadf3a9795b5699d2cfed282489256755fd512d Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 13:47:56 -0500
Subject: [PATCH 022/106] requirements: fix pytest < 5 for python2/3
compatibility
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index d8dea6d06..346c828a6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,6 +40,7 @@ twilio==6.10.3
# problematic API changes in 2.3.1 and beyond that will need to be handled
# appropriately.
pyparsing==2.3.0
+pytest<5
pytest-cov==2.6.1
pytest-django==3.4.8
pytz==2018.9
--
GitLab
From a1372b5f667ca365d9bcbefc3ae3ee28716225b9 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 13:48:23 -0500
Subject: [PATCH 023/106] Python 3: fix all StringIO usage
---
gracedb/api/v1/events/views.py | 4 ++--
gracedb/core/middleware/profiling.py | 4 ++--
.../migrations/0036_populate_voevent_fields.py | 7 +++++--
gracedb/events/models.py | 12 ++++++------
gracedb/events/reports.py | 4 ++--
gracedb/events/translator.py | 4 ++--
.../migrations/0004_populate_voevent_fields.py | 7 +++++--
gracedb/superevents/models.py | 6 +++---
8 files changed, 27 insertions(+), 21 deletions(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index 1321a43ef..eebfa6a28 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -4,9 +4,9 @@ import logging
import os
import shutil
try:
- from io import StringIO
-except ImportError: # python < 3
from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
from django.conf import settings
from django.contrib.auth.models import User, Permission, Group as DjangoGroup
diff --git a/gracedb/core/middleware/profiling.py b/gracedb/core/middleware/profiling.py
index 6e6b0254d..b78257d8c 100644
--- a/gracedb/core/middleware/profiling.py
+++ b/gracedb/core/middleware/profiling.py
@@ -7,9 +7,9 @@ import os
import re
import tempfile
try:
- from io import StringIO
-except ImportError: # python < 3
from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
import hotshot, hotshot.stats
diff --git a/gracedb/events/migrations/0036_populate_voevent_fields.py b/gracedb/events/migrations/0036_populate_voevent_fields.py
index f5da68f22..32ac1bacb 100644
--- a/gracedb/events/migrations/0036_populate_voevent_fields.py
+++ b/gracedb/events/migrations/0036_populate_voevent_fields.py
@@ -1,7 +1,10 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.20 on 2019-06-10 16:58
from __future__ import unicode_literals
-from cStringIO import StringIO
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
from hashlib import sha1
import os
from lxml import etree
@@ -66,7 +69,7 @@ def get_datadir(event_or_superevent):
hash_input = str(event_or_superevent.id)
if event_or_superevent.__class__.__name__.lower() == 'superevent':
hash_input = 'superevent' + hash_input
- hdf = StringIO(sha1(hash_input).hexdigest())
+ hdf = StringIO(sha1(hash_input.encode()).hexdigest())
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index c46b9992a..05f6ed93d 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -35,7 +35,10 @@ from django.conf import settings
import pytz
import calendar
-from io import StringIO
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
from hashlib import sha1
import shutil
@@ -241,11 +244,8 @@ class Event(models.Model):
@property
def datadir(self):
# Create a file-like object which is the SHA-1 hexdigest of the Event's primary key
- hid = sha1(str(self.id).encode("utf-8")).hexdigest()
- try:
- hdf = StringIO(hid.decode("utf-8"))
- except AttributeError: # python < 3
- hdf = StringIO(hid)
+ hid = sha1(str(self.id).encode()).hexdigest()
+ hdf = StringIO(hid)
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
diff --git a/gracedb/events/reports.py b/gracedb/events/reports.py
index b6d70420e..01b824b41 100644
--- a/gracedb/events/reports.py
+++ b/gracedb/events/reports.py
@@ -30,9 +30,9 @@ from django.utils import timezone
import pytz
import json
try:
- from io import StringIO
-except ImportError: # python < 3
from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
@internal_user_required
def histo(request):
diff --git a/gracedb/events/translator.py b/gracedb/events/translator.py
index 0877df9c2..93105fb74 100644
--- a/gracedb/events/translator.py
+++ b/gracedb/events/translator.py
@@ -19,9 +19,9 @@ from core.vfile import VersionedFile
import json
try:
- from io import StringIO
-except ImportError: # python < 3
from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
from math import sqrt
diff --git a/gracedb/superevents/migrations/0004_populate_voevent_fields.py b/gracedb/superevents/migrations/0004_populate_voevent_fields.py
index a693ff835..2adc34c7f 100644
--- a/gracedb/superevents/migrations/0004_populate_voevent_fields.py
+++ b/gracedb/superevents/migrations/0004_populate_voevent_fields.py
@@ -1,7 +1,10 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.20 on 2019-06-10 16:58
from __future__ import unicode_literals
-from cStringIO import StringIO
+try:
+ from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
from hashlib import sha1
import os
from lxml import etree
@@ -62,7 +65,7 @@ def get_datadir(event_or_superevent):
hash_input = str(event_or_superevent.id)
if event_or_superevent.__class__.__name__.lower() == 'superevent':
hash_input = 'superevent' + hash_input
- hdf = StringIO(sha1(hash_input).hexdigest())
+ hdf = StringIO(sha1(hash_input.encode()).hexdigest())
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
diff --git a/gracedb/superevents/models.py b/gracedb/superevents/models.py
index 815603832..11b0c9125 100644
--- a/gracedb/superevents/models.py
+++ b/gracedb/superevents/models.py
@@ -1,7 +1,7 @@
try:
- from io import StringIO
-except ImportError: # python < 3
from StringIO import StringIO
+except ImportError: # python >= 3
+ from io import StringIO
import datetime
from hashlib import sha1
import logging
@@ -365,7 +365,7 @@ class Superevent(CleanSaveModel, AutoIncrementModel):
# object's primary key. We prepend 'superevent' so as to not
# have collisions with Event files
hash_input = 'superevent' + str(self.id)
- hdf = StringIO(sha1(hash_input).hexdigest().decode('utf-8'))
+ hdf = StringIO(sha1(hash_input.encode()).hexdigest())
# Build up the nodes of the directory structure
nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS]
--
GitLab
From 5d70a3cdb0c91ebad776f44f000db642a9321dce Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 13:51:43 -0500
Subject: [PATCH 024/106] Import 'reduce' for Python 3
---
gracedb/alerts/recipients.py | 5 +++++
gracedb/api/tests/utils.py | 4 ++++
gracedb/search/query/events.py | 4 ++++
3 files changed, 13 insertions(+)
diff --git a/gracedb/alerts/recipients.py b/gracedb/alerts/recipients.py
index 43f9356fe..18f6b7081 100644
--- a/gracedb/alerts/recipients.py
+++ b/gracedb/alerts/recipients.py
@@ -1,3 +1,8 @@
+try:
+ from functools import reduce
+except ImportError: # python < 3
+ pass
+
from django.conf import settings
from django.db.models import Q
diff --git a/gracedb/api/tests/utils.py b/gracedb/api/tests/utils.py
index 872a333e0..014902207 100644
--- a/gracedb/api/tests/utils.py
+++ b/gracedb/api/tests/utils.py
@@ -1,4 +1,8 @@
from copy import deepcopy
+try:
+ from functools import reduce
+except ImportError: # python < 3
+ pass
try:
from unittest import mock
except ImportError: # python < 3
diff --git a/gracedb/search/query/events.py b/gracedb/search/query/events.py
index 6644bb4ca..0db5f7b44 100644
--- a/gracedb/search/query/events.py
+++ b/gracedb/search/query/events.py
@@ -15,6 +15,10 @@ from pyparsing import Word, nums, Literal, CaselessLiteral, delimitedList, \
oneOf, stringStart, stringEnd, FollowedBy, ParseResults, ParseException, \
CaselessKeyword
import pytz
+try:
+ from functools import reduce
+except ImportError: # python < 3
+ pass
from django.db.models import Q
from django.db.models.query import QuerySet
--
GitLab
From 15fb3678ce79ab10d26adc0ed4bc399f46ee597b Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 18 Jul 2019 14:25:41 -0500
Subject: [PATCH 025/106] Python 3: cast outputs of map() to lists
---
gracedb/api/v1/events/views.py | 2 +-
gracedb/api/v1/superevents/serializers.py | 2 +-
gracedb/api/v1/superevents/tests/test_access.py | 4 ++--
gracedb/core/models.py | 2 +-
gracedb/core/time_utils.py | 4 ++--
gracedb/events/nltime.py | 8 ++++----
gracedb/events/tests/test_access.py | 4 ++--
gracedb/events/view_logic.py | 14 +++++++-------
gracedb/search/constants.py | 2 +-
gracedb/search/query/events.py | 10 +++++-----
gracedb/search/query/superevents.py | 5 +++--
11 files changed, 29 insertions(+), 28 deletions(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index eebfa6a28..4fcc1bb85 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -755,7 +755,7 @@ class EventNeighbors(InheritPermissionsAPIView):
if delta.find(',') < 0:
neighborhood = (int(delta), int(delta))
else:
- neighborhood = map(int, delta.split(','))
+ neighborhood = list(map(int, delta.split(',')))
except ValueError:
pass
else:
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index 8009b7911..29b852e37 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -783,7 +783,7 @@ class SupereventEMObservationSerializer(serializers.ModelSerializer):
list_length = len(ra_list)
all_lists = (ra_list, dec_list, ra_width_list, dec_width_list,
start_time_list, duration_list)
- if not all(map(lambda l: len(l) == list_length, all_lists)):
+ if not all(list(map(lambda l: len(l) == list_length, all_lists))):
self.fail('list_lengths')
return data
diff --git a/gracedb/api/v1/superevents/tests/test_access.py b/gracedb/api/v1/superevents/tests/test_access.py
index 85c49b920..b0b6ecd48 100644
--- a/gracedb/api/v1/superevents/tests/test_access.py
+++ b/gracedb/api/v1/superevents/tests/test_access.py
@@ -2683,8 +2683,8 @@ class TestSupereventEMObservationList(SupereventSetup, GraceDbApiTestBase):
# Define EMObservation data for POST-ing
cls.emgroup_name = 'fake_emgroup'
now = datetime.datetime.now()
- start_time_list = map(lambda i:
- (now + datetime.timedelta(seconds=i)).isoformat(), [0, 1, 2, 3])
+ start_time_list = list(map(lambda i:
+ (now + datetime.timedelta(seconds=i)).isoformat(), [0, 1, 2, 3]))
cls.emobservation_data = {
'group': cls.emgroup_name,
'ra_list': [1, 2, 3, 4],
diff --git a/gracedb/core/models.py b/gracedb/core/models.py
index b3f36fb00..7d346f658 100644
--- a/gracedb/core/models.py
+++ b/gracedb/core/models.py
@@ -115,7 +115,7 @@ class AutoIncrementModel(models.Model):
qn = compiler.quote_name_unless_alias
# Compile multiple constraints with AND
- constraint_fields = map(meta.get_field, self.AUTO_CONSTRAINTS)
+ constraint_fields = list(map(meta.get_field, self.AUTO_CONSTRAINTS))
constraint_list = ["{0}=%s".format(qn(f.column))
for f in constraint_fields]
constraint_values = [f.get_db_prep_value(getattr(self, f.column),
diff --git a/gracedb/core/time_utils.py b/gracedb/core/time_utils.py
index 6c1f93f5f..713ec8f1d 100644
--- a/gracedb/core/time_utils.py
+++ b/gracedb/core/time_utils.py
@@ -14,7 +14,7 @@ import calendar
gpsEpoch = calendar.timegm((1980, 1, 6, 0, 0, 0, 0, 0, 0))
-leapSeconds = map(calendar.timegm, [
+leapSeconds = list(map(calendar.timegm, [
(1981, 7, 0, 0, 0, 0, 0, 0, 0),
(1982, 7, 0, 0, 0, 0, 0, 0, 0),
(1983, 7, 0, 0, 0, 0, 0, 0, 0),
@@ -33,7 +33,7 @@ leapSeconds = map(calendar.timegm, [
(2012, 7, 0, 0, 0, 0, 0, 0, 0),
(2015, 7, 0, 0, 0, 0, 0, 0, 0),
(2017, 1, 0, 0, 0, 0, 0, 0, 0),
-])
+]))
def gpsToPosixTime(gpsTime):
if gpsTime is None:
diff --git a/gracedb/events/nltime.py b/gracedb/events/nltime.py
index fc2eaf59b..f039d0e89 100755
--- a/gracedb/events/nltime.py
+++ b/gracedb/events/nltime.py
@@ -91,11 +91,11 @@ def calculateTime(toks):
# grammar definitions
CL = CaselessLiteral
-today, tomorrow, yesterday, noon, midnight, now = map( CL,
- "today tomorrow yesterday noon midnight now".split())
+today, tomorrow, yesterday, noon, midnight, now = list(map( CL,
+ "today tomorrow yesterday noon midnight now".split()))
plural = lambda s : Combine(CL(s) + Optional(CL("s")))
-month, week, day, hour, minute, second = map( plural,
- "month week day hour minute second".split())
+month, week, day, hour, minute, second = list(map(plural,
+ "month week day hour minute second".split()))
am = CL("am")
pm = CL("pm")
COLON = Suppress(':')
diff --git a/gracedb/events/tests/test_access.py b/gracedb/events/tests/test_access.py
index 8e724a6a8..d9f1c6758 100644
--- a/gracedb/events/tests/test_access.py
+++ b/gracedb/events/tests/test_access.py
@@ -917,8 +917,8 @@ class TestEventCreateEMObservation(EventSetup, GraceDbTestBase):
# Define EMObservation data for POST-ing
cls.emgroup_name = 'fake_emgroup'
now = datetime.datetime.now()
- start_time_list = map(lambda i:
- (now + datetime.timedelta(seconds=i)).isoformat(), [0, 1, 2, 3])
+ start_time_list = list(map(lambda i:
+ (now + datetime.timedelta(seconds=i)).isoformat(), [0, 1, 2, 3]))
cls.emobservation_data = {
'group': cls.emgroup_name,
'ra_list': [1, 2, 3, 4],
diff --git a/gracedb/events/view_logic.py b/gracedb/events/view_logic.py
index 8e54e94a4..1ecaf9247 100644
--- a/gracedb/events/view_logic.py
+++ b/gracedb/events/view_logic.py
@@ -496,17 +496,17 @@ def create_emobservation(request, event):
# Handle case where comma-separated strings are submitted rather than lists
if isinstance(raList, six.string_types):
- raList = map(lambda x: float(x.strip()), raList.split(','))
+ raList = list(map(lambda x: float(x.strip()), raList.split(',')))
if isinstance(raWidthList, six.string_types):
- raWidthList = map(lambda x: float(x.strip()), raWidthList.split(','))
+ raWidthList = list(map(lambda x: float(x.strip()), raWidthList.split(',')))
if isinstance(decList, six.string_types):
- decList = map(lambda x: float(x.strip()), decList.split(','))
+ decList = list(map(lambda x: float(x.strip()), decList.split(',')))
if isinstance(decWidthList, six.string_types):
- decWidthList = map(lambda x: float(x.strip()), decWidthList.split(','))
+ decWidthList = list(map(lambda x: float(x.strip()), decWidthList.split(',')))
if isinstance(startTimeList, six.string_types):
- startTimeList = map(lambda x: x.strip(), startTimeList.split(','))
+ startTimeList = list(map(lambda x: x.strip(), startTimeList.split(',')))
if isinstance(durationList, six.string_types):
- durationList = map(lambda x: int(x.strip()), durationList.split(','))
+ durationList = list(map(lambda x: int(x.strip()), durationList.split(',')))
all_lists = (raList, raWidthList, decList, decWidthList, startTimeList,
durationList)
@@ -516,7 +516,7 @@ def create_emobservation(request, event):
# Check all list lengths
list_length = len(all_lists[0])
- if not all(map(lambda l: len(l) == list_length, all_lists)):
+ if not all(list(map(lambda l: len(l) == list_length, all_lists))):
raise ValueError('ra_list, dec_list, ra_width_list, dec_width_list, '
'start_time_list, and duration_list must be the same length.')
diff --git a/gracedb/search/constants.py b/gracedb/search/constants.py
index 5e0c74c96..d6d77fe35 100644
--- a/gracedb/search/constants.py
+++ b/gracedb/search/constants.py
@@ -11,7 +11,7 @@ EXPR_OPERATORS = {
">": "__gt",
">=": "__gte",
}
-ExpressionOperator = Or(map(Literal, list(EXPR_OPERATORS)))
+ExpressionOperator = Or(list(map(Literal, list(EXPR_OPERATORS))))
ExpressionOperator.setParseAction(lambda toks: EXPR_OPERATORS[toks[0]])
diff --git a/gracedb/search/query/events.py b/gracedb/search/query/events.py
index 0db5f7b44..602b60151 100644
--- a/gracedb/search/query/events.py
+++ b/gracedb/search/query/events.py
@@ -45,7 +45,7 @@ gpsQ = Optional(Suppress(Keyword("gpstime:"))) + (gpstime^gpstimeRange)
gpsQ = gpsQ.setParseAction(maybeRange("gpstime"))
# run ids
-runid = Or(map(CaselessLiteral, list(RUN_MAP))).setName("run id")
+runid = Or(list(map(CaselessLiteral, list(RUN_MAP)))).setName("run id")
#runidList = OneOrMore(runid).setName("run id list")
runQ = (Optional(Suppress(Keyword("runid:"))) + runid)
runQ = runQ.setParseAction(lambda toks: ("gpstime", Q(gpstime__range=
@@ -98,7 +98,7 @@ submitterQ = submitterQ.setParseAction(lambda toks: ("submitter", toks[0]))
nltimeRange = nltime + Suppress("..") + nltime
def doTime(tok):
- x = datetime.datetime(*(map(int, tok)))
+ x = datetime.datetime(*(list(map(int, tok))))
return pytz.utc.localize(x)
dash = Suppress('-')
@@ -257,7 +257,7 @@ def parseQuery(s):
# XXX Querying the database at module compile time is a bad idea!
# See: https://docs.djangoproject.com/en/1.8/topics/testing/overview/
groupNames = list(Group.objects.values_list('name', flat=True))
- group = Or(map(CaselessLiteral, groupNames)).setName("analysis group name")
+ group = Or(list(map(CaselessLiteral, groupNames))).setName("analysis group name")
#groupList = delimitedList(group, delim='|').setName("analysis group list")
groupList = OneOrMore(group).setName("analysis group list")
groupQ = (Optional(Suppress(Keyword("group:"))) + groupList)
@@ -266,7 +266,7 @@ def parseQuery(s):
# Pipeline
pipelineNames = list(Pipeline.objects.values_list('name', flat=True))
- pipeline = Or(map(CaselessLiteral, pipelineNames)).setName("pipeline name")
+ pipeline = Or(list(map(CaselessLiteral, pipelineNames))).setName("pipeline name")
pipelineList = OneOrMore(pipeline).setName("pipeline list")
pipelineQ = (Optional(Suppress(Keyword("pipeline:"))) + pipelineList)
pipelineQ = pipelineQ.setParseAction(lambda toks: ("pipeline",
@@ -274,7 +274,7 @@ def parseQuery(s):
# Search
searchNames = list(Search.objects.values_list('name', flat=True))
- search = Or(map(CaselessLiteral, searchNames)).setName("search name")
+ search = Or(list(map(CaselessLiteral, searchNames))).setName("search name")
# XXX Branson: The change below was made 2/17/15 to fix a bug in which
# searches like 'grbevent.ra > 0' failed due to the 'grb' being peeled off
# and assumed to be part of a 'Search' query. So we don't consume a token
diff --git a/gracedb/search/query/superevents.py b/gracedb/search/query/superevents.py
index d2440ef7e..909f68554 100644
--- a/gracedb/search/query/superevents.py
+++ b/gracedb/search/query/superevents.py
@@ -96,7 +96,8 @@ parameter_dicts = {
'runid': {
'keyword': 'runid',
'keywordOptional': True,
- 'value': Or(map(CaselessLiteral, list(RUN_MAP))).setName("run id"),
+ 'value': Or(list(map(CaselessLiteral, list(RUN_MAP)))).setName(
+ "run id"),
'doRange': False,
'parseAction': lambda toks: ("t_0", Q(t_0__range=RUN_MAP[toks[0]])),
},
@@ -194,7 +195,7 @@ parameter_dicts = {
Word(nums, exact=2) + Optional(Suppress(':') + \
Word(nums, exact=2)))).setParseAction(lambda toks:
pytz.timezone(settings.TIME_ZONE).localize(
- datetime.datetime(*map(int, toks)))),
+ datetime.datetime(*list(map(int, toks))))),
'parseAction': maybeRange("created"),
},
# test OR category: test
--
GitLab
From bcdcfa323f84d6709fd9a245db61fde27e8ec9cd Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 19 Jul 2019 08:02:54 -0500
Subject: [PATCH 026/106] Python 3: remove unnecessary use of long()
---
gracedb/core/utils.py | 2 +-
gracedb/events/templatetags/timeutil.py | 8 ++++----
gracedb/events/views.py | 4 ++--
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/gracedb/core/utils.py b/gracedb/core/utils.py
index 40fe4e781..0c629ea0e 100644
--- a/gracedb/core/utils.py
+++ b/gracedb/core/utils.py
@@ -19,7 +19,7 @@ def int_to_letters(num, positive_only=True):
"""
# Argument checking
- if not isinstance(num, (int, long)):
+ if not isinstance(num, int):
# Coerce to int
logger.warning('Coercing argument of type {0} to int'.format(
type(num)))
diff --git a/gracedb/events/templatetags/timeutil.py b/gracedb/events/templatetags/timeutil.py
index dc72a5c95..95fdf8b62 100644
--- a/gracedb/events/templatetags/timeutil.py
+++ b/gracedb/events/templatetags/timeutil.py
@@ -53,7 +53,7 @@ def get_multitime_value(t, label, autoescape, format):
dt = dt.astimezone(SERVER_TZ)
posix_time = time.mktime(dt.timetuple())
gps_time = int(posixToGpsTime(posix_time))
- elif isinstance(t, int) or isinstance(t, long):
+ elif isinstance(t, int):
gps_time = t
dt = gpsToUtc(t)
# Note: must convert to server timezone before calling mktime
@@ -66,7 +66,7 @@ def get_multitime_value(t, label, autoescape, format):
return "N/A"
return '<time utc="%s" gps="%s" llo="%s" lho="%s" virgo="%s" jsparsable="%s"%s>%s</time>' % \
8*("N/A",)
- # raise ValueError("time must be type int, long or datetime, not '%s'" % type(t))
+ # raise ValueError("time must be type int or datetime, not '%s'" % type(t))
# JavaScript -- parsable by Date() object constructor
# "Jan 2, 1985 00:00:00 UTC"
@@ -167,7 +167,7 @@ def timeSelections(t):
dt = dt.astimezone(SERVER_TZ)
posix_time = time.mktime(dt.timetuple())
gps_time = int(posixToGpsTime(posix_time))
- elif isinstance(t, int) or isinstance(t, long):
+ elif isinstance(t, int):
gps_time = t
dt = gpsToUtc(t)
posix_time = time.mktime(dt.astimezone(SERVER_TZ).timetuple())
@@ -176,7 +176,7 @@ def timeSelections(t):
dt = gpsToUtc(t)
posix_time = time.mktime(dt.astimezone(SERVER_TZ).timetuple())
else:
- raise ValueError("time must be type int, long or datetime, not '%s'" % type(t))
+ raise ValueError("time must be type int or datetime, not '%s'" % type(t))
# JavaScript -- parsable by Date() object constructor
# "Jan 2, 1985 00:00:00 UTC"
diff --git a/gracedb/events/views.py b/gracedb/events/views.py
index 7a981ddef..c79206a1b 100644
--- a/gracedb/events/views.py
+++ b/gracedb/events/views.py
@@ -323,13 +323,13 @@ def logentry(request, event, num=None):
def neighbors(request, event, delta1, delta2=None):
context = {}
try:
- delta1 = long(delta1)
+ delta1 = int(delta1)
if delta2 is None:
delta2 = delta1
delta1 = -delta1
else:
- delta2 = long(delta2)
+ delta2 = int(delta2)
except ValueError: pass
except: pass
--
GitLab
From 19efbbcfbcacabb33f8b161f387192ed1b922935 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 19 Jul 2019 08:03:20 -0500
Subject: [PATCH 027/106] Python 3: fix b64encode usage in tests
---
gracedb/api/tests/test_authentication.py | 26 +++++++++-----
gracedb/api/tests/test_backends.py | 44 ++++++++++++++++--------
2 files changed, 48 insertions(+), 22 deletions(-)
diff --git a/gracedb/api/tests/test_authentication.py b/gracedb/api/tests/test_authentication.py
index a43591594..ecb04cf80 100644
--- a/gracedb/api/tests/test_authentication.py
+++ b/gracedb/api/tests/test_authentication.py
@@ -34,9 +34,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can authenticate to API with correct password"""
# Set up and make request
url = api_reverse('api:root')
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
headers = {
'HTTP_AUTHORIZATION': 'Basic {0}'.format(user_and_pass),
}
@@ -56,8 +59,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can't authenticate with wrong password"""
# Set up and make request
url = api_reverse('api:root')
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password='b4d')).decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password='b4d'
+ ).encode()
+ ).decode("ascii")
headers = {
'HTTP_AUTHORIZATION': 'Basic {0}'.format(user_and_pass),
}
@@ -76,9 +83,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
# Set up and make request
url = api_reverse('api:root')
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
headers = {
'HTTP_AUTHORIZATION': 'Basic {0}'.format(user_and_pass),
}
diff --git a/gracedb/api/tests/test_backends.py b/gracedb/api/tests/test_backends.py
index e0caec2b4..7c451a6d1 100644
--- a/gracedb/api/tests/test_backends.py
+++ b/gracedb/api/tests/test_backends.py
@@ -45,9 +45,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can authenticate to API with correct password"""
# Set up request
request = self.factory.get(api_reverse('api:root'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Authentication attempt
@@ -60,8 +63,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can't authenticate with wrong password"""
# Set up request
request = self.factory.get(api_reverse('api:root'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password='b4d')).decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password='b4d'
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Authentication attempt should fail
@@ -77,9 +84,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
# Set up request
request = self.factory.get(api_reverse('api:root'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Authentication attempt should fail
@@ -91,9 +101,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
"""User can't authenticate to a non-API URL path"""
# Set up request
request = self.factory.get(reverse('home'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Try to authenticate
@@ -108,9 +121,12 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
# Set up request
request = self.factory.get(api_reverse('api:root'))
- user_and_pass = b64encode(b"{username}:{password}".format(
- username=self.lvem_user.username, password=self.password)) \
- .decode("ascii")
+ user_and_pass = b64encode(
+ "{username}:{password}".format(
+ username=self.lvem_user.username,
+ password=self.password
+ ).encode()
+ ).decode("ascii")
request.META['HTTP_AUTHORIZATION'] = 'Basic {0}'.format(user_and_pass)
# Authentication attempt should fail
--
GitLab
From 2fa7eb10920ed2147417f70dc61172ae22572687 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 19 Jul 2019 08:04:46 -0500
Subject: [PATCH 028/106] Python 3: fix response content checks in unit tests
---
gracedb/api/tests/test_authentication.py | 16 +-
gracedb/api/tests/test_throttling.py | 3 +-
gracedb/api/v1/events/tests/test_access.py | 169 +++++---
gracedb/api/v1/main/tests/test_access.py | 16 +-
.../api/v1/superevents/tests/test_access.py | 403 +++++++++++-------
gracedb/events/tests/test_access.py | 91 ++--
6 files changed, 443 insertions(+), 255 deletions(-)
diff --git a/gracedb/api/tests/test_authentication.py b/gracedb/api/tests/test_authentication.py
index ecb04cf80..8d650fc3f 100644
--- a/gracedb/api/tests/test_authentication.py
+++ b/gracedb/api/tests/test_authentication.py
@@ -71,8 +71,8 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
response = self.client.get(url, data=None, **headers)
# Check response
- self.assertEqual(response.status_code, 403)
- self.assertIn('Invalid username/password', response.content)
+ self.assertContains(response, 'Invalid username/password',
+ status_code=403)
def test_user_authenticate_to_api_with_expired_password(self):
"""User can't authenticate with expired password"""
@@ -95,8 +95,8 @@ class TestGraceDbBasicAuthentication(GraceDbApiTestBase):
response = self.client.get(url, data=None, **headers)
# Check response
- self.assertEqual(response.status_code, 403)
- self.assertIn('Your password has expired', response.content)
+ self.assertContains(response, 'Your password has expired',
+ status_code=403)
class TestGraceDbX509Authentication(GraceDbApiTestBase):
@@ -152,8 +152,8 @@ class TestGraceDbX509Authentication(GraceDbApiTestBase):
response = self.client.get(url, data=None, **headers)
# Check response
- self.assertEqual(response.status_code, 401)
- self.assertIn("Invalid certificate subject", response.content)
+ self.assertContains(response, 'Invalid certificate subject',
+ status_code=401)
def test_inactive_user_authenticate(self):
"""Inactive user can't authenticate"""
@@ -169,8 +169,8 @@ class TestGraceDbX509Authentication(GraceDbApiTestBase):
response = self.client.get(url, data=None, **headers)
# Check response
- self.assertEqual(response.status_code, 401)
- self.assertIn("User inactive or deleted", response.content)
+ self.assertContains(response, 'User inactive or deleted',
+ status_code=401)
def test_authenticate_cert_with_proxy(self):
"""User can authenticate to API with proxied X509 certificate"""
diff --git a/gracedb/api/tests/test_throttling.py b/gracedb/api/tests/test_throttling.py
index f917f12bf..9c8323f8d 100644
--- a/gracedb/api/tests/test_throttling.py
+++ b/gracedb/api/tests/test_throttling.py
@@ -27,5 +27,4 @@ class TestThrottling(GraceDbApiTestBase):
# Second response should get throttled
response = self.request_as_user(url, "GET")
- self.assertEqual(response.status_code, 429)
- self.assertIn('Request was throttled', response.content)
+ self.assertContains(response, 'Request was throttled', status_code=429)
diff --git a/gracedb/api/v1/events/tests/test_access.py b/gracedb/api/v1/events/tests/test_access.py
index 1cd2b54c0..fb25c523c 100644
--- a/gracedb/api/v1/events/tests/test_access.py
+++ b/gracedb/api/v1/events/tests/test_access.py
@@ -22,9 +22,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_detail(self):
"""Unauthenticated user can't access event detail"""
@@ -32,9 +34,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_log_list(self):
"""Unauthenticated user can't access event log list"""
@@ -42,9 +46,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_log_detail(self):
"""Unauthenticated user can't access event log detail"""
@@ -52,9 +58,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_voevent_list(self):
"""Unauthenticated user can't access event VOEvent list"""
@@ -62,9 +70,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_voevent_detail(self):
"""Unauthenticated user can't access event VOEvent detail"""
@@ -72,9 +82,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_embbeventlog_list(self):
"""Unauthenticated user can't access event EMBBEventLog list"""
@@ -82,9 +94,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_embbeventlog_detail(self):
"""Unauthenticated user can't access event EMBBEventLog detail"""
@@ -92,9 +106,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_emobservation_list(self):
"""Unauthenticated user can't access event EMObservation list"""
@@ -102,9 +118,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "POST"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_emobservation_detail(self):
"""Unauthenticated user can't access event EMObservation detail"""
@@ -112,9 +130,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_tag_list(self):
"""Unauthenticated user can't access event tag list"""
@@ -122,9 +142,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_tag_detail(self):
"""Unauthenticated user can't access event tag detail"""
@@ -132,9 +154,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_log_tag_list(self):
"""Unauthenticated user can't access event log tag list"""
@@ -142,9 +166,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_log_tag_detail(self):
"""Unauthenticated user can't access event log tag detail"""
@@ -153,9 +179,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT", "DELETE"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_permission_list(self):
"""Unauthenticated user can't access event permission list"""
@@ -163,9 +191,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_group_permission_list(self):
"""Unauthenticated user can't access event group permission list"""
@@ -174,9 +204,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_group_permission_detail(self):
"""Unauthenticated user can't access event group permission list"""
@@ -185,9 +217,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT", "DELETE"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_files(self):
"""Unauthenticated user can't access event files (list or detail)"""
@@ -195,9 +229,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_labels(self):
"""Unauthenticated user can't access event labels (list or detail)"""
@@ -205,9 +241,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET", "PUT", "DELETE"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_neighbors(self):
"""Unauthenticated user can't access event neighbors list"""
@@ -215,9 +253,11 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_event_signoff_list(self):
"""Unauthenticated user can't access event signoff list"""
@@ -225,7 +265,8 @@ class TestPublicAccess(EventSetup, GraceDbApiTestBase):
methods = ["GET"]
for http_method in methods:
response = self.request_as_user(url, http_method)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
-
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
diff --git a/gracedb/api/v1/main/tests/test_access.py b/gracedb/api/v1/main/tests/test_access.py
index 19854c438..4d94ed1a7 100644
--- a/gracedb/api/v1/main/tests/test_access.py
+++ b/gracedb/api/v1/main/tests/test_access.py
@@ -31,14 +31,18 @@ class TestPublicAccess(GraceDbApiTestBase):
"""Unauthenticated user can't access performance info"""
url = v_reverse('performance-info')
response = self.request_as_user(url, "GET")
- self.assertEqual(response.status_code, 403)
- self.assertIn("Authentication credentials were not provided",
- response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials were not provided',
+ status_code=403
+ )
def test_lvem_user_performance_info(self):
"""LV-EM user can't access performance info"""
url = v_reverse('performance-info')
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn("Forbidden", response.content)
-
+ self.assertContains(
+ response,
+ 'Forbidden',
+ status_code=403
+ )
diff --git a/gracedb/api/v1/superevents/tests/test_access.py b/gracedb/api/v1/superevents/tests/test_access.py
index b0b6ecd48..e608840f2 100644
--- a/gracedb/api/v1/superevents/tests/test_access.py
+++ b/gracedb/api/v1/superevents/tests/test_access.py
@@ -503,9 +503,11 @@ class TestSupereventConfirmAsGw(SupereventManagersGroupAndUserSetup,
assign_perm('superevents.view_superevent', self.lvem_obs_group,
obj=self.production_superevent)
response = self.request_as_user(url, "POST", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to confirm superevents as GWs',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to confirm superevents as GWs',
+ status_code=403
+ )
def test_lvem_user_confirm_mdc_superevent(self):
"""LV-EM user can't confirm mdc superevent as GW"""
@@ -520,9 +522,11 @@ class TestSupereventConfirmAsGw(SupereventManagersGroupAndUserSetup,
assign_perm('superevents.view_superevent', self.lvem_obs_group,
obj=self.mdc_superevent)
response = self.request_as_user(url, "POST", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to confirm MDC superevents as GWs',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to confirm MDC superevents as GWs',
+ status_code=403
+ )
def test_lvem_user_confirm_test_superevent(self):
"""LV-EM user can't confirm test superevent as GW"""
@@ -537,9 +541,11 @@ class TestSupereventConfirmAsGw(SupereventManagersGroupAndUserSetup,
assign_perm('superevents.view_superevent', self.lvem_obs_group,
obj=self.test_superevent)
response = self.request_as_user(url, "POST", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to confirm test superevents as GWs',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to confirm test superevents as GWs',
+ status_code=403
+ )
def test_public_user_confirm_superevents(self):
"""Public user can't confirm any superevents as GWs"""
@@ -551,16 +557,22 @@ class TestSupereventConfirmAsGw(SupereventManagersGroupAndUserSetup,
url = v_reverse('superevents:superevent-confirm-as-gw',
args=[s.superevent_id])
response = self.request_as_user(url, "POST")
- self.assertEqual(response.status_code, 403)
- self.assertIn('Authentication credentials', response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials',
+ status_code=403
+ )
# Expose it, should still get same 403 since the permission
# checking process still fails at the outset
assign_perm('superevents.view_superevent', self.public_group,
obj=self.test_superevent)
response = self.request_as_user(url, "POST")
- self.assertEqual(response.status_code, 403)
- self.assertIn('Authentication credentials', response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials',
+ status_code=403
+ )
class TestSupereventLabelList(SupereventSetup, GraceDbApiTestBase):
@@ -738,8 +750,11 @@ class TestSupereventLabelList(SupereventSetup, GraceDbApiTestBase):
args=[self.public_superevent.superevent_id])
data = {'name': label.name}
response = self.request_as_user(url, "POST", data=data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('Authentication credentials', response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials',
+ status_code=403
+ )
class TestSupereventLabelDetail(SupereventSetup, GraceDbApiTestBase):
@@ -873,9 +888,11 @@ class TestSupereventLabelDetail(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-label-detail',
args=[self.public_superevent.superevent_id, l.name])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('You do not have permission to remove labels',
- response.content)
+ self.assertContains(
+ response,
+ 'You do not have permission to remove labels',
+ status_code=403
+ )
def test_public_user_delete_label(self):
"""Public user cannot remove labels from any superevents"""
@@ -905,8 +922,11 @@ class TestSupereventLabelDetail(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-label-detail',
args=[self.public_superevent.superevent_id, l.name])
response = self.request_as_user(url, "DELETE")
- self.assertEqual(response.status_code, 403)
- self.assertIn('Authentication credentials', response.content)
+ self.assertContains(
+ response,
+ 'Authentication credentials',
+ status_code=403
+ )
class TestSupereventEventList(SupereventManagersGroupAndUserSetup,
@@ -1128,9 +1148,11 @@ class TestSupereventEventList(SupereventManagersGroupAndUserSetup,
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.lvem_user,
data={'event': ev.graceid})
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to add events to superevents',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to add events to superevents',
+ status_code=403
+ )
def test_public_user_add_event_to_superevent(self):
"""Public user can't add events to superevents"""
@@ -1151,8 +1173,11 @@ class TestSupereventEventList(SupereventManagersGroupAndUserSetup,
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST",
data={'event': ev.graceid})
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventEventDetail(SupereventManagersGroupAndUserSetup,
@@ -1351,9 +1376,11 @@ class TestSupereventEventDetail(SupereventManagersGroupAndUserSetup,
args=[self.lvem_superevent.superevent_id,
self.event2.graceid])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to remove events from superevents',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to remove events from superevents',
+ status_code=403
+ )
def test_public_user_remove_event_from_superevent(self):
"""
@@ -1378,8 +1405,11 @@ class TestSupereventEventDetail(SupereventManagersGroupAndUserSetup,
args=[self.public_superevent.superevent_id,
self.event2.graceid])
response = self.request_as_user(url, "DELETE")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventLogList(AccessManagersGroupAndUserSetup,
@@ -1547,10 +1577,11 @@ class TestSupereventLogList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data=log_data)
# Check response and data
- self.assertEqual(response.status_code, 403)
- # Make sure correct 403 error is provided
- self.assertIn('not allowed to expose superevent log messages to LV-EM',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevent log messages to LV-EM',
+ status_code=403
+ )
def test_internal_user_create_log_with_public_tag(self):
"""Basic internal user can't create logs with public access tag"""
@@ -1567,10 +1598,11 @@ class TestSupereventLogList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data=log_data)
# Check response and data
- self.assertEqual(response.status_code, 403)
- # Make sure correct 403 error is provided
- self.assertIn(('not allowed to expose superevent log messages to the '
- 'public'), response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevent log messages to the public',
+ status_code=403
+ )
def test_access_manager_create_log_with_lvem_tag(self):
"""Access manager user can create logs with external access tag"""
@@ -1672,9 +1704,11 @@ class TestSupereventLogList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.lvem_user,
data=log_data)
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('You are not allowed to post log messages with tags',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to post log messages with tags',
+ status_code=403
+ )
def test_public_user_create_log(self):
"""Public user can't create any logs"""
@@ -1699,8 +1733,11 @@ class TestSupereventLogList(AccessManagersGroupAndUserSetup,
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", data=log_data)
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventLogDetail(SupereventSetup, GraceDbApiTestBase):
@@ -2015,10 +2052,11 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data={'name': settings.EXTERNAL_ACCESS_TAGNAME})
# Check response and data
- self.assertEqual(response.status_code, 403)
- # Make sure correct 403 error is provided
- self.assertIn('not allowed to expose superevent log messages to LV-EM',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevent log messages to LV-EM',
+ status_code=403
+ )
def test_internal_user_tag_log_with_public(self):
"""Basic internal user add public access tag"""
@@ -2035,10 +2073,11 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data={'name': settings.PUBLIC_ACCESS_TAGNAME})
# Check response and data
- self.assertEqual(response.status_code, 403)
- # Make sure correct 403 error is provided
- self.assertIn(('not allowed to expose superevent log messages to the '
- 'public'), response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevent log messages to the public',
+ status_code=403
+ )
def test_access_manager_tag_log_with_lvem(self):
"""Access manager user can tag logs with external access tag"""
@@ -2076,8 +2115,11 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST", self.am_user,
data={'name': settings.PUBLIC_ACCESS_TAGNAME})
# Check response and data
- self.assertEqual(response.status_code, 201)
- self.assertIn(response.data['name'], settings.PUBLIC_ACCESS_TAGNAME)
+ self.assertContains(
+ response,
+ settings.PUBLIC_ACCESS_TAGNAME,
+ status_code=201
+ )
def test_lvem_user_tag_log(self):
"""LV-EM user can't tag logs for any superevents"""
@@ -2147,8 +2189,11 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
response = self.request_as_user(url, "POST",
data={'name': self.tag1.name})
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventLogTagDetail(AccessManagersGroupAndUserSetup,
@@ -2398,8 +2443,11 @@ class TestSupereventLogTagDetail(AccessManagersGroupAndUserSetup,
args=[self.public_superevent.superevent_id,
self.log_dict['public_public'].N, self.tag1.name])
response = self.request_as_user(url, "DELETE")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventVOEventList(SupereventSetup, GraceDbApiTestBase):
@@ -2544,9 +2592,11 @@ class TestSupereventVOEventList(SupereventSetup, GraceDbApiTestBase):
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.lvem_user,
data=self.voevent_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to create VOEvents',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to create VOEvents',
+ status_code=403
+ )
def test_public_user_create_voevent_for_hidden_superevent(self):
"""Public user can't create VOEvents for hidden superevents"""
@@ -2560,9 +2610,11 @@ class TestSupereventVOEventList(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-voevent-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", data=self.voevent_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventVOEventDetail(SupereventSetup, GraceDbApiTestBase):
@@ -2844,8 +2896,11 @@ class TestSupereventEMObservationList(SupereventSetup, GraceDbApiTestBase):
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST",
data=self.emobservation_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventEMObservationDetail(SupereventSetup, GraceDbApiTestBase):
@@ -3307,18 +3362,22 @@ class TestSupereventGroupObjectPermissionList(SupereventSetup,
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "GET", self.lvem_user)
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to view superevent permissions',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to view superevent permissions',
+ status_code=403
+ )
# Public superevent
url = v_reverse('superevents:superevent-permission-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "GET", self.lvem_user)
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to view superevent permissions',
- response.content)
+ self.assertContains(
+ response,
+ 'not allowed to view superevent permissions',
+ status_code=403
+ )
def test_public_user_get_permissions(self):
"""Public user can't get permission list"""
@@ -3341,8 +3400,11 @@ class TestSupereventGroupObjectPermissionList(SupereventSetup,
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "GET")
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventGroupObjectPermissionModify(SupereventSetup,
@@ -3356,9 +3418,11 @@ class TestSupereventGroupObjectPermissionModify(SupereventSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data={'action': 'expose'})
# Check response
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to expose superevents',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to expose superevents',
+ status_code=403
+ )
def test_internal_user_hide_exposed_superevent(self):
"""Internal user can't modify permissions to hide superevent"""
@@ -3367,9 +3431,11 @@ class TestSupereventGroupObjectPermissionModify(SupereventSetup,
response = self.request_as_user(url, "POST", self.internal_user,
data={'action': 'hide'})
# Check response
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to hide superevents',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'not allowed to hide superevents',
+ status_code=403
+ )
def test_access_manager_expose_internal_superevent(self):
"""Access manager can modify permissions to expose superevent"""
@@ -3410,8 +3476,11 @@ class TestSupereventGroupObjectPermissionModify(SupereventSetup,
response = self.request_as_user(url, "POST", self.lvem_user,
data={'action': 'hide'})
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('not allowed to hide superevent', response.content)
+ self.assertContains(
+ response,
+ 'not allowed to hide superevents',
+ status_code=403
+ )
def test_public_user_modify_permissions(self):
"""Public user can't modify permissions"""
@@ -3427,8 +3496,11 @@ class TestSupereventGroupObjectPermissionModify(SupereventSetup,
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", data={'action': 'hide'})
# Check response and data
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffList(SupereventSetup, GraceDbApiTestBase):
@@ -3477,17 +3549,21 @@ class TestSupereventSignoffList(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-signoff-list',
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to view superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to view superevent signoffs',
+ status_code=403
+ )
# Public superevent
url = v_reverse('superevents:superevent-signoff-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to view superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to view superevent signoffs',
+ status_code=403
+ )
def test_public_get_hidden_signoff_list(self):
"""Public user can't view list of signoffs for hidden superevent"""
@@ -3509,9 +3585,11 @@ class TestSupereventSignoffList(SupereventSetup, GraceDbApiTestBase):
url = v_reverse('superevents:superevent-signoff-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "GET")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffDetail(SupereventSetup, GraceDbApiTestBase):
@@ -3570,9 +3648,11 @@ class TestSupereventSignoffDetail(SupereventSetup, GraceDbApiTestBase):
args=[self.lvem_superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to view superevent signoffs',
- response.content)
+ self.assertContains(
+ response,
+ 'do not have permission to view superevent signoffs',
+ status_code=403
+ )
# Public superevent
signoff = self.public_superevent.signoff_set.first()
@@ -3580,9 +3660,11 @@ class TestSupereventSignoffDetail(SupereventSetup, GraceDbApiTestBase):
args=[self.public_superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "GET", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to view superevent signoffs',
- response.content)
+ self.assertContains(
+ response,
+ 'do not have permission to view superevent signoffs',
+ status_code=403
+ )
def test_public_user_get_signoff_detail_for_hidden_superevent(self):
"""Public user can't view signoff detail for hidden superevent"""
@@ -3609,8 +3691,11 @@ class TestSupereventSignoffDetail(SupereventSetup, GraceDbApiTestBase):
args=[self.public_superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "GET")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.content)
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
@@ -3657,9 +3742,11 @@ class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
args=[self.internal_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.internal_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to create superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to create superevent signoffs',
+ status_code=403
+ )
def test_H1_control_room_create_H1_signoff(self):
"""H1 control room user can create H1 signoffs"""
@@ -3679,9 +3766,11 @@ class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
args=[self.internal_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.H1_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to do L1 signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to do L1 signoffs',
+ status_code=403
+ )
def test_advocate_create_adv_signoff(self):
"""EM advocate user can create advocate signoffs"""
@@ -3711,18 +3800,22 @@ class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
args=[self.lvem_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.lvem_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to create superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to create superevent signoffs',
+ status_code=403
+ )
# Public superevent
url = v_reverse('superevents:superevent-signoff-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", self.lvem_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to create superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to create superevent signoffs',
+ status_code=403
+ )
def test_public_user_create_signoff_for_hidden_superevent(self):
"""Public user can't create signoffs for hidden superevents"""
@@ -3743,9 +3836,11 @@ class TestSupereventSignoffCreation(SignoffGroupsAndUsersSetup,
url = v_reverse('superevents:superevent-signoff-list',
args=[self.public_superevent.superevent_id])
response = self.request_as_user(url, "POST", data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
@@ -3813,9 +3908,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", self.internal_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to change superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to change superevent signoffs',
+ status_code=403
+ )
def test_H1_control_room_update_H1_signoff(self):
"""H1 control room user can update H1 signoffs"""
@@ -3842,9 +3939,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", self.H1_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to do L1 signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to do L1 signoffs',
+ status_code=403
+ )
def test_advocate_update_adv_signoff(self):
"""EM advocate user can update advocate signoffs"""
@@ -3881,9 +3980,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", self.lvem_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to change superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to change superevent signoffs',
+ status_code=403
+ )
# Public superevent
signoff = self.public_signoff
@@ -3892,9 +3993,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", self.lvem_user,
data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to change superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to change superevent signoffs',
+ status_code=403
+ )
def test_public_user_update_signoff_for_hidden_superevent(self):
"""Public user can't update signoffs for hidden superevents"""
@@ -3921,8 +4024,11 @@ class TestSupereventSignoffUpdate(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "PATCH", data=self.signoff_data)
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
@@ -3979,9 +4085,11 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[self.internal_superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE", self.internal_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to delete superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to delete superevent signoffs',
+ status_code=403
+ )
def test_H1_control_room_delete_H1_signoff(self):
"""H1 control room user can delete H1 signoffs"""
@@ -4004,9 +4112,11 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE", self.H1_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to do L1 signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to do L1 signoffs',
+ status_code=403
+ )
def test_advocate_delete_adv_signoff(self):
"""EM advocate user can delete advocate signoffs"""
@@ -4038,9 +4148,11 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to delete superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to delete superevent signoffs',
+ status_code=403
+ )
# Public superevent
signoff = self.public_signoff
@@ -4048,9 +4160,11 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertIn('do not have permission to delete superevent signoffs',
- response.data['detail'])
+ self.assertContains(
+ response,
+ 'do not have permission to delete superevent signoffs',
+ status_code=403
+ )
def test_public_user_delete_signoff_for_hidden_superevent(self):
"""Public user can't delete signoffs for hidden superevents"""
@@ -4077,5 +4191,8 @@ class TestSupereventSignoffDeletion(SignoffGroupsAndUsersSetup,
args=[signoff.superevent.superevent_id,
signoff.signoff_type + signoff.instrument])
response = self.request_as_user(url, "DELETE")
- self.assertEqual(response.status_code, 403)
- self.assertIn('credentials were not provided', response.data['detail'])
+ self.assertContains(
+ response,
+ 'credentials were not provided',
+ status_code=403
+ )
diff --git a/gracedb/events/tests/test_access.py b/gracedb/events/tests/test_access.py
index d9f1c6758..1fce19db1 100644
--- a/gracedb/events/tests/test_access.py
+++ b/gracedb/events/tests/test_access.py
@@ -389,9 +389,11 @@ class TestEventCreationPage(GraceDbTestBase):
"""Basic internal user can't create production events"""
response = self.request_as_user(self.url, "POST", self.internal_user,
data=self.event_data)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content,
- "You do not have permission to submit events to this pipeline.")
+ self.assertContains(
+ response,
+ 'You do not have permission to submit events to this pipeline',
+ status_code=403
+ )
def test_basic_internal_user_create_test_event(self):
"""Basic internal user can create test events"""
@@ -490,9 +492,11 @@ class TestEventModifyPermissions(EventSetup, GraceDbTestBase):
url = reverse('modify_permissions', args=[self.internal_event.graceid])
response = self.request_as_user(url, "POST", self.internal_user,
data=self.perm_data)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content,
- "You aren't authorized to create permission objects.")
+ self.assertContains(
+ response,
+ "You aren't authorized to create permission objects",
+ status_code=403
+ )
def test_privileged_internal_user_modify_permissions(self):
"""Privileged internal user can modify event permissions"""
@@ -521,9 +525,11 @@ class TestEventModifyPermissions(EventSetup, GraceDbTestBase):
url = reverse('modify_permissions', args=[self.internal_event.graceid])
response = self.request_as_user(url, "POST", self.internal_user,
data=remove_data)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content,
- "You aren't authorized to delete permission objects.")
+ self.assertContains(
+ response,
+ "You aren't authorized to delete permission objects",
+ status_code=403
+ )
# Give remove permission to user and try again - should succeed
p_remove.user_set.add(self.internal_user)
@@ -548,9 +554,11 @@ class TestEventModifyPermissions(EventSetup, GraceDbTestBase):
url = reverse('modify_permissions', args=[self.lvem_event.graceid])
response = self.request_as_user(url, "POST", self.lvem_user,
data=self.perm_data)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content,
- "You aren't authorized to create permission objects.")
+ self.assertContains(
+ response,
+ "You aren't authorized to create permission objects",
+ status_code=403
+ )
def test_public_user_modify_permissions(self):
"""Public user can't modify event permissions"""
@@ -584,9 +592,11 @@ class TestEventModifySignoff(SignoffGroupsAndUsersSetup, EventSetup,
url = reverse('modify_signoff', args=[self.internal_event.graceid])
response = self.request_as_user(url, "POST", self.internal_user,
self.op_signoff_data)
- self.assertEqual(response.status_code, 400)
- self.assertEqual(response.content,
- "Unknown instrument/control room for signoff.")
+ self.assertContains(
+ response,
+ 'Unknown instrument/control room for signoff.',
+ status_code=400
+ )
def test_H1_control_room_modify_signoff(self):
"""H1 control room can create H1 operator signoff for event"""
@@ -624,9 +634,11 @@ class TestEventModifySignoff(SignoffGroupsAndUsersSetup, EventSetup,
url = reverse('modify_signoff', args=[self.lvem_event.graceid])
response = self.request_as_user(url, "POST", self.lvem_user,
self.op_signoff_data)
- self.assertEqual(response.status_code, 400)
- self.assertEqual(response.content,
- "Unknown instrument/control room for signoff.")
+ self.assertContains(
+ response,
+ 'Unknown instrument/control room for signoff.',
+ status_code=400
+ )
def test_public_user_modify_signoff(self):
"""Public user can't modify event signoffs"""
@@ -773,8 +785,11 @@ class TestEventLogTag(EventSetup, GraceDbTestBase):
url = reverse('taglogentry', args=[self.lvem_event.graceid,
log.N, self.tag_name])
response = self.request_as_user(url, "POST", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content, 'Forbidden')
+ self.assertContains(
+ response,
+ 'Forbidden',
+ status_code=403
+ )
# Exposed log on exposed event
log = self.lvem_event.eventlog_set.get(
@@ -843,10 +858,13 @@ class TestEventLogUntag(EventSetup, GraceDbTestBase):
response = self.request_as_user(url, "DELETE", self.internal_user)
# Returns a 200 with a message on success
- self.assertEqual(response.status_code, 200)
- self.assertEqual(response.content,
- "Removed tag {tag} for message {N}.".format(tag=self.test_tag.name,
- N=log.N))
+ self.assertContains(
+ response,
+ "Removed tag {tag} for message {N}.".format(
+ tag=self.test_tag.name, N=log.N
+ ),
+ status_code=200
+ )
def test_lvem_user_untag_log_hidden_event(self):
"""LV-EM user can't untag event logs for hidden events"""
@@ -878,8 +896,11 @@ class TestEventLogUntag(EventSetup, GraceDbTestBase):
url = reverse('taglogentry', args=[self.lvem_event.graceid,
log.N, self.test_tag.name])
response = self.request_as_user(url, "DELETE", self.lvem_user)
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content, 'Forbidden')
+ self.assertContains(
+ response,
+ 'Forbidden',
+ status_code=403
+ )
# Exposed log on exposed event
log = self.lvem_event.eventlog_set.get(
@@ -888,10 +909,13 @@ class TestEventLogUntag(EventSetup, GraceDbTestBase):
log.N, self.test_tag.name])
response = self.request_as_user(url, "DELETE", self.lvem_user)
# Test response
- self.assertEqual(response.status_code, 200)
- self.assertEqual(response.content,
- "Removed tag {tag} for message {N}.".format(tag=self.test_tag.name,
- N=log.N))
+ self.assertContains(
+ response,
+ "Removed tag {tag} for message {N}.".format(
+ tag=self.test_tag.name, N=log.N
+ ),
+ status_code=200
+ )
# Test tags on event log
self.assertEqual(log.tags.count(), 1)
self.assertIn(self.lvem_tag, log.tags.all())
@@ -990,5 +1014,8 @@ class TestEventCreateEMObservation(EventSetup, GraceDbTestBase):
for log in ev.eventlog_set.all():
url = reverse('emobservation_entry', args=[ev.graceid, ""])
response = self.request_as_user(url, "POST")
- self.assertEqual(response.status_code, 403)
- self.assertEqual(response.content, 'Forbidden')
+ self.assertContains(
+ response,
+ 'Forbidden',
+ status_code=403
+ )
--
GitLab
From 2218851b9d1274be302d511cdc3931c4b91f1bc4 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 19 Jul 2019 09:10:06 -0500
Subject: [PATCH 029/106] Python 3: remove use of assertItemsEqual in unit
tests
---
gracedb/api/v1/superevents/tests/test_access.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/gracedb/api/v1/superevents/tests/test_access.py b/gracedb/api/v1/superevents/tests/test_access.py
index e608840f2..7105f3bf1 100644
--- a/gracedb/api/v1/superevents/tests/test_access.py
+++ b/gracedb/api/v1/superevents/tests/test_access.py
@@ -1927,8 +1927,10 @@ class TestSupereventLogTagList(AccessManagersGroupAndUserSetup,
self.assertEqual(response.status_code, 200)
# Lists of log tags from
log_tags = [t.name for t in l.tags.all()]
- self.assertItemsEqual(
- [t['name'] for t in response.data['tags']], log_tags)
+ self.assertEqual(
+ sorted([t['name'] for t in response.data['tags']]),
+ sorted(log_tags)
+ )
def test_lvem_user_get_log_tag_list_for_hidden_superevent(self):
"""LV-EM user can't get tags for any logs on hidden superevent"""
@@ -2795,7 +2797,7 @@ class TestSupereventEMObservationList(SupereventSetup, GraceDbApiTestBase):
api_emo_nums = [emo['N'] for emo in response.data['observations']]
db_emo_nums = [emo.N for emo in
self.public_superevent.emobservation_set.all()]
- self.assertItemsEqual(api_emo_nums, db_emo_nums)
+ self.assertEqual(sorted(api_emo_nums), sorted(db_emo_nums))
def test_public_user_get_list_for_hidden_superevent(self):
"""Public user can't get any EMObservations for hidden superevents"""
@@ -2823,7 +2825,7 @@ class TestSupereventEMObservationList(SupereventSetup, GraceDbApiTestBase):
api_emo_nums = [emo['N'] for emo in response.data['observations']]
db_emo_nums = [emo.N for emo in
self.public_superevent.emobservation_set.all()]
- self.assertItemsEqual(api_emo_nums, db_emo_nums)
+ self.assertEqual(sorted(api_emo_nums), sorted(db_emo_nums))
def test_internal_user_create_emobservation(self):
"""Internal user can create EMObservations for all superevents"""
--
GitLab
From 6db0c2c7733aa680067297aad9fc6e14c26490d4 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 19 Jul 2019 09:30:56 -0500
Subject: [PATCH 030/106] Python 3: fix upload file tests
---
gracedb/api/v1/superevents/tests/test_access.py | 8 ++++----
gracedb/events/tests/test_access.py | 8 ++++----
gracedb/superevents/tests/test_access.py | 4 ++--
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/gracedb/api/v1/superevents/tests/test_access.py b/gracedb/api/v1/superevents/tests/test_access.py
index 7105f3bf1..958e9e2e1 100644
--- a/gracedb/api/v1/superevents/tests/test_access.py
+++ b/gracedb/api/v1/superevents/tests/test_access.py
@@ -3002,8 +3002,8 @@ class TestSupereventFileList(SupereventSetup, GraceDbApiTestBase):
super(TestSupereventFileList, cls).setUpTestData()
# Create files for internal superevent
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_superevent, filename=cls.file1['filename'],
@@ -3145,8 +3145,8 @@ class TestSupereventFileDetail(SupereventSetup, GraceDbApiTestBase):
super(TestSupereventFileDetail, cls).setUpTestData()
# Create files for internal superevent
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_superevent, filename=cls.file1['filename'],
diff --git a/gracedb/events/tests/test_access.py b/gracedb/events/tests/test_access.py
index 1fce19db1..0703958e3 100644
--- a/gracedb/events/tests/test_access.py
+++ b/gracedb/events/tests/test_access.py
@@ -157,8 +157,8 @@ class TestEventFileListView(ExposeLogMixin, EventSetup, GraceDbTestBase):
super(TestEventFileListView, cls).setUpTestData()
# Create files for internal and exposed events
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_event, filename=cls.file1['filename'],
@@ -254,8 +254,8 @@ class TestEventFileDownloadView(ExposeLogMixin, EventSetup, GraceDbTestBase):
super(TestEventFileDownloadView, cls).setUpTestData()
# Create files for internal and exposed events
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_event, filename=cls.file1['filename'],
diff --git a/gracedb/superevents/tests/test_access.py b/gracedb/superevents/tests/test_access.py
index ddbdca090..be35272af 100644
--- a/gracedb/superevents/tests/test_access.py
+++ b/gracedb/superevents/tests/test_access.py
@@ -199,8 +199,8 @@ class TestSupereventFileListView(SupereventSetup, GraceDbTestBase):
super(TestSupereventFileListView, cls).setUpTestData()
# Create files for internal and exposed superevents
- cls.file1 = {'filename': 'file1.txt', 'content': 'test content 1'}
- cls.file2 = {'filename': 'file2.txt', 'content': 'test content 2'}
+ cls.file1 = {'filename': 'file1.txt', 'content': b'test content 1'}
+ cls.file2 = {'filename': 'file2.txt', 'content': b'test content 2'}
for i in range(4):
log1 = create_log(cls.internal_user, 'upload file1',
cls.internal_superevent, filename=cls.file1['filename'],
--
GitLab
From 2ca28a5f92f169ff6fc9bbfb9f69f50f1f39c39e Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 19 Jul 2019 09:31:16 -0500
Subject: [PATCH 031/106] core: add handling for bytes file content in
VersionedFiles
---
gracedb/core/vfile.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/gracedb/core/vfile.py b/gracedb/core/vfile.py
index 5a8786722..523daf9d6 100644
--- a/gracedb/core/vfile.py
+++ b/gracedb/core/vfile.py
@@ -255,11 +255,12 @@ def create_versioned_file(filename, file_dir, file_contents):
full_path = os.path.join(file_dir, filename)
# Create file
- fdest = VersionedFile(full_path, 'w')
if isinstance(file_contents, six.string_types):
+ fdest = VersionedFile(full_path, 'w')
fdest.write(file_contents)
elif isinstance(file_contents, (UploadedFile, InMemoryUploadedFile,
- TemporaryUploadedFile, SimpleUploadedFile)):
+ TemporaryUploadedFile, SimpleUploadedFile, bytes)):
+ fdest = VersionedFile(full_path, 'wb')
for chunk in file_contents.chunks():
fdest.write(chunk)
fdest.close()
--
GitLab
From 7675bff41128a562718398e25ec7035d40e2e40d Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 19 Jul 2019 10:15:38 -0500
Subject: [PATCH 032/106] Python 3: remove use of basestring
---
gracedb/events/nltime.py | 2 +-
gracedb/events/translator.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/gracedb/events/nltime.py b/gracedb/events/nltime.py
index f039d0e89..8515d1e97 100755
--- a/gracedb/events/nltime.py
+++ b/gracedb/events/nltime.py
@@ -58,7 +58,7 @@ def convertToAbsTime(toks):
else:
day = pytz.utc.localize(datetime(now.year, now.month, now.day))
if "timeOfDay" in toks:
- if isinstance(toks.timeOfDay,basestring):
+ if isinstance(toks.timeOfDay, str):
timeOfDay = {
"now" : timedelta(0, (now.hour*60+now.minute)*60+now.second, now.microsecond),
"noon" : timedelta(0,0,0,0,0,12),
diff --git a/gracedb/events/translator.py b/gracedb/events/translator.py
index 93105fb74..3ebd08d49 100644
--- a/gracedb/events/translator.py
+++ b/gracedb/events/translator.py
@@ -47,7 +47,7 @@ def cleanData(val, field_name, table_name='events_event'):
return maxval
else:
return val
- elif isinstance(val, basestring):
+ elif isinstance(val, str):
raise ValueError("Unrecognized string in the %s column" % field_name)
else:
raise ValueError("Unrecognized value in column %s" % field_name)
--
GitLab
From 6f8eb63f455ea1ae12d6646267281d63c719b1cf Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 8 Aug 2019 10:05:22 -0500
Subject: [PATCH 033/106] requirements: install VOEventLib from PyPI
For Python 2, we install VOEventLib from a git fork since upstream
is not Python 2-compatible. But we don't need to do that for Python 3!
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index 346c828a6..e60ce6eec 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -35,6 +35,7 @@ service_identity==17.0.0
simplejson==3.15.0
Sphinx==1.7.0
twilio==6.10.3
+voeventlib==1.2
# Do NOT upgrade pyparsing beyond 2.3.0 (even to 2.3.1) without carefully
# testing and modifying the search query code. There were a number of
# problematic API changes in 2.3.1 and beyond that will need to be handled
--
GitLab
From f1015549bd72b1eea9772a5b251b353bcec65908 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 13 Sep 2019 15:27:13 -0500
Subject: [PATCH 034/106] fix usage of VersionedFile through events application
---
gracedb/api/v1/events/views.py | 52 ++++++++--------------------------
gracedb/core/vfile.py | 4 ++-
gracedb/events/serialize.py | 25 ++++++++--------
gracedb/events/translator.py | 17 +++--------
gracedb/events/view_logic.py | 7 ++---
gracedb/events/views.py | 13 ++-------
6 files changed, 37 insertions(+), 81 deletions(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index 4fcc1bb85..95da954e4 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -40,7 +40,7 @@ from alerts.issuers.events import EventAlertIssuer, EventLogAlertIssuer, \
EventVOEventAlertIssuer, EventPermissionsAlertIssuer
from api.throttling import BurstAnonRateThrottle
from core.http import check_and_serve_file
-from core.vfile import VersionedFile
+from core.vfile import create_versioned_file
from events.buildVOEvent import buildVOEvent, VOEventBuilderException
from events.forms import CreateEventForm
from events.models import Event, Group, Search, Pipeline, EventLog, Tag, \
@@ -610,29 +610,14 @@ class EventDetail(InheritPermissionsAPIView):
# return Response("\n".join(messages),
# status=status.HTTP_400_BAD_REQUEST)
- # XXX handle duplicate file names.
+ # Create versioned file
f = request.data['eventFile']
- uploadDestination = os.path.join(event.datadir, f.name)
- fdest = VersionedFile(uploadDestination, 'w')
- #for chunk in f.chunks():
- # fdest.write(chunk)
- #fdest.close()
- shutil.copyfileobj(f, fdest.file)
- fdest.close()
+ version = create_versioned_file(f.name, event.datadir, f)
# Extract Info from uploaded data
- try:
- handle_uploaded_data(event, uploadDestination)
- event.submitter = request.user
- except:
- # XXX Bad news. If the log file fails to save because of
- # race conditions, then this will also be the the message
- # returned. Somehow, I think there are other things that
- # could go wrong inside handle_uploaded_data besides just
- # bad data. We should probably check for different types
- # of exceptions here.
- return Response("Bad Data",
- status=status.HTTP_400_BAD_REQUEST)
+ uploadDestination = os.path.join(event.datadir, f.name)
+ handle_uploaded_data(event, uploadDestination)
+ event.submitter = request.user
# Save event
event.save()
@@ -894,16 +879,11 @@ class EventLogList(InheritPermissionsAPIView):
file_version = None
if uploadedFile:
filename = uploadedFile.name
- filepath = os.path.join(event.datadir, filename)
try:
# Open / Write the file.
- fdest = VersionedFile(filepath, 'w')
- for chunk in uploadedFile.chunks():
- fdest.write(chunk)
- fdest.close()
- # Ascertain the version assigned to this particular file.
- file_version = fdest.version
+ file_version = create_versioned_file(filename, event.datadir,
+ uploadedFile)
except Exception as e:
# XXX This needs some thought.
response = Response(str(e), status=status.HTTP_400_BAD_REQUEST)
@@ -1613,20 +1593,15 @@ class Files(InheritPermissionsAPIView):
def put(self, request, event, filename=""):
""" File uploader. Implements file versioning. """
filename = filename or ""
- filepath = os.path.join(event.datadir, filename)
try:
# Open / Write the file.
- fdest = VersionedFile(filepath, 'w')
f = request.data['upload']
- for chunk in f.chunks():
- fdest.write(chunk)
- fdest.close()
- file_version = fdest.version
+ file_version = create_versioned_file(filename, event.datadir, f)
rv = {}
# XXX this seems wobbly.
- longname = fdest.name
+ longname = os.path.join(event.datadir, filename)
shortname = longname[longname.rfind(filename):]
rv['permalink'] = api_reverse(
"events:files", args=[event.graceid, shortname], request=request)
@@ -1756,11 +1731,8 @@ class VOEventList(InheritPermissionsAPIView):
voevent_display_type = dict(VOEvent.VOEVENT_TYPE_CHOICES)[voevent_type].capitalize()
filename = "%s-%d-%s.xml" % (event.graceid, voevent.N, voevent_display_type)
- filepath = os.path.join(event.datadir, filename)
- fdest = VersionedFile(filepath, 'w')
- fdest.write(voevent_text)
- fdest.close()
- file_version = fdest.version
+ file_version = create_versioned_file(filename, event.datadir,
+ voevent_text)
voevent.filename = filename
voevent.file_version = file_version
diff --git a/gracedb/core/vfile.py b/gracedb/core/vfile.py
index 523daf9d6..c5ae8abe8 100644
--- a/gracedb/core/vfile.py
+++ b/gracedb/core/vfile.py
@@ -250,7 +250,6 @@ class VersionedFile(object):
def create_versioned_file(filename, file_dir, file_contents):
-
# Get full file path
full_path = os.path.join(file_dir, filename)
@@ -263,6 +262,9 @@ def create_versioned_file(filename, file_dir, file_contents):
fdest = VersionedFile(full_path, 'wb')
for chunk in file_contents.chunks():
fdest.write(chunk)
+ else:
+ raise TypeError('Unexpected file contents in '
+ 'core.vfile.create_versioned_file')
fdest.close()
return fdest.version
diff --git a/gracedb/events/serialize.py b/gracedb/events/serialize.py
index 1a40f7cae..f36143349 100644
--- a/gracedb/events/serialize.py
+++ b/gracedb/events/serialize.py
@@ -1,6 +1,5 @@
-#!/usr/bin/python
-
from math import log
+import os
from time import gmtime, strftime
from glue.lal import LIGOTimeGPS
@@ -8,7 +7,7 @@ from glue.ligolw import ligolw
from glue.ligolw import table
from glue.ligolw import lsctables
-from core.vfile import VersionedFile
+from core.vfile import VersionedFile, create_versioned_file
##############################################################################
#
@@ -84,17 +83,19 @@ def compute_mchirp_eta(m1,m2):
def write_output_files(root_dir, xmldoc, log_content, \
xml_fname = 'coinc.xml', log_fname = 'event.log'):
- """
- write the xml-format coinc tables and log file
- """
+ """
+ Write the xml-format coinc tables and log file
+ """
- f = VersionedFile(root_dir+'/'+xml_fname,'w')
- xmldoc.write(f.file)
- f.close()
+ # Write xml-formatted coinc table
+ # We do it this way instead of using create_versioned_file since the
+ # xmldoc is designed to write to a file object.
+ file_path = os.path.join(root_dir, xml_fname)
+ f = VersionedFile(file_path, 'w')
+ xmldoc.write(f.file)
- f = VersionedFile(root_dir+'/'+log_fname,'w')
- f.write(log_content)
- f.close()
+ # Write log file
+ create_versioned_file(log_fname, root_dir, log_content)
def get_ifos_for_cwb(cwb_ifos):
"""
diff --git a/gracedb/events/translator.py b/gracedb/events/translator.py
index 3ebd08d49..64c5ee5a0 100644
--- a/gracedb/events/translator.py
+++ b/gracedb/events/translator.py
@@ -15,7 +15,7 @@ from .serialize import populate_omega_tables, write_output_files
from VOEventLib.Vutil import parse, getWhereWhen, findParam, getParamNames
from core.time_utils import isoToGps, isoToGpsFloat
-from core.vfile import VersionedFile
+from core.vfile import create_versioned_file
import json
try:
@@ -178,9 +178,6 @@ def handle_uploaded_data(event, datafilename,
event.snr = snr
event.false_alarm_rate = getattr(coinc_table, "false_alarm_rate", None)
event.combined_far = far
-
- # XXX xml_filename unused
- #xml_filename = os.path.join(output_dir, coinc_table_filename)
event.save()
# Extract Single Inspiral Information
@@ -292,10 +289,6 @@ def handle_uploaded_data(event, datafilename,
event.instruments = coinc_table.ifos
event.nevents = coinc_event_table.nevents
event.likelihood = cleanData(coinc_event_table.likelihood, 'likelihood')
-
- # XXX xml_filename unused.
- #xml_filename = os.path.join(output_dir, coinc_table_filename)
-
event.save()
elif pipeline in ['CWB', 'CWB2G']:
@@ -317,7 +310,7 @@ def handle_uploaded_data(event, datafilename,
comment="Coinc Table Created")
log.save()
- if data.writeLogfile( os.path.join(outputDataDir, "event.log") ):
+ if data.writeLogfile(outputDataDir, "event.log"):
log = EventLog(event=event,
filename="event.log",
file_version=0,
@@ -452,12 +445,10 @@ class Translator(object):
logdata.append("FAR: %s" % val_or_dashes(data.get('far')))
return "\n".join(logdata)
- def writeLogfile(self, path):
+ def writeLogfile(self, data_directory, filename):
data = self.logData()
if data:
- f = VersionedFile(path, 'w')
- f.write(data)
- f.close()
+ create_versioned_file(filename, data_directory, data)
return True
diff --git a/gracedb/events/view_logic.py b/gracedb/events/view_logic.py
index 1ecaf9247..3c8ab99a0 100644
--- a/gracedb/events/view_logic.py
+++ b/gracedb/events/view_logic.py
@@ -17,7 +17,7 @@ from .permission_utils import assign_default_event_perms
from alerts.issuers.events import EventAlertIssuer, EventLabelAlertIssuer, \
EventEMObservationAlertIssuer, EventEMBBEventLogAlertIssuer
-from core.vfile import VersionedFile
+from core.vfile import create_versioned_file
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import Permission
@@ -105,10 +105,7 @@ def _createEventFromForm(request, form):
# Write the event data file to disk.
f = request.FILES['eventFile']
uploadDestination = os.path.join(eventDir, f.name)
- fdest = VersionedFile(uploadDestination, 'w')
- for chunk in f.chunks():
- fdest.write(chunk)
- fdest.close()
+ version = create_versioned_file(f.name, event.datadir, f)
file_contents = None
# Extract Info from uploaded data
diff --git a/gracedb/events/views.py b/gracedb/events/views.py
index c79206a1b..3f0d2cfe6 100644
--- a/gracedb/events/views.py
+++ b/gracedb/events/views.py
@@ -48,7 +48,7 @@ log = logging.getLogger(__name__)
import os
from django.conf import settings
-from core.vfile import VersionedFile
+from core.vfile import create_versioned_file
# XXX This should be configurable / moddable or something
MAX_QUERY_RESULTS = 1000
@@ -211,16 +211,9 @@ def logentry(request, event, num=None):
file_version = None
if uploadedFile:
filename = uploadedFile.name
- filepath = os.path.join(event.datadir, filename)
-
try:
- # Open / Write the file.
- fdest = VersionedFile(filepath, 'w')
- for chunk in uploadedFile.chunks():
- fdest.write(chunk)
- fdest.close()
- # Ascertain the version assigned to this particular file.
- file_version = fdest.version
+ file_version = create_versioned_file(filename, event.datadir,
+ uploadedFile)
except Exception as e:
return HttpResponseServerError(str(e))
--
GitLab
From 96319a00594d42525c80a799a1b22b2f77f864ae Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 13 Sep 2019 15:28:08 -0500
Subject: [PATCH 035/106] Python 3: fix a few tests with decoding response
content
---
gracedb/api/v1/events/tests/test_eventgraceidfield.py | 2 +-
gracedb/api/v1/events/tests/test_update_grbevent_view.py | 7 ++++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/gracedb/api/v1/events/tests/test_eventgraceidfield.py b/gracedb/api/v1/events/tests/test_eventgraceidfield.py
index bff49b445..ca2744223 100644
--- a/gracedb/api/v1/events/tests/test_eventgraceidfield.py
+++ b/gracedb/api/v1/events/tests/test_eventgraceidfield.py
@@ -50,4 +50,4 @@ def test_valid_graceids(graceid):
call_args, _ = mock_super_tiv.call_args
assert mock_super_tiv.call_count == 1
assert len(call_args) == 1
- assert call_args[0].encode() == graceid.upper().strip()
+ assert call_args[0] == graceid.upper().strip()
diff --git a/gracedb/api/v1/events/tests/test_update_grbevent_view.py b/gracedb/api/v1/events/tests/test_update_grbevent_view.py
index 64078c84d..06d68be1e 100644
--- a/gracedb/api/v1/events/tests/test_update_grbevent_view.py
+++ b/gracedb/api/v1/events/tests/test_update_grbevent_view.py
@@ -147,7 +147,8 @@ def test_update_with_no_new_data(grb_user, internal_group, data):
# Check response
assert response.status_code == 400
- assert 'Request would not modify the GRB event' in response.content
+ assert 'Request would not modify the GRB event' \
+ in response.content.decode()
@pytest.mark.parametrize("data",
@@ -176,7 +177,7 @@ def test_update_with_bad_data(grb_user, internal_group, data):
# Check response
assert response.status_code == 400
- assert 'must be a float' in response.content
+ assert 'must be a float' in response.content.decode()
@pytest.mark.django_db
@@ -206,4 +207,4 @@ def test_update_non_grbevent(grb_user, internal_group):
# Check response
assert response.status_code == 400
assert 'Cannot update GRB event parameters for non-GRB event' \
- in response.content
+ in response.content.decode()
--
GitLab
From 97f6c93637fa9a88cc5eda3acb399d654c9fdd44 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 13 Sep 2019 15:28:32 -0500
Subject: [PATCH 036/106] requirements: remove unnecessary package (future)
---
requirements.txt | 1 -
1 file changed, 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index e60ce6eec..ac3075785 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -51,6 +51,5 @@ pytz==2018.9
pyasn1==0.3.6
pyasn1-modules==0.1.5
# Installing future for gunicorn gthreads:
-future==0.17.1
futures==3.2.0
gunicorn[gthread]==19.9.0
--
GitLab
From ce7d5f3818248fcdb5ab06897c084074143fae61 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 13 Sep 2019 15:40:59 -0500
Subject: [PATCH 037/106] requirements: fix voeventlib install to be only
Python 3
---
requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index ac3075785..af65afd3a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -35,7 +35,7 @@ service_identity==17.0.0
simplejson==3.15.0
Sphinx==1.7.0
twilio==6.10.3
-voeventlib==1.2
+voeventlib==1.2; python_version >= '3'
# Do NOT upgrade pyparsing beyond 2.3.0 (even to 2.3.1) without carefully
# testing and modifying the search query code. There were a number of
# problematic API changes in 2.3.1 and beyond that will need to be handled
--
GitLab
From d78bf493d75c2053900657ec4e10a124aa6aab43 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 13 Sep 2019 15:41:32 -0500
Subject: [PATCH 038/106] requirements: fix pytest version for python 2 vs 3
---
requirements.txt | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index af65afd3a..767ffadb5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -41,7 +41,8 @@ voeventlib==1.2; python_version >= '3'
# problematic API changes in 2.3.1 and beyond that will need to be handled
# appropriately.
pyparsing==2.3.0
-pytest<5
+pytest<5; python_version < '3'
+pytest==5.1.2; python_version >= '3'
pytest-cov==2.6.1
pytest-django==3.4.8
pytz==2018.9
--
GitLab
From 4adbae8aba12566dedd3f9adf5bc3961ff7eeff9 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 13 Sep 2019 15:41:48 -0500
Subject: [PATCH 039/106] requirements: fix futures install to be python 2 only
---
requirements.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/requirements.txt b/requirements.txt
index 767ffadb5..6209a0bee 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,6 +12,7 @@ djangorestframework==3.9.0
djangorestframework-guardian==0.1.1
dnspython==1.15.0
flake8==3.5.0
+gunicorn[gthread]==19.9.0
html5lib==1.0.1
ipdb==0.10.2
ipython==5.5.0
@@ -51,6 +52,5 @@ pytz==2018.9
# https://github.com/etingof/pyasn1/issues/112
pyasn1==0.3.6
pyasn1-modules==0.1.5
-# Installing future for gunicorn gthreads:
-futures==3.2.0
-gunicorn[gthread]==19.9.0
+# Installing futures for gunicorn gthreads (Python 2 only):
+futures==3.2.0; python_version < '3'
--
GitLab
From e5ffd2c6ff207c25ade8ef6f033adcca3bf3176e Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Tue, 17 Sep 2019 09:42:39 -0500
Subject: [PATCH 040/106] core.vfile: fixed file writing for bytes
---
gracedb/core/vfile.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/gracedb/core/vfile.py b/gracedb/core/vfile.py
index c5ae8abe8..a362993b9 100644
--- a/gracedb/core/vfile.py
+++ b/gracedb/core/vfile.py
@@ -257,8 +257,11 @@ def create_versioned_file(filename, file_dir, file_contents):
if isinstance(file_contents, six.string_types):
fdest = VersionedFile(full_path, 'w')
fdest.write(file_contents)
+ elif isinstance(file_contents, bytes):
+ fdest = VersionedFile(full_path, 'wb')
+ fdest.write(file_contents)
elif isinstance(file_contents, (UploadedFile, InMemoryUploadedFile,
- TemporaryUploadedFile, SimpleUploadedFile, bytes)):
+ TemporaryUploadedFile, SimpleUploadedFile)):
fdest = VersionedFile(full_path, 'wb')
for chunk in file_contents.chunks():
fdest.write(chunk)
--
GitLab
From 75f34a849511c148899e8da8bc9b238a6843552b Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 13 Sep 2019 15:48:03 -0500
Subject: [PATCH 041/106] Python 3: remove Python 2 from CI config
Also, use Python 3.5 instead of Python 3.7 - I think we're going
to be using 3.5 for a bit since that's default for Debian 9.
---
.gitlab-ci.yml | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 46fe1652c..b9e999545 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -46,7 +46,7 @@ before_script:
# set python version
- PYTHON_VERSION="${CI_JOB_NAME##*:}"
- PYTHON_MAJOR="${PYTHON_VERSION:0:1}"
- - if [[ "${PYTHON_MAJOR}" -eq 2 ]]; then PYTHON="python"; else PYTHON="python3"; fi
+ - PYTHON="python3"
# install build requirements
- apt-get -yqq update
- apt-get -o dir::cache::archives="${APT_CACHE_DIR}" install -yqq
@@ -58,8 +58,6 @@ before_script:
libxml2-dev
swig
${PYTHON}-pip
- # install voeventlib for python2
- - if [[ "${PYTHON_MAJOR}" -eq 2 ]]; then apt-get -o dir::cache::archives="${APT_CACHE_DIR}" install -yqq python-voeventlib; fi
# install everything else from pip
- ${PYTHON} -m pip install -r requirements.txt
# create logs path required for tests
@@ -79,13 +77,9 @@ before_script:
- .cache/pip
- .cache/apt
-test:2.7:
+test:3.5:
<<: *test
-test:3.7:
- <<: *test
- allow_failure: true
-
branch_image:
stage: branch
script:
--
GitLab
From c1a78e7016c66e9a8da8080609d2162a434c444d Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Tue, 17 Sep 2019 09:40:41 -0500
Subject: [PATCH 042/106] Rework VOEvent building infrastructure
* Switch from VOEventLib to voevent-parse due to significant
string-bytes issues in VOEventLib
* Remove bad exception handling in events API for creating a
VOEvent
* Add parameter checking to events API for creating a VOEvent
* Combine VOEvent building code for events and superevents into
a single instance which handles both cases
---
config/settings/base.py | 9 +-
docs/admin_docs/source/miscellaneous.rst | 13 +-
gracedb/annotations/__init__.py | 0
gracedb/annotations/voevent_utils.py | 599 ++++++++++++++++++++++
gracedb/api/v1/events/views.py | 68 ++-
gracedb/api/v1/superevents/serializers.py | 2 +-
gracedb/api/v1/superevents/views.py | 5 +-
gracedb/events/buildVOEvent.py | 482 -----------------
gracedb/events/models.py | 3 +
gracedb/superevents/buildVOEvent.py | 456 ----------------
gracedb/superevents/utils.py | 2 +-
requirements.txt | 2 +-
12 files changed, 663 insertions(+), 978 deletions(-)
create mode 100644 gracedb/annotations/__init__.py
create mode 100644 gracedb/annotations/voevent_utils.py
delete mode 100644 gracedb/events/buildVOEvent.py
delete mode 100644 gracedb/superevents/buildVOEvent.py
diff --git a/config/settings/base.py b/config/settings/base.py
index 855096930..e605585da 100644
--- a/config/settings/base.py
+++ b/config/settings/base.py
@@ -201,11 +201,9 @@ GRB_PIPELINES = [
'Swift',
]
-# SkyAlert stuff - used for VOEvents (?) --------------------------------------
-IVORN_PREFIX = "ivo://gwnet/LVC#"
-SKYALERT_ROLE = "test"
-SKYALERT_DESCRIPTION = "Report of a candidate gravitational wave event"
-SKYALERT_SUBMITTERS = ['Patrick Brady', 'Brian Moe']
+# VOEvent stream --------------------------------------------------------------
+VOEVENT_STREAM = 'gwnet/LVC'
+
# Stuff related to report/plot generation -------------------------------------
@@ -343,6 +341,7 @@ INSTALLED_APPS = [
'django.contrib.staticfiles',
'django.contrib.messages',
'alerts',
+ 'annotations',
'api',
'core',
'events',
diff --git a/docs/admin_docs/source/miscellaneous.rst b/docs/admin_docs/source/miscellaneous.rst
index 35be9e290..4aa34efab 100644
--- a/docs/admin_docs/source/miscellaneous.rst
+++ b/docs/admin_docs/source/miscellaneous.rst
@@ -87,12 +87,15 @@ but you still have to go through the same sequence of steps that you would
for a true developement task. I recommend the workflow described in :ref:`new_server_feature`.
In this particular case, the only necessary code change is to edit the
-file ``gracedb/events/buildVOEvent.py`` and add something like::
+file ``gracedb/annotations/voevent_utils.py`` and add something like::
- w.add_Param(Param(name="MyParam",
- dataType="float",
+ p_new = vp.Param(
+ "MyParam",
value=getMyParamForEvent(event),
- Description=["My lovely new parameter"]))
+ dataType="float"
+ )
+ p_new.Description = "My lovely new parameter"
+ v.What.append(p_new)
working by analogy with the other parameters present. I only wanted to give
this example here, because it seems likely that such a task will be considered
@@ -108,7 +111,7 @@ A good starting point is to search the GraceDB server code for "L1" to see where
Specifics (assume X1 is the IFO code):
1. Add X1OPS, X1OK, X1NO labels, update ``gracedb/templates/gracedb/event_detail_script.js`` with description, and update ``gracedb/templates/search/query_help_frag.html``
-2. Add to instruments in ``gracedb/events/buildVOEvent.py``
+2. Add to instruments in ``gracedb/annotations/voevent_utils.py``
3. Update ifoList in ``gracedb/events/query.py``
4. Add entry to ``CONTROL_ROOM_IPS`` in ``gracedb/config/settings/base.py``
5. Add signoff option for X1 in ``gracedb/templates/gracedb/event_detail.html``
diff --git a/gracedb/annotations/__init__.py b/gracedb/annotations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
new file mode 100644
index 000000000..518334c31
--- /dev/null
+++ b/gracedb/annotations/voevent_utils.py
@@ -0,0 +1,599 @@
+# See the VOEvent specification for details
+# http://www.ivoa.net/Documents/latest/VOEvent.html
+
+import datetime
+import logging
+import os
+
+from scipy.constants import c, G, pi
+import voeventparse as vp
+
+from django.conf import settings
+from django.urls import reverse
+
+from core.time_utils import gpsToUtc
+from core.urls import build_absolute_uri
+from events.models import VOEventBase
+from events.models import CoincInspiralEvent, MultiBurstEvent, \
+ LalInferenceBurstEvent
+from superevents.shortcuts import is_superevent
+
+# Set up logger
+logger = logging.getLogger(__name__)
+
+
+###############################################################################
+# SETUP #######################################################################
+###############################################################################
+# Dict of VOEvent type abbreviations and full strings
+VOEVENT_TYPE_DICT = dict(VOEventBase.VOEVENT_TYPE_CHOICES)
+
+
+# Used to create the Packet_Type parameter block
+PACKET_TYPES = {
+ VOEventBase.VOEVENT_TYPE_PRELIMINARY: (150, 'LVC_PRELIMINARY'),
+ VOEventBase.VOEVENT_TYPE_INITIAL: (151, 'LVC_INITIAL'),
+ VOEventBase.VOEVENT_TYPE_UPDATE: (152, 'LVC_UPDATE'),
+ VOEventBase.VOEVENT_TYPE_RETRACTION: (164, 'LVC_RETRACTION'),
+}
+
+
+# Description strings
+DEFAULT_DESCRIPTION = \
+ "Candidate gravitational wave event identified by low-latency analysis"
+INSTRUMENT_DESCRIPTIONS = {
+ "H1": "H1: LIGO Hanford 4 km gravitational wave detector",
+ "L1": "L1: LIGO Livingston 4 km gravitational wave detector",
+ "V1": "V1: Virgo 3 km gravitational wave detector",
+}
+
+
+###############################################################################
+# MAIN ########################################################################
+###############################################################################
+def construct_voevent_file(obj, voevent, request=None):
+
+ # Setup ###################################################################
+ ## Determine event or superevent
+ obj_is_superevent = False
+ if is_superevent(obj):
+ obj_is_superevent = True
+ event = obj.preferred_event
+ graceid = obj.default_superevent_id
+ obj_view_name = "superevents:view"
+ fits_view_name = "api:default:superevents:superevent-file-detail"
+ else:
+ event = obj
+ graceid = obj.graceid
+ obj_view_name = "view"
+ fits_view_name = "api:default:events:files"
+
+ # Get the event subclass (CoincInspiralEvent, MultiBurstEvent, etc.) and
+ # set that as the event
+ event = event.get_subclass_or_self()
+
+ ## Let's convert that voevent_type to something nicer looking
+ voevent_type = VOEVENT_TYPE_DICT[voevent.voevent_type]
+
+ ## Now build the IVORN.
+ type_string = voevent_type.capitalize()
+ voevent_id = '{gid}-{N}-{type_str}'.format(type_str=type_string,
+ gid=graceid, N=voevent.N)
+
+ ## Determine role
+ if event.is_mdc() or event.is_test():
+ role = vp.definitions.roles.test
+ else:
+ role = vp.definitions.roles.observation
+
+ ## Instantiate VOEvent
+ v = vp.Voevent(settings.VOEVENT_STREAM, voevent_id, role)
+
+ ## Set root Description
+ if voevent_type != 'retraction':
+ v.Description = "Report of a candidate gravitational wave event"
+
+ # Who #####################################################################
+ ## Remove Who.Description
+ v.Who.remove(v.Who.Description)
+
+ ## Set Who.Date
+ vp.set_who(
+ v,
+ date=datetime.datetime.utcnow()
+ )
+
+ ## Set Who.Author
+ vp.set_author(
+ v,
+ contactName="LIGO Scientific Collaboration and Virgo Collaboration"
+ )
+
+ # How #####################################################################
+ if voevent_type != 'retraction':
+ descriptions = [DEFAULT_DESCRIPTION]
+
+ # Add instrument descriptions
+ instruments = event.instruments.split(',')
+ for inst in INSTRUMENT_DESCRIPTIONS:
+ if inst in instruments:
+ descriptions.append(INSTRUMENT_DESCRIPTIONS[inst])
+ if voevent.coinc_comment:
+ descriptions.append("A gravitational wave trigger identified a "
+ "possible counterpart GRB")
+ vp.add_how(v, descriptions=descriptions)
+
+ # What ####################################################################
+ # UCD = Unified Content Descriptors
+ # http://monet.uni-sw.gwdg.de/twiki/bin/view/VOEvent/UnifiedContentDescriptors
+ # OR -- (from VOTable document, [21] below)
+ # http://www.ivoa.net/twiki/bin/view/IVOA/IvoaUCD
+ # http://cds.u-strasbg.fr/doc/UCD.htx
+ #
+ # which somehow gets you to:
+ # http://www.ivoa.net/Documents/REC/UCD/UCDlist-20070402.html
+ # where you might find some actual information.
+
+ # Unit / Section 4.3 of [21] which relies on [25]
+ # [21] http://www.ivoa.net/Documents/latest/VOT.html
+ # [25] http://vizier.u-strasbg.fr/doc/catstd-3.2.htx
+ #
+ # Basically, a string that makes sense to humans about what units a value
+ # is. eg. "m/s"
+
+ ## Packet_Type param
+ p_packet_type = vp.Param(
+ "Packet_Type",
+ value=PACKET_TYPES[voevent.voevent_type][0],
+ ac=True
+ )
+ p_packet_type.Description = ("The Notice Type number is assigned/used "
+ "within GCN, eg type={typenum} is an {typedesc} notice").format(
+ typenum=PACKET_TYPES[voevent.voevent_type][0],
+ typedesc=PACKET_TYPES[voevent.voevent_type][1]
+ )
+ v.What.append(p_packet_type)
+
+ # Internal param
+ p_internal = vp.Param(
+ "internal",
+ value=int(voevent.internal),
+ ac=True
+ )
+ p_internal.Description = ("Indicates whether this event should be "
+ "distributed to LSC/Virgo members only")
+ v.What.append(p_internal)
+
+ ## Packet serial number
+ p_serial_num = vp.Param(
+ "Pkt_Ser_Num",
+ value=voevent.N,
+ ac=True
+ )
+ p_serial_num.Description = ("A number that increments by 1 each time a "
+ "new revision is issued for this event")
+ v.What.append(p_serial_num)
+
+ ## Event graceid or superevent ID
+ p_gid = vp.Param(
+ "GraceID",
+ value=graceid,
+ ucd="meta.id",
+ dataType="string"
+ )
+ p_gid.Description = "Identifier in GraceDB"
+ v.What.append(p_gid)
+
+ ## Alert type parameter
+ p_alert_type = vp.Param(
+ "AlertType",
+ value = voevent_type.capitalize(),
+ ucd="meta.version",
+ dataType="string"
+ )
+ p_alert_type.Description = "VOEvent alert type"
+ v.What.append(p_alert_type)
+
+ ## Whether the event is a hardware injection or not
+ p_hardware_inj = vp.Param(
+ "HardwareInj",
+ value=int(voevent.hardware_inj),
+ ucd="meta.number",
+ ac=True
+ )
+ p_hardware_inj.Description = ("Indicates that this event is a hardware "
+ "injection if 1, no if 0")
+ v.What.append(p_hardware_inj)
+
+ ## Open alert parameter
+ p_open_alert = vp.Param(
+ "OpenAlert",
+ value=int(voevent.open_alert),
+ ucd="meta.number",
+ ac=True
+ )
+ p_open_alert.Description = ("Indicates that this event is an open alert "
+ "if 1, no if 0")
+ v.What.append(p_open_alert)
+
+ ## Superevent page
+ p_detail_url = vp.Param(
+ "EventPage",
+ value=build_absolute_uri(
+ reverse(obj_view_name, args=[graceid]),
+ request
+ ),
+ ucd="meta.ref.url",
+ dataType="string"
+ )
+ p_detail_url.Description = ("Web page for evolving status of this GW "
+ "candidate")
+ v.What.append(p_detail_url)
+
+ ## Only for non-retractions
+ if voevent_type != 'retraction':
+ ## Instruments
+ p_instruments = vp.Param(
+ "Instruments",
+ value=event.instruments,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_instruments.Description = ("List of instruments used in analysis to "
+ "identify this event")
+ v.What.append(p_instruments)
+
+ ## False alarm rate
+ if event.far:
+ p_far = vp.Param(
+ "FAR",
+ value=float(max(event.far, settings.VOEVENT_FAR_FLOOR)),
+ ucd="arith.rate;stat.falsealarm",
+ unit="Hz",
+ ac=True
+ )
+ p_far.Description = ("False alarm rate for GW candidates with "
+ "this strength or greater")
+ v.What.append(p_far)
+
+ ## Analysis group
+ p_group = vp.Param(
+ "Group",
+ value=event.group.name,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_group.Description = "Data analysis working group"
+ v.What.append(p_group)
+
+ ## Analysis pipeline
+ p_pipeline = vp.Param(
+ "Pipeline",
+ value=event.pipeline.name,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_pipeline.Description = "Low-latency data analysis pipeline"
+ v.What.append(p_pipeline)
+
+ ## Search type
+ if event.search:
+ p_search = vp.Param(
+ "Search",
+ value=event.search.name,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_search.Description = "Specific low-latency search"
+ v.What.append(p_search)
+
+ # initial and update VOEvents must have a skymap.
+ # new feature (10/24/2016): preliminary VOEvents can have a skymap,
+ # but they don't have to.
+ if (voevent_type in ["initial", "update"] or
+ (voevent_type == "preliminary" and voevent.skymap_filename != None)):
+
+ ## Skymap group
+ ### fits skymap URL
+ fits_skymap_url = build_absolute_uri(
+ reverse(fits_view_name, args=[graceid, voevent.skymap_filename]),
+ request
+ )
+ p_fits_url = vp.Param(
+ "skymap_fits",
+ value=fits_skymap_url,
+ ucd="meta.ref.url",
+ dataType="string"
+ )
+ p_fits_url.Description = "Sky Map FITS"
+
+ ### Create skymap group with params
+ skymap_group = vp.Group(
+ [p_fits_url],
+ name="GW_SKYMAP",
+ type="GW_SKYMAP",
+ )
+
+ ### Add to What
+ v.What.append(skymap_group)
+
+ ## Analysis specific attributes
+ if voevent_type != 'retraction':
+ ### Classification group (EM-Bright params; CBC only)
+ em_bright_params = []
+ source_properties_params = []
+ if (isinstance(event, CoincInspiralEvent) and
+ voevent_type != 'retraction'):
+
+ # EM-Bright mass classifier information for CBC event candidates
+ if voevent.prob_bns is not None:
+ p_pbns = vp.Param(
+ "BNS",
+ value=voevent.prob_bns,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pbns.Description = \
+ ("Probability that the source is a binary neutron star "
+ "merger (both objects lighter than 3 solar masses)")
+ em_bright_params.append(p_pbns)
+
+ if voevent.prob_nsbh is not None:
+ p_pnsbh = vp.Param(
+ "NSBH",
+ value=voevent.prob_nsbh,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pnsbh.Description = \
+ ("Probability that the source is a neutron star-black "
+ "hole merger (primary heavier than 5 solar masses, "
+ "secondary lighter than 3 solar masses)")
+ em_bright_params.append(p_pnsbh)
+
+ if voevent.prob_bbh is not None:
+ p_pbbh = vp.Param(
+ "BBH",
+ value=voevent.prob_bbh,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pbbh.Description = ("Probability that the source is a "
+ "binary black hole merger (both objects "
+ "heavier than 5 solar masses)")
+ em_bright_params.append(p_pbbh)
+
+ if voevent.prob_mass_gap is not None:
+ p_pmassgap = vp.Param(
+ "MassGap",
+ value=voevent.prob_mass_gap,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pmassgap.Description = ("Probability that the source has at "
+ "least one object between 3 and 5 "
+ "solar masses")
+ em_bright_params.append(p_pmassgap)
+
+ if voevent.prob_terrestrial is not None:
+ p_pterr = vp.Param(
+ "Terrestrial",
+ value=voevent.prob_terrestrial,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_pterr.Description = ("Probability that the source is "
+ "terrestrial (i.e., a background noise "
+ "fluctuation or a glitch)")
+ em_bright_params.append(p_pterr)
+
+ # Add to source properties group
+ if voevent.prob_has_ns is not None:
+ p_phasns = vp.Param(
+ name="HasNS",
+ value=voevent.prob_has_ns,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_phasns.Description = ("Probability that at least one object "
+ "in the binary has a mass that is "
+ "less than 3 solar masses")
+ source_properties_params.append(p_phasns)
+
+ if voevent.prob_has_remnant is not None:
+ p_phasremnant = vp.Param(
+ "HasRemnant",
+ value=voevent.prob_has_remnant,
+ ucd="stat.probability",
+ ac=True
+ )
+ p_phasremnant.Description = ("Probability that a nonzero mass "
+ "was ejected outside the central "
+ "remnant object")
+ source_properties_params.append(p_phasremnant)
+
+ elif isinstance(event, MultiBurstEvent):
+ ### Central frequency
+ p_central_freq = vp.Param(
+ "CentralFreq",
+ value=float(event.central_freq),
+ ucd="gw.frequency",
+ unit="Hz",
+ dataType="float"
+ )
+ p_central_freq.Description = \
+ "Central frequency of GW burst signal"
+ v.What.append(p_central_freq)
+
+ ### Duration
+ p_duration = vp.Param(
+ "Duration",
+ value=float(event.duration),
+ unit="s",
+ ucd="time.duration",
+ dataType="float"
+ )
+ p_duration.Description = "Measured duration of GW burst signal"
+ v.What.append(p_duration)
+
+ # XXX Calculate the fluence. Unfortunately, this requires parsing the trigger.txt
+ # file for hrss values. These should probably be pulled into the database.
+ # But there is no consensus on whether hrss or fluence is meaningful. So I will
+ # put off changing the schema for now.
+ try:
+ # Go find the data file.
+ log = event.eventlog_set.filter(comment__startswith="Original Data").all()[0]
+ filename = log.filename
+ filepath = os.path.join(event.datadir,filename)
+ if os.path.isfile(filepath):
+ datafile = open(filepath,"r")
+ else:
+ raise VOEventBase.VOEventBuilderException(
+ "No file found.")
+ # Now parse the datafile.
+ # The line we want looks like:
+ # hrss: 1.752741e-23 2.101590e-23 6.418900e-23
+ for line in datafile:
+ if line.startswith('hrss:'):
+ hrss_values = [float(hrss) for hrss in line.split()[1:]]
+ max_hrss = max(hrss_values)
+ # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
+ # Note that hrss here actually has units of s^(-1/2)
+ fluence = pi * pow(c,3) * pow(event.central_freq,2)
+ fluence = fluence * pow(max_hrss,2)
+ fluence = fluence / (4.0*G)
+
+ p_fluence = vp.Param(
+ "Fluence",
+ value=fluence,
+ ucd="gw.fluence",
+ unit="erg/cm^2",
+ ac=True,
+ )
+ p_fluence.Description = "Estimated fluence of GW burst signal"
+ v.What.append(p_fluence)
+ except Exception as e:
+ logger.exception(e)
+
+ elif isinstance(event, LalInferenceBurstEvent):
+ p_freq = vp.Param(
+ "frequency",
+ value=float(event.frequency_mean),
+ ucd="gw.frequency",
+ unit="Hz",
+ dataType="float"
+ )
+ p_freq.Description = "Mean frequency of GW burst signal"
+ v.What.append(p_freq)
+
+ # Calculate the fluence.
+ # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
+ # Note that hrss here actually has units of s^(-1/2)
+ # XXX obviously need to refactor here.
+ try:
+ fluence = pi * pow(c,3) * pow(event.frequency,2)
+ fluence = fluence * pow(event.hrss,2)
+ fluence = fluence / (4.0*G)
+
+ p_fluence = vp.Param(
+ "Fluence",
+ value=fluence,
+ ucd="gw.fluence",
+ unit="erg/cm^2",
+ ac=True
+ )
+ p_fluence.Description = "Estimated fluence of GW burst signal"
+ v.What.append(p_fluence)
+ except Exception as e:
+ logger.exception(e)
+
+ ## Create classification group
+ classification_group = vp.Group(
+ em_bright_params,
+ name='Classification',
+ type='Classification' # keep this only for backwards compatibility
+ )
+ classification_group.Description = \
+ ("Source classification: binary neutron star (BNS), neutron star-"
+ "black hole (NSBH), binary black hole (BBH), MassGap, or "
+ "terrestrial (noise)")
+ v.What.append(classification_group)
+
+ ## Create properties group
+ properties_group = vp.Group(
+ source_properties_params,
+ name='Properties',
+ type='Properties' # keep this only for backwards compatibility
+ )
+ properties_group.Description = \
+ ("Qualitative properties of the source, conditioned on the "
+ "assumption that the signal is an astrophysical compact binary "
+ "merger")
+ v.What.append(properties_group)
+
+ # WhereWhen ###############################################################
+ # NOTE: we use a fake ra, dec, err, and units for creating the coords
+ # object. We are required to provide them by the voeventparse code, but
+ # our "format" for VOEvents didn't have a Position2D entry. So to make
+ # the code work but maintain the same format, we add fake information here,
+ # then remove it later.
+ coords = vp.Position2D(
+ ra=1, dec=2, err=3, units='degrees',
+ system=vp.definitions.sky_coord_system.utc_fk5_geo
+ )
+ observatory_id = 'LIGO Virgo'
+ vp.add_where_when(
+ v,
+ coords,
+ gpsToUtc(event.gpstime),
+ observatory_id
+ )
+ # NOTE: now remove position 2D so the fake ra, dec, err, and units
+ # don't show up.
+ ol = v.WhereWhen.ObsDataLocation.ObservationLocation
+ ol.AstroCoords.remove(ol.AstroCoords.Position2D)
+
+ # Citations ###############################################################
+ if obj.voevent_set.count() > 1:
+
+ ## Loop over previous VOEvents for this event or superevent and
+ ## add them to citations
+ event_ivorns_list = []
+ for ve in obj.voevent_set.all():
+ # Oh, actually we need to exclude *this* voevent.
+ if ve.N == voevent.N:
+ continue
+
+ # Get cite type
+ if voevent_type == 'retraction':
+ cite_type = vp.definitions.cite_types.retraction
+ else:
+ cite_type = vp.definitions.cite_types.supersedes
+
+ # Set up event ivorn
+ ei = vp.EventIvorn(ve.ivorn, cite_type)
+
+ # Add event ivorn
+ event_ivorns_list.append(ei)
+
+ # Add citations
+ vp.add_citations(
+ v,
+ event_ivorns_list
+ )
+ # Get description for citation
+ desc = None
+ if voevent_type == 'preliminary':
+ desc = 'Initial localization is now available (preliminary)'
+ elif voevent_type == 'initial':
+ desc = 'Initial localization is now available'
+ elif voevent_type == 'update':
+ desc = 'Updated localization is now available'
+ elif voevent_type == 'retraction':
+ desc = 'Determined to not be a viable GW event candidate'
+ if desc is not None:
+ v.Citations.Description = desc
+
+ # Return the document as a string, along with the IVORN ###################
+ xml = vp.dumps(v, pretty_print=True)
+ return xml, v.get('ivorn')
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index 95da954e4..fbb41cde9 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -38,10 +38,10 @@ from rest_framework.views import APIView
from alerts.issuers.events import EventAlertIssuer, EventLogAlertIssuer, \
EventVOEventAlertIssuer, EventPermissionsAlertIssuer
+from annotations.voevent_utils import construct_voevent_file
from api.throttling import BurstAnonRateThrottle
from core.http import check_and_serve_file
from core.vfile import create_versioned_file
-from events.buildVOEvent import buildVOEvent, VOEventBuilderException
from events.forms import CreateEventForm
from events.models import Event, Group, Search, Pipeline, EventLog, Tag, \
Label, Labelling, EMGroup, EMBBEventLog, EMSPECTRUM, VOEvent, GrbEvent
@@ -1673,16 +1673,11 @@ class VOEventList(InheritPermissionsAPIView):
@event_and_auth_required
def post(self, request, event):
+ # Get data from request
voevent_type = request.data.get('voevent_type', None)
- if not voevent_type:
- msg = "You must provide a valid voevent_type."
- return Response({'error': msg}, status = status.HTTP_400_BAD_REQUEST)
-
internal = request.data.get('internal', 1)
-
skymap_type = request.data.get('skymap_type', None)
skymap_filename = request.data.get('skymap_filename', None)
-
open_alert = request.data.get('open_alert', 0)
hardware_inj = request.data.get('hardware_inj', 0)
CoincComment = request.data.get('CoincComment', None)
@@ -1694,9 +1689,44 @@ class VOEventList(InheritPermissionsAPIView):
Terrestrial = request.data.get('Terrestrial', None)
MassGap = request.data.get('MassGap', None)
- if (skymap_filename and not skymap_type) or (skymap_type and not skymap_filename):
- msg = "Both or neither of skymap_type and skymap_filename must be specified."
- return Response({'error': msg}, status = status.HTTP_400_BAD_REQUEST)
+ # Get VOEvent types as a dict (key = short form, value = long form)
+ VOEVENT_TYPE_DICT = dict(VOEvent.VOEVENT_TYPE_CHOICES)
+
+ # Check data
+ error = False
+ if not voevent_type or voevent_type not in VOEVENT_TYPE_DICT:
+ error = True
+ msg = "You must provide a valid voevent_type."
+ elif ((skymap_filename and not skymap_type) or
+ (skymap_type and not skymap_filename)):
+ error = True
+ msg = ("Both or neither of skymap_type and skymap_filename must "
+ "be specified.")
+ elif not event.gpstime:
+ error = True
+ msg = "Cannot build a VOEvent because event has no gpstime."
+ elif not event.far:
+ error = True
+ msg = "Cannot build a VOEvent because event has no FAR."
+ elif (voevent_type in ["IN", "UP"] or
+ voevent_type == "PR" and skymap_filename is not None):
+ if skymap_filename is None:
+ error = True
+ msg = "Skymap filename not provided."
+ if skymap_type is None:
+ error = True
+ msg = "Skymap type must be provided."
+
+ # Check if skymap file exists
+ skymap_file_path = os.path.join(event.datadir, skymap_filename)
+ if not os.path.exists(skymap_file_path):
+ error = True
+ msg = "Skymap file {fname} does not exist".format(
+ fname=skymap_filename)
+
+ # If there's an error, return a 400 response
+ if error:
+ return Response({'error': msg}, status=status.HTTP_400_BAD_REQUEST)
# Instantiate the voevent and save in order to get the serial number
voevent = VOEvent(event=event, issuer=request.user,
@@ -1716,20 +1746,10 @@ class VOEventList(InheritPermissionsAPIView):
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# Now, you need to actually build the VOEvent.
- try:
- voevent_text, ivorn = buildVOEvent(event, voevent, request=request)
- except VOEventBuilderException as e:
- voevent_text, ivorn = buildVOEvent(event, voevent.N, voevent_type, request,
- skymap_filename = skymap_filename, skymap_type = skymap_type,
- internal = internal, open_alert=open_alert,
- hardware_inj=hardware_inj, CoincComment=CoincComment,
- ProbHasNS=ProbHasNS, ProbHasRemnant=ProbHasRemnant, BNS=BNS,
- NSBH=NSBH, BBH=BBH, Terrestrial=Terrestrial, MassGap=MassGap)
-
- msg = "Problem building VOEvent: %s" % str(e)
- return Response({'error': msg}, status = status.HTTP_400_BAD_REQUEST)
-
- voevent_display_type = dict(VOEvent.VOEVENT_TYPE_CHOICES)[voevent_type].capitalize()
+ voevent_text, ivorn = construct_voevent_file(event, voevent,
+ request=request)
+
+ voevent_display_type = VOEVENT_TYPE_DICT[voevent_type].capitalize()
filename = "%s-%d-%s.xml" % (event.graceid, voevent.N, voevent_display_type)
file_version = create_versioned_file(filename, event.datadir,
voevent_text)
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index 29b852e37..8221091dc 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -653,7 +653,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
issuer = validated_data.pop('user')
# Call create function - creates VOEvent object and also runs
- # buildVOEvent to create the related file.
+ # construct_voevent_file to create the related file.
voevent = create_voevent_for_superevent(superevent, issuer,
**validated_data)
diff --git a/gracedb/api/v1/superevents/views.py b/gracedb/api/v1/superevents/views.py
index be85aa3ad..aa7c4735c 100644
--- a/gracedb/api/v1/superevents/views.py
+++ b/gracedb/api/v1/superevents/views.py
@@ -19,8 +19,7 @@ from core.vfile import FileVersionError, FileVersionNameError
from events.models import Event, Label
from events.view_utils import reverse as gracedb_reverse
from ligoauth.utils import is_internal
-from superevents.buildVOEvent import VOEventBuilderException
-from superevents.models import Superevent, Log, Signoff
+from superevents.models import Superevent, Log, Signoff, VOEvent
from superevents.utils import remove_tag_from_log, \
remove_event_from_superevent, remove_label_from_superevent, \
confirm_superevent_as_gw, get_superevent_by_date_id_or_404, \
@@ -321,7 +320,7 @@ class SupereventVOEventViewSet(SafeCreateMixin, InheritDefaultPermissionsMixin,
serializer_class = SupereventVOEventSerializer
pagination_class = BasePaginationFactory(results_name='voevents')
permission_classes = (SupereventVOEventModelPermissions,)
- create_error_classes = (VOEventBuilderException)
+ create_error_classes = (VOEvent.VOEventBuilderException)
lookup_url_kwarg = 'N'
lookup_field = 'N'
list_view_order_by = ('N',)
diff --git a/gracedb/events/buildVOEvent.py b/gracedb/events/buildVOEvent.py
deleted file mode 100644
index 67b159665..000000000
--- a/gracedb/events/buildVOEvent.py
+++ /dev/null
@@ -1,482 +0,0 @@
-
-# Taken from VOEventLib example code, which is:
-# Copyright 2010 Roy D. Williams
-# then modified
-"""
-buildVOEvent: Creates a complex VOEvent with tables
-See the VOEvent specification for details
-http://www.ivoa.net/Documents/latest/VOEvent.html
-"""
-from scipy.constants import c, G, pi
-
-from VOEventLib.VOEvent import VOEvent, Who, Author, Param, How, What, Group
-from VOEventLib.VOEvent import Citations, EventIVORN
-#from VOEventLib.VOEvent import Why
-#from VOEventLib.Vutil import makeWhereWhen, stringVOEvent
-from VOEventLib.Vutil import stringVOEvent
-
-from VOEventLib.VOEvent import AstroCoords, AstroCoordSystem
-from VOEventLib.VOEvent import ObservationLocation, ObservatoryLocation
-from VOEventLib.VOEvent import ObsDataLocation, WhereWhen
-from VOEventLib.VOEvent import Time, TimeInstant
-
-from core.time_utils import gpsToUtc
-from core.urls import build_absolute_uri
-from django.utils import timezone
-from django.conf import settings
-from django.urls import reverse
-from .models import CoincInspiralEvent, MultiBurstEvent
-from .models import VOEvent as GraceDBVOEvent
-from .models import LalInferenceBurstEvent
-
-import os
-
-class VOEventBuilderException(Exception):
- pass
-
-# Used to create the Packet_Type parameter block
-PACKET_TYPES = {
- GraceDBVOEvent.VOEVENT_TYPE_PRELIMINARY: (150, 'LVC_PRELIMINARY'),
- GraceDBVOEvent.VOEVENT_TYPE_INITIAL: (151, 'LVC_INITIAL'),
- GraceDBVOEvent.VOEVENT_TYPE_UPDATE: (152, 'LVC_UPDATE'),
- GraceDBVOEvent.VOEVENT_TYPE_RETRACTION: (164, 'LVC_RETRACTION'),
-}
-
-VOEVENT_TYPE_DICT = dict(GraceDBVOEvent.VOEVENT_TYPE_CHOICES)
-
-def get_voevent_type(short_name):
- for t in GraceDBVOEvent.VOEVENT_TYPE_CHOICES:
- if short_name in t:
- return t[1]
- return None
-
-def buildVOEvent(event, voevent, request=None):
-
-# XXX Branson commenting out. Reed's MDC events do not have FAR for some reason.
-# if not event.far:
-# raise VOEventBuilderException("Cannot build a VOEvent because event has no FAR.")
-
- if not event.gpstime:
- raise VOEventBuilderException("Cannot build a VOEvent because event has no gpstime.")
-
- if not voevent.voevent_type in VOEVENT_TYPE_DICT:
- raise VOEventBuilderException("voevent_type must be preliminary, initial, update, or retraction")
-
- # Let's convert that voevent_type to something nicer looking
- voevent_type = VOEVENT_TYPE_DICT[voevent.voevent_type]
-
- objid = event.graceid
-
- # Now build the IVORN.
- type_string = voevent_type.capitalize()
- event_id = "%s-%d-%s" % (objid, voevent.N, type_string)
- ivorn = settings.IVORN_PREFIX + event_id
-
- ############ VOEvent header ############################
- v = VOEvent(version="2.0")
- v.set_ivorn(ivorn)
-
- if event.search and event.search.name == 'MDC':
- v.set_role("test")
- elif event.group.name == 'Test':
- v.set_role("test")
- else:
- v.set_role("observation")
- if voevent_type != 'retraction':
- v.set_Description(settings.SKYALERT_DESCRIPTION)
-
- ############ Who ############################
- w = Who()
- a = Author()
- a.add_contactName("LIGO Scientific Collaboration and Virgo Collaboration")
- #a.add_contactEmail("postmaster@ligo.org")
- w.set_Author(a)
- w.set_Date(timezone.now().strftime("%Y-%m-%dT%H:%M:%S"))
- v.set_Who(w)
-
- ############ Why ############################
- # Moving this information into the 'How' section.
- #if voevent_type != 'retraction':
- # y = Why()
- # y.add_Description("Candidate gravitational wave event identified by low-latency analysis")
- # v.set_Why(y)
-
- ############ How ############################
-
- if voevent_type != 'retraction':
- h = How()
- h.add_Description("Candidate gravitational wave event identified by low-latency analysis")
- instruments = event.instruments.split(',')
- if 'H1' in instruments:
- h.add_Description("H1: LIGO Hanford 4 km gravitational wave detector")
- if 'L1' in instruments:
- h.add_Description("L1: LIGO Livingston 4 km gravitational wave detector")
- if 'V1' in instruments:
- h.add_Description("V1: Virgo 3 km gravitational wave detector")
- if int(voevent.coinc_comment) == 1:
- h.add_Description("A gravitational wave trigger identified a possible counterpart GRB")
- v.set_How(h)
-
- ############ What ############################
- w = What()
-
- # UCD = Unified Content Descriptors
- # http://monet.uni-sw.gwdg.de/twiki/bin/view/VOEvent/UnifiedContentDescriptors
- # OR -- (from VOTable document, [21] below)
- # http://www.ivoa.net/twiki/bin/view/IVOA/IvoaUCD
- # http://cds.u-strasbg.fr/doc/UCD.htx
- #
- # which somehow gets you to: http://www.ivoa.net/Documents/REC/UCD/UCDlist-20070402.html
- # where you might find some actual information.
-
- # Unit / Section 4.3 of [21] which relies on [25]
- # [21] http://www.ivoa.net/Documents/latest/VOT.html
- # [25] http://vizier.u-strasbg.fr/doc/catstd-3.2.htx
- #
- # basically, a string that makes sense to humans about what units a value is. eg. "m/s"
-
- # Add Packet_Type for GCNs
- w.add_Param(Param(name="Packet_Type",
- value=PACKET_TYPES[voevent.voevent_type][0], dataType="int",
- Description=[("The Notice Type number is assigned/used within GCN, eg "
- "type={typenum} is an {typedesc} notice").format(
- typenum=PACKET_TYPES[voevent.voevent_type][0],
- typedesc=PACKET_TYPES[voevent.voevent_type][1])]))
-
- # Whether the alert is internal or not
- w.add_Param(Param(name="internal", value=int(voevent.internal),
- dataType="int", Description=['Indicates whether this event should be '
- 'distributed to LSC/Virgo members only']))
-
- # The serial number
- w.add_Param(Param(name="Pkt_Ser_Num", value=voevent.N,
- Description=["A number that increments by 1 each time a new revision "
- "is issued for this event"]))
-
- # The GraceID
- w.add_Param(Param(name="GraceID",
- dataType="string",
- ucd="meta.id",
- value=objid,
- Description=["Identifier in GraceDB"]))
-
- # Alert type parameter
- w.add_Param(Param(name="AlertType",
- dataType="string",
- ucd="meta.version",
- value = voevent_type.capitalize(),
- Description=["VOEvent alert type"]))
-
- # Shib protected event page
- # Whether the event is a hardware injection or not
- w.add_Param(Param(name="HardwareInj",
- dataType="int",
- ucd="meta.number",
- value=int(voevent.hardware_inj),
- Description=['Indicates that this event is a hardware injection if 1, no if 0']))
-
- w.add_Param(Param(name="OpenAlert",
- dataType="int",
- ucd="meta.number",
- value=int(voevent.open_alert),
- Description=['Indicates that this event is an open alert if 1, no if 0']))
-
- w.add_Param(Param(name="EventPage",
- ucd="meta.ref.url",
- value=build_absolute_uri(reverse('view', args=[objid]), request),
- Description=["Web page for evolving status of this candidate event"]))
-
- if voevent_type != 'retraction':
- # Instruments
- w.add_Param(Param(name="Instruments",
- dataType="string",
- ucd="meta.code",
- value=event.instruments,
- Description=["List of instruments used in analysis to identify this event"]))
-
- # False alarm rate
- if event.far:
- w.add_Param(Param(name="FAR",
- dataType="float",
- ucd="arith.rate;stat.falsealarm",
- unit="Hz",
- value=float(max(event.far, settings.VOEVENT_FAR_FLOOR)),
- Description=["False alarm rate for GW candidates with this strength or greater"]))
-
- # Group
- w.add_Param(Param(name="Group",
- dataType="string",
- ucd="meta.code",
- value=event.group.name,
- Description=["Data analysis working group"]))
-
- # Pipeline
- w.add_Param(Param(name="Pipeline",
- dataType="string",
- ucd="meta.code",
- value=event.pipeline.name,
- Description=["Low-latency data analysis pipeline"]))
-
- # Search
- if event.search:
- w.add_Param(Param(name="Search",
- ucd="meta.code",
- dataType="string",
- value=event.search.name,
- Description=["Specific low-latency search"]))
-
- # initial and update VOEvents must have a skymap.
- # new feature (10/24/5/2016): preliminary VOEvents can have a skymap,
- # but they don't have to.
- if (voevent_type in ["initial", "update"] or
- (voevent_type == "preliminary" and voevent.skymap_filename != None)):
- if not voevent.skymap_filename:
- raise VOEventBuilderException("Skymap filename not provided.")
-
- fits_name = voevent.skymap_filename
- fits_path = os.path.join(event.datadir, fits_name)
- if not os.path.exists(fits_path):
- raise VOEventBuilderException("Skymap file does not exist: %s" % voevent.skymap_filename)
-
- if not voevent.skymap_type:
- raise VOEventBuilderException("Skymap type must be provided.")
-
- # Skymaps. Create group and set fits file name
- g = Group('GW_SKYMAP', voevent.skymap_type)
-
- fits_skymap_url = build_absolute_uri(reverse(
- "api:default:events:files", args=[objid, fits_name]), request)
-
- # Add parameters to the skymap group
- g.add_Param(Param(name="skymap_fits", dataType="string",
- ucd="meta.ref.url", value=fits_skymap_url,
- Description=["Sky Map FITS"]))
-
- w.add_Group(g)
-
- # Analysis specific attributes
- if voevent_type != 'retraction':
- classification_group = Group('Classification', Description=["Source "
- "classification: binary neutron star (BNS), neutron star-black "
- "hole (NSBH), binary black hole (BBH), MassGap, or terrestrial "
- "(noise)"])
- properties_group = Group('Properties', Description=["Qualitative "
- "properties of the source, conditioned on the assumption that the "
- "signal is an astrophysical compact binary merger"])
- if isinstance(event,CoincInspiralEvent) and voevent_type != 'retraction':
- # get mchirp and mass
- mchirp = float(event.mchirp)
- mass = float(event.mass)
- # calculate eta = (mchirp/total_mass)**(5/3)
- eta = pow((mchirp/mass),5.0/3.0)
-
- # EM-Bright mass classifier information for CBC event candidates
- if voevent.prob_bns is not None:
- classification_group.add_Param(Param(name="BNS",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_bns, Description=["Probability that "
- "the source is a binary neutron star merger (both objects "
- "lighter than 3 solar masses)"]))
-
- if voevent.prob_nsbh is not None:
- classification_group.add_Param(Param(name="NSBH",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_nsbh, Description=["Probability that "
- "the source is a neutron star-black hole merger (primary "
- "heavier than 5 solar masses, secondary lighter than 3 "
- "solar masses)"]))
-
- if voevent.prob_bbh is not None:
- classification_group.add_Param(Param(name="BBH",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_bbh, Description=["Probability that "
- "the source is a binary black hole merger (both objects "
- "heavier than 5 solar masses)"]))
-
- if voevent.prob_mass_gap is not None:
- classification_group.add_Param(Param(name="MassGap",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_mass_gap, Description=["Probability "
- "that the source has at least one object between 3 and 5 "
- "solar masses"]))
-
- if voevent.prob_terrestrial is not None:
- classification_group.add_Param(Param(name="Terrestrial",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_terrestrial, Description=["Probability "
- "that the source is terrestrial (i.e., a background noise "
- "fluctuation or a glitch)"]))
-
- # Add to source properties group
- if voevent.prob_has_ns is not None:
- properties_group.add_Param(Param(name="HasNS",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_has_ns,
- Description=["Probability that at least one object in the "
- "binary has a mass that is less than 3 solar masses"]))
-
- if voevent.prob_has_remnant is not None:
- properties_group.add_Param(Param(name="HasRemnant",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_has_remnant, Description=["Probability "
- "that a nonzero mass was ejected outside the central "
- "remnant object"]))
-
- # build up MaxDistance. event.singleinspiral_set.all()?
- # Each detector calculates an effective distance assuming the inspiral is
- # optimally oriented. It is the maximum distance at which a source of the
- # given parameters would've been seen by that particular detector. To get
- # an effective 'maximum distance', we just find the minumum over detectors
- max_distance = float('inf')
- for obj in event.singleinspiral_set.all():
- if obj.eff_distance < max_distance:
- max_distance = obj.eff_distance
-# if max_distance < float('inf'):
-# w.add_Param(Param(name="MaxDistance",
-# dataType="float",
-# ucd="pos.distance",
-# unit="Mpc",
-# value=max_distance,
-# Description=["Estimated maximum distance for CBC event"]))
-
- elif isinstance(event,MultiBurstEvent):
- w.add_Param(Param(name="CentralFreq",
- dataType="float",
- ucd="gw.frequency",
- unit="Hz",
- value=float(event.central_freq),
- Description=["Central frequency of GW burst signal"]))
- w.add_Param(Param(name="Duration",
- dataType="float",
- ucd="time.duration",
- unit="s",
- value=float(event.duration),
- Description=["Measured duration of GW burst signal"]))
-
- # XXX Calculate the fluence. Unfortunately, this requires parsing the trigger.txt
- # file for hrss values. These should probably be pulled into the database.
- # But there is no consensus on whether hrss or fluence is meaningful. So I will
- # put off changing the schema for now.
- try:
- # Go find the data file.
- log = event.eventlog_set.filter(comment__startswith="Original Data").all()[0]
- filename = log.filename
- filepath = os.path.join(event.datadir,filename)
- if os.path.isfile(filepath):
- datafile = open(filepath,"r")
- else:
- raise Exception("No file found.")
- # Now parse the datafile.
- # The line we want looks like:
- # hrss: 1.752741e-23 2.101590e-23 6.418900e-23
- for line in datafile:
- if line.startswith('hrss:'):
- hrss_values = [float(hrss) for hrss in line.split()[1:]]
- max_hrss = max(hrss_values)
- # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
- # Note that hrss here actually has units of s^(-1/2)
- fluence = pi * pow(c,3) * pow(event.central_freq,2)
- fluence = fluence * pow(max_hrss,2)
- fluence = fluence / (4.0*G)
-
- w.add_Param(Param(name="Fluence",
- dataType="float",
- ucd="gw.fluence",
- unit="erg/cm^2",
- value=fluence,
- Description=["Estimated fluence of GW burst signal"]))
- except Exception:
- pass
- elif isinstance(event,LalInferenceBurstEvent):
- w.add_Param(Param(name="frequency",
- dataType="float",
- ucd="gw.frequency",
- unit="Hz",
- value=float(event.frequency_mean),
- Description=["Mean frequency of GW burst signal"]))
-
- # Calculate the fluence.
- # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
- # Note that hrss here actually has units of s^(-1/2)
- # XXX obviously need to refactor here.
- try:
- fluence = pi * pow(c,3) * pow(event.frequency,2)
- fluence = fluence * pow(event.hrss,2)
- fluence = fluence / (4.0*G)
-
- w.add_Param(Param(name="Fluence",
- dataType="float",
- ucd="gw.fluence",
- unit="erg/cm^2",
- value=fluence,
- Description=["Estimated fluence of GW burst signal"]))
- except:
- pass
- else:
- pass
- # Add Groups to What block
- w.add_Group(classification_group)
- w.add_Group(properties_group)
-
- v.set_What(w)
-
- ############ Wherewhen ############################
-# The old way of making the WhereWhen section led to a pointless position
-# location.
-# wwd = {'observatory': 'LIGO Virgo',
-# 'coord_system': 'UTC-FK5-GEO',
-# # XXX time format
-# 'time': str(gpsToUtc(event.gpstime).isoformat())[:-6], #'1918-11-11T11:11:11',
-# #'timeError': 1.0,
-# 'longitude': 0.0,
-# 'latitude': 0.0,
-# 'positionalError': 180.0,
-# }
-#
-# ww = makeWhereWhen(wwd)
-# if ww: v.set_WhereWhen(ww)
-
- coord_system_id = 'UTC-FK5-GEO'
- event_time = str(gpsToUtc(event.gpstime).isoformat())[:-6]
- observatory_id = 'LIGO Virgo'
- ac = AstroCoords(coord_system_id=coord_system_id)
- acs = AstroCoordSystem(id=coord_system_id)
- ac.set_Time(Time(TimeInstant = TimeInstant(event_time)))
-
- onl = ObservationLocation(acs, ac)
- oyl = ObservatoryLocation(id=observatory_id)
- odl = ObsDataLocation(oyl, onl)
- ww = WhereWhen()
- ww.set_ObsDataLocation(odl)
- v.set_WhereWhen(ww)
-
- ############ Citation ############################
- if event.voevent_set.count()>1:
- c = Citations()
- for ve in event.voevent_set.all():
- # Oh, actually we need to exclude *this* voevent.
- if ve.N == voevent.N:
- continue
- if voevent_type == 'initial':
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Initial localization is now available')
- elif voevent_type == 'update':
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Updated localization is now available')
- elif voevent_type == 'retraction':
- ei = EventIVORN('retraction', ve.ivorn)
- c.set_Description('Determined to not be a viable GW event candidate')
- elif voevent_type == 'preliminary':
- # For cases when an additional preliminary VOEvent is sent
- # in order to add a preliminary skymap.
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Initial localization is now available (preliminary)')
- c.add_EventIVORN(ei)
-
- v.set_Citations(c)
-
- ############ output the event ############################
- xml = stringVOEvent(v)
- #schemaURL = "http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd")
- return xml, ivorn
-
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index 05f6ed93d..a36143cc7 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -1012,6 +1012,9 @@ class VOEventBase(CleanSaveModel):
# Override this method on derived classes
return NotImplemented
+ class VOEventBuilderException(Exception):
+ pass
+
class VOEvent(VOEventBase, AutoIncrementModel):
"""VOEvent class for events"""
diff --git a/gracedb/superevents/buildVOEvent.py b/gracedb/superevents/buildVOEvent.py
deleted file mode 100644
index 10bbdaaa6..000000000
--- a/gracedb/superevents/buildVOEvent.py
+++ /dev/null
@@ -1,456 +0,0 @@
-
-# Taken from VOEventLib example code, which is:
-# Copyright 2010 Roy D. Williams
-# then modified
-"""
-buildVOEvent: Creates a complex VOEvent with tables
-See the VOEvent specification for details
-http://www.ivoa.net/Documents/latest/VOEvent.html
-"""
-from scipy.constants import c, G, pi
-
-
-from VOEventLib.VOEvent import VOEvent, Who, Author, Param, How, What, Group
-from VOEventLib.VOEvent import Citations, EventIVORN
-from VOEventLib.Vutil import stringVOEvent
-
-from VOEventLib.VOEvent import AstroCoords, AstroCoordSystem
-from VOEventLib.VOEvent import ObservationLocation, ObservatoryLocation
-from VOEventLib.VOEvent import ObsDataLocation, WhereWhen
-from VOEventLib.VOEvent import Time, TimeInstant
-
-from core.urls import build_absolute_uri
-from core.time_utils import gpsToUtc
-from django.conf import settings
-from django.urls import reverse
-from django.utils import timezone
-from django.db.models import Min
-from events.models import CoincInspiralEvent, MultiBurstEvent, \
- LalInferenceBurstEvent
-from .models import VOEvent as GraceDBVOEvent
-
-import os
-import logging
-logger = logging.getLogger(__name__)
-
-VOEVENT_TYPE_DICT = dict(GraceDBVOEvent.VOEVENT_TYPE_CHOICES)
-
-class VOEventBuilderException(Exception):
- pass
-
-# Used to create the Packet_Type parameter block
-PACKET_TYPES = {
- GraceDBVOEvent.VOEVENT_TYPE_PRELIMINARY: (150, 'LVC_PRELIMINARY'),
- GraceDBVOEvent.VOEVENT_TYPE_INITIAL: (151, 'LVC_INITIAL'),
- GraceDBVOEvent.VOEVENT_TYPE_UPDATE: (152, 'LVC_UPDATE'),
- GraceDBVOEvent.VOEVENT_TYPE_RETRACTION: (164, 'LVC_RETRACTION'),
-}
-
-def get_voevent_type(short_name):
- for t in GraceDBVOEvent.VOEVENT_TYPE_CHOICES:
- if short_name in t:
- return t[1]
- return None
-
-
-def construct_voevent_file(superevent, voevent, request=None):
-
- # Set preferred_event as event to be used in most of this
- # Get the event subclass (CoincInspiralEvent, MultiBurstEvent, etc.) and
- # set that as the event
- event = superevent.preferred_event.get_subclass_or_self()
-
- # Let's convert that voevent_type to something nicer looking
- voevent_type = VOEVENT_TYPE_DICT[voevent.voevent_type]
-
- # Now build the IVORN.
- type_string = voevent_type.capitalize()
- voevent_id = '{s_id}-{N}-{type_str}'.format(type_str=type_string,
- s_id=superevent.default_superevent_id, N=voevent.N)
- ivorn = settings.IVORN_PREFIX + voevent_id
-
- ############ VOEvent header ############################
- v = VOEvent(version="2.0")
- v.set_ivorn(ivorn)
-
- if event.search and event.search.name == 'MDC':
- v.set_role("test")
- elif event.group.name == 'Test':
- v.set_role("test")
- else:
- v.set_role("observation")
- if voevent_type != 'retraction':
- v.set_Description(settings.SKYALERT_DESCRIPTION)
-
- ############ Who ############################
- w = Who()
- a = Author()
- a.add_contactName("LIGO Scientific Collaboration and Virgo Collaboration")
- #a.add_contactEmail("postmaster@ligo.org")
- w.set_Author(a)
- w.set_Date(timezone.now().strftime("%Y-%m-%dT%H:%M:%S"))
- v.set_Who(w)
-
- ############ How ############################
-
- if voevent_type != 'retraction':
- h = How()
- h.add_Description("Candidate gravitational wave event identified by low-latency analysis")
- instruments = event.instruments.split(',')
- if 'H1' in instruments:
- h.add_Description("H1: LIGO Hanford 4 km gravitational wave detector")
- if 'L1' in instruments:
- h.add_Description("L1: LIGO Livingston 4 km gravitational wave detector")
- if 'V1' in instruments:
- h.add_Description("V1: Virgo 3 km gravitational wave detector")
- if voevent.coinc_comment:
- h.add_Description("A gravitational wave trigger identified a possible counterpart GRB")
- v.set_How(h)
-
- ############ What ############################
- w = What()
-
- # UCD = Unified Content Descriptors
- # http://monet.uni-sw.gwdg.de/twiki/bin/view/VOEvent/UnifiedContentDescriptors
- # OR -- (from VOTable document, [21] below)
- # http://www.ivoa.net/twiki/bin/view/IVOA/IvoaUCD
- # http://cds.u-strasbg.fr/doc/UCD.htx
- #
- # which somehow gets you to: http://www.ivoa.net/Documents/REC/UCD/UCDlist-20070402.html
- # where you might find some actual information.
-
- # Unit / Section 4.3 of [21] which relies on [25]
- # [21] http://www.ivoa.net/Documents/latest/VOT.html
- # [25] http://vizier.u-strasbg.fr/doc/catstd-3.2.htx
- #
- # basically, a string that makes sense to humans about what units a value is. eg. "m/s"
-
- # Add Packet_Type for GCNs
- w.add_Param(Param(name="Packet_Type",
- value=PACKET_TYPES[voevent.voevent_type][0], dataType="int",
- Description=[("The Notice Type number is assigned/used within GCN, eg "
- "type={typenum} is an {typedesc} notice").format(
- typenum=PACKET_TYPES[voevent.voevent_type][0],
- typedesc=PACKET_TYPES[voevent.voevent_type][1])]))
-
- # Whether the alert is internal or not
- w.add_Param(Param(name="internal", value=int(voevent.internal),
- dataType="int", Description=['Indicates whether this event should be '
- 'distributed to LSC/Virgo members only']))
-
- # The serial number
- w.add_Param(Param(name="Pkt_Ser_Num", value=voevent.N,
- Description=["A number that increments by 1 each time a new revision "
- "is issued for this event"]))
-
- # The superevent ID
- w.add_Param(Param(name="GraceID",
- dataType="string",
- ucd="meta.id",
- value=superevent.default_superevent_id,
- Description=["Identifier in GraceDB"]))
-
- # Alert type parameter
- w.add_Param(Param(name="AlertType",
- dataType="string",
- ucd="meta.version",
- value = voevent_type.capitalize(),
- Description=["VOEvent alert type"]))
-
- # Whether the event is a hardware injection or not
- w.add_Param(Param(name="HardwareInj",
- dataType="int",
- ucd="meta.number",
- value=int(voevent.hardware_inj),
- Description=['Indicates that this event is a hardware injection if 1, no if 0']))
-
- w.add_Param(Param(name="OpenAlert",
- dataType="int",
- ucd="meta.number",
- value=int(voevent.open_alert),
- Description=['Indicates that this event is an open alert if 1, no if 0']))
-
- # Superevent page
- w.add_Param(Param(name="EventPage",
- ucd="meta.ref.url",
- value=build_absolute_uri(reverse("superevents:view",
- args=[superevent.default_superevent_id]), request),
- Description=["Web page for evolving status of this GW candidate"]))
-
- if voevent_type != 'retraction':
- # Instruments
- w.add_Param(Param(name="Instruments",
- dataType="string",
- ucd="meta.code",
- value=event.instruments,
- Description=["List of instruments used in analysis to identify this event"]))
-
- # False alarm rate
- if event.far:
- w.add_Param(Param(name="FAR",
- dataType="float",
- ucd="arith.rate;stat.falsealarm",
- unit="Hz",
- value=float(max(event.far, settings.VOEVENT_FAR_FLOOR)),
- Description=["False alarm rate for GW candidates with this strength or greater"]))
-
- # Group
- w.add_Param(Param(name="Group",
- dataType="string",
- ucd="meta.code",
- value=event.group.name,
- Description=["Data analysis working group"]))
-
- # Pipeline
- w.add_Param(Param(name="Pipeline",
- dataType="string",
- ucd="meta.code",
- value=event.pipeline.name,
- Description=["Low-latency data analysis pipeline"]))
-
- # Search
- if event.search:
- w.add_Param(Param(name="Search",
- ucd="meta.code",
- dataType="string",
- value=event.search.name,
- Description=["Specific low-latency search"]))
-
- # initial and update VOEvents must have a skymap.
- # new feature (10/24/2016): preliminary VOEvents can have a skymap,
- # but they don't have to.
- if (voevent_type in ["initial", "update"] or
- (voevent_type == "preliminary" and voevent.skymap_filename != None)):
-
- # Skymaps. Create group and set fits file name
- g = Group('GW_SKYMAP', voevent.skymap_type)
-
- fits_skymap_url = build_absolute_uri(reverse(
- "api:default:superevents:superevent-file-detail",
- args=[superevent.default_superevent_id, voevent.skymap_filename]),
- request)
-
- # Add parameters to the skymap group
- g.add_Param(Param(name="skymap_fits", dataType="string",
- ucd="meta.ref.url", value=fits_skymap_url,
- Description=["Sky Map FITS"]))
-
- w.add_Group(g)
-
- # Analysis specific attributes
- if voevent_type != 'retraction':
- classification_group = Group('Classification', Description=["Source "
- "classification: binary neutron star (BNS), neutron star-black "
- "hole (NSBH), binary black hole (BBH), MassGap, or terrestrial "
- "(noise)"])
- properties_group = Group('Properties', Description=["Qualitative "
- "properties of the source, conditioned on the assumption that the "
- "signal is an astrophysical compact binary merger"])
- if isinstance(event, CoincInspiralEvent) and voevent_type != 'retraction':
- # get mchirp and mass
- mchirp = float(event.mchirp)
- mass = float(event.mass)
- # calculate eta = (mchirp/total_mass)**(5/3)
- eta = pow((mchirp/mass),5.0/3.0)
-
- # EM-Bright mass classifier information for CBC event candidates
- if voevent.prob_bns is not None:
- classification_group.add_Param(Param(name="BNS",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_bns, Description=["Probability that "
- "the source is a binary neutron star merger (both objects "
- "lighter than 3 solar masses)"]))
-
- if voevent.prob_nsbh is not None:
- classification_group.add_Param(Param(name="NSBH",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_nsbh, Description=["Probability that "
- "the source is a neutron star-black hole merger (primary "
- "heavier than 5 solar masses, secondary lighter than 3 "
- "solar masses)"]))
-
- if voevent.prob_bbh is not None:
- classification_group.add_Param(Param(name="BBH",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_bbh, Description=["Probability that "
- "the source is a binary black hole merger (both objects "
- "heavier than 5 solar masses)"]))
-
- if voevent.prob_mass_gap is not None:
- classification_group.add_Param(Param(name="MassGap",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_mass_gap,
- Description=["Probability that the source has at least "
- "one object between 3 and 5 solar masses"]))
-
- if voevent.prob_terrestrial is not None:
- classification_group.add_Param(Param(name="Terrestrial",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_terrestrial, Description=["Probability "
- "that the source is terrestrial (i.e., a background noise "
- "fluctuation or a glitch)"]))
-
- # Add to source properties group
- if voevent.prob_has_ns is not None:
- properties_group.add_Param(Param(name="HasNS",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_has_ns,
- Description=["Probability that at least one object in the "
- "binary has a mass that is less than 3 solar masses"]))
-
- if voevent.prob_has_remnant is not None:
- properties_group.add_Param(Param(name="HasRemnant",
- dataType="float", ucd="stat.probability",
- value=voevent.prob_has_remnant, Description=["Probability "
- "that a nonzero mass was ejected outside the central "
- "remnant object"]))
-
- # build up MaxDistance. event.singleinspiral_set.all()?
- # Each detector calculates an effective distance assuming the inspiral is
- # optimally oriented. It is the maximum distance at which a source of the
- # given parameters would've been seen by that particular detector. To get
- # an effective 'maximum distance', we just find the minumum over detectors
- max_distance = event.singleinspiral_set.all().aggregate(
- max_dist=Min('eff_distance'))
- max_distance = max_distance or float('inf')
-
- elif isinstance(event, MultiBurstEvent):
- w.add_Param(Param(name="CentralFreq",
- dataType="float",
- ucd="gw.frequency",
- unit="Hz",
- value=float(event.central_freq),
- Description=["Central frequency of GW burst signal"]))
- w.add_Param(Param(name="Duration",
- dataType="float",
- ucd="time.duration",
- unit="s",
- value=float(event.duration),
- Description=["Measured duration of GW burst signal"]))
-
- # XXX Calculate the fluence. Unfortunately, this requires parsing the trigger.txt
- # file for hrss values. These should probably be pulled into the database.
- # But there is no consensus on whether hrss or fluence is meaningful. So I will
- # put off changing the schema for now.
- try:
- # Go find the data file.
- log = event.eventlog_set.filter(comment__startswith="Original Data").all()[0]
- filename = log.filename
- filepath = os.path.join(event.datadir,filename)
- if os.path.isfile(filepath):
- datafile = open(filepath,"r")
- else:
- raise VOEventBuilderException("No file found.")
- # Now parse the datafile.
- # The line we want looks like:
- # hrss: 1.752741e-23 2.101590e-23 6.418900e-23
- for line in datafile:
- if line.startswith('hrss:'):
- hrss_values = [float(hrss) for hrss in line.split()[1:]]
- max_hrss = max(hrss_values)
- # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
- # Note that hrss here actually has units of s^(-1/2)
- fluence = pi * pow(c,3) * pow(event.central_freq,2)
- fluence = fluence * pow(max_hrss,2)
- fluence = fluence / (4.0*G)
-
- w.add_Param(Param(name="Fluence",
- dataType="float",
- ucd="gw.fluence",
- unit="erg/cm^2",
- value=fluence,
- Description=["Estimated fluence of GW burst signal"]))
- except Exception as e:
- logger.exception(e)
-
- elif isinstance(event, LalInferenceBurstEvent):
- w.add_Param(Param(name="frequency",
- dataType="float",
- ucd="gw.frequency",
- unit="Hz",
- value=float(event.frequency_mean),
- Description=["Mean frequency of GW burst signal"]))
-
- # Calculate the fluence.
- # From Min-A Cho: fluence = pi*(c**3)*(freq**2)*(hrss_max**2)*(10**3)/(4*G)
- # Note that hrss here actually has units of s^(-1/2)
- # XXX obviously need to refactor here.
- try:
- fluence = pi * pow(c,3) * pow(event.frequency,2)
- fluence = fluence * pow(event.hrss,2)
- fluence = fluence / (4.0*G)
-
- w.add_Param(Param(name="Fluence",
- dataType="float",
- ucd="gw.fluence",
- unit="erg/cm^2",
- value=fluence,
- Description=["Estimated fluence of GW burst signal"]))
- except Exception as e:
- logger.exception(e)
-
- # Add Groups to What block
- w.add_Group(classification_group)
- w.add_Group(properties_group)
-
- v.set_What(w)
-
- ############ Wherewhen ############################
-# The old way of making the WhereWhen section led to a pointless position
-# location.
-# wwd = {'observatory': 'LIGO Virgo',
-# 'coord_system': 'UTC-FK5-GEO',
-# # XXX time format
-# 'time': str(gpsToUtc(event.gpstime).isoformat())[:-6], #'1918-11-11T11:11:11',
-# #'timeError': 1.0,
-# 'longitude': 0.0,
-# 'latitude': 0.0,
-# 'positionalError': 180.0,
-# }
-#
-# ww = makeWhereWhen(wwd)
-# if ww: v.set_WhereWhen(ww)
-
- coord_system_id = 'UTC-FK5-GEO'
- event_time = str(gpsToUtc(event.gpstime).isoformat())[:-6]
- observatory_id = 'LIGO Virgo'
- ac = AstroCoords(coord_system_id=coord_system_id)
- acs = AstroCoordSystem(id=coord_system_id)
- ac.set_Time(Time(TimeInstant = TimeInstant(event_time)))
-
- onl = ObservationLocation(acs, ac)
- oyl = ObservatoryLocation(id=observatory_id)
- odl = ObsDataLocation(oyl, onl)
- ww = WhereWhen()
- ww.set_ObsDataLocation(odl)
- v.set_WhereWhen(ww)
-
- ############ Citation ############################
- if superevent.voevent_set.count() > 1:
- c = Citations()
- for ve in superevent.voevent_set.all():
- # Oh, actually we need to exclude *this* voevent.
- if ve.N == voevent.N:
- continue
- if voevent_type == 'initial':
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Initial localization is now available')
- elif voevent_type == 'update':
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Updated localization is now available')
- elif voevent_type == 'retraction':
- ei = EventIVORN('retraction', ve.ivorn)
- c.set_Description('Determined to not be a viable GW event candidate')
- elif voevent_type == 'preliminary':
- # For cases when an additional preliminary VOEvent is sent
- # in order to add a preliminary skymap.
- ei = EventIVORN('supersedes', ve.ivorn)
- c.set_Description('Initial localization is now available (preliminary)')
- c.add_EventIVORN(ei)
-
- v.set_Citations(c)
-
- ############ output the event ############################
- xml = stringVOEvent(v)
- #schemaURL = "http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd")
- return xml, ivorn
diff --git a/gracedb/superevents/utils.py b/gracedb/superevents/utils.py
index 57a73d836..d520a6c7c 100644
--- a/gracedb/superevents/utils.py
+++ b/gracedb/superevents/utils.py
@@ -6,7 +6,6 @@ from django.http import Http404
from django.shortcuts import get_object_or_404
from django.contrib.auth.models import Group as DjangoGroup
-from .buildVOEvent import construct_voevent_file
from .models import Superevent, Log, Labelling, EMObservation, EMFootprint, \
VOEvent, Signoff
from .shortcuts import is_superevent
@@ -15,6 +14,7 @@ from alerts.issuers.superevents import SupereventAlertIssuer, \
SupereventLogAlertIssuer, SupereventLabelAlertIssuer, \
SupereventVOEventAlertIssuer, SupereventEMObservationAlertIssuer, \
SupereventSignoffAlertIssuer, SupereventPermissionsAlertIssuer
+from annotations.voevent_utils import construct_voevent_file
from core.permissions import expose_log_to_lvem, expose_log_to_public, \
hide_log_from_lvem, hide_log_from_public, assign_perms_to_obj, \
remove_perms_from_obj
diff --git a/requirements.txt b/requirements.txt
index 6209a0bee..d9415c0e5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -36,7 +36,7 @@ service_identity==17.0.0
simplejson==3.15.0
Sphinx==1.7.0
twilio==6.10.3
-voeventlib==1.2; python_version >= '3'
+voevent-parse==1.0.3
# Do NOT upgrade pyparsing beyond 2.3.0 (even to 2.3.1) without carefully
# testing and modifying the search query code. There were a number of
# problematic API changes in 2.3.1 and beyond that will need to be handled
--
GitLab
From 70bbbde2c3a9935b878f9b1e6d6bc1d2498164d4 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 20 Sep 2019 12:03:33 -0500
Subject: [PATCH 043/106] Add newer version of numpy for compatibility with
voeventparse
This is necessary since we use --system-site-packages to create
the virtualenv, and the system version of numpy is too old.
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index d9415c0e5..98331bc6a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -26,6 +26,7 @@ lxml==4.2.0
matplotlib==2.0.0
mock==2.0.0
mysqlclient==1.4.2
+numpy==1.17.2
packaging==17.1
phonenumbers==8.8.11
python-ldap==3.1.0
--
GitLab
From 4b4c266f75119e6c7ab9277b00c7601d6041a645 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Tue, 17 Sep 2019 14:21:03 -0500
Subject: [PATCH 044/106] core.time_utils: fix up and add some time handling
utilities
---
gracedb/core/time_utils.py | 29 ++++++++++-------------------
1 file changed, 10 insertions(+), 19 deletions(-)
diff --git a/gracedb/core/time_utils.py b/gracedb/core/time_utils.py
index 713ec8f1d..360dec808 100644
--- a/gracedb/core/time_utils.py
+++ b/gracedb/core/time_utils.py
@@ -68,28 +68,10 @@ def gpsToUtc(gpsTime):
t = gpsToPosixTime(gpsTime)
return datetime.datetime.fromtimestamp(t, pytz.utc)
-def isoToGps(t):
- # The input is a string in ISO time format: 2012-10-28T05:04:31.91
- # First strip out whitespace, then split off the factional
- # second. We'll add that back later.
- if not t:
- return None
- t=t.strip()
- ISOTime = t.split('.')[0]
- ISOTime = datetime.datetime.strptime(ISOTime,"%Y-%m-%dT%H:%M:%S")
- # Need to set UTC time zone or this is interpreted as local time.
- ISOTime = ISOTime.replace(tzinfo=pytz.utc)
- sec_substr = t.split('.')[1]
- if sec_substr:
- fracSec = float('0.' + sec_substr)
- else:
- fracSec = 0
- posixTime = calendar.timegm(ISOTime.utctimetuple()) + fracSec
- return int(round(posixToGpsTime(posixTime)))
def isoToGpsFloat(t):
# The input is a string in ISO time format: 2012-10-28T05:04:31.91
- # First strip out whitespace, then split off the factional
+ # First strip out whitespace, then split off the fractional
# second. We'll add that back later.
if not t:
return None
@@ -105,3 +87,12 @@ def isoToGpsFloat(t):
fracSec = 0
posixTime = calendar.timegm(ISOTime.utctimetuple()) + fracSec
return posixToGpsTime(posixTime)
+
+
+def isoToGps(t):
+ return int(round(isoToGpsFloat(t)))
+
+
+def utc_datetime_to_gps_float(dt):
+ posix_time = calendar.timegm(dt.timetuple()) + (dt.microsecond * 1e-6)
+ return posixToGpsTime(posix_time)
--
GitLab
From c877bcfc2cca9a90b7d498600855080a0425023b Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Tue, 17 Sep 2019 14:21:19 -0500
Subject: [PATCH 045/106] events.translator: rework Fermi/Swift event parsing
Use voeventparse instead of VOEventLib
---
gracedb/events/translator.py | 94 +++++++++++++++++-------------------
1 file changed, 44 insertions(+), 50 deletions(-)
diff --git a/gracedb/events/translator.py b/gracedb/events/translator.py
index 64c5ee5a0..288637f58 100644
--- a/gracedb/events/translator.py
+++ b/gracedb/events/translator.py
@@ -1,31 +1,29 @@
-from math import isnan
+import json
+import logging
+from math import isnan, sqrt
import numbers
import os
-from .models import EventLog
-from .models import SingleInspiral
-from .models import SimInspiralEvent
-
from glue.ligolw.utils import load_filename, load_fileobj
from glue.ligolw.lsctables import CoincInspiralTable, SnglInspiralTable, use_in
from glue.ligolw.lsctables import SimInspiralTable, MultiBurstTable, CoincTable
from glue.ligolw.ligolw import LIGOLWContentHandler
+import voeventparse as vp
-from .serialize import populate_omega_tables, write_output_files
-
-from VOEventLib.Vutil import parse, getWhereWhen, findParam, getParamNames
-from core.time_utils import isoToGps, isoToGpsFloat
+from core.time_utils import utc_datetime_to_gps_float
from core.vfile import create_versioned_file
+from .models import EventLog
+from .models import SingleInspiral
+from .models import SimInspiralEvent
+from .serialize import populate_omega_tables, write_output_files
-import json
try:
from StringIO import StringIO
except ImportError: # python >= 3
from io import StringIO
-from math import sqrt
-import logging
+# Set up logger
logger = logging.getLogger(__name__)
use_in(LIGOLWContentHandler)
@@ -338,17 +336,7 @@ def handle_uploaded_data(event, datafilename,
elif pipeline in ['Swift', 'Fermi', 'SNEWS']:
# Get the event time from the VOEvent file
error = None
- try:
- #event.gpstime = getGpsFromVOEvent(datafilename)
- populateGrbEventFromVOEventFile(datafilename, event)
- except Exception as e:
- error = "Problem parsing VOEvent: %s" % e.__repr__()
- event.save()
- if error is not None:
- log = EventLog(event=event,
- issuer=event.submitter,
- comment=error)
- log.save()
+ populateGrbEventFromVOEventFile(datafilename, event)
elif pipeline == 'oLIB':
# lambda function for converting to a type if not None
typecast = lambda t, v: t(v) if v is not None else v
@@ -608,43 +596,46 @@ class CwbData(Translator):
def writeCoincFile(self, path):
pass
-def getGpsFromVOEvent(filename):
- v = parse(filename)
- wwd = getWhereWhen(v)
- gpstime = isoToGps(wwd['time'])
- return gpstime
def populateGrbEventFromVOEventFile(filename, event):
- v = parse(filename)
- wherewhen = getWhereWhen(v)
- event.gpstime = isoToGpsFloat(wherewhen['time'])
- event.ivorn = v.ivorn
-
- event.author_shortname = v.get_Who().Author.shortName[0]
- event.author_ivorn = v.get_Who().AuthorIVORN
-
- event.observatory_location_id = wherewhen['observatory']
- event.coord_system = wherewhen['coord_system']
- event.ra = wherewhen['longitude']
- event.dec = wherewhen['latitude']
- event.error_radius = wherewhen['positionalError']
-
- event.how_description = v.get_How().get_Description()[0]
- event.how_reference_url = v.get_How().get_Reference()[0].uri
-
- # try to find a trigger_duration value
+ # Load file into vp.Voevent instance
+ with open(filename, 'rb') as f:
+ v = vp.load(f)
+
+ # Get gpstime
+ utc_time = vp.convenience.get_event_time_as_utc(v)
+ gpstime = utc_datetime_to_gps_float(utc_time)
+
+ # Get event position
+ pos2d = vp.get_event_position(v)
+
+ # Assign information to event
+ event.gpstime = gpstime
+ event.ivorn = v.get('ivorn')
+ event.author_shortname = v.Who.Author.shortName
+ event.author_ivorn = v.Who.AuthorIVORN
+ event.observatory_location_id = \
+ v.WhereWhen.ObsDataLocation.ObservatoryLocation.get('id')
+ event.coord_system = pos2d.system
+ event.ra = pos2d.ra
+ event.dec = pos2d.dec
+ event.error_radius = pos2d.err
+ event.how_description = v.How.Description
+ event.how_reference_url = v.How.Reference.get('uri')
+
+ # Try to find a trigger_duration value
# Fermi uses Trig_Dur or Data_Integ, while Swift uses Integ_Time
# One or the other may be present, but not both
- VOEvent_params = [pn[1] for pn in getParamNames(v)]
+ VOEvent_params = vp.convenience.get_toplevel_params(v)
trig_dur_params = ["Trig_Dur", "Trans_Duration", "Data_Integ",
"Integ_Time", "Trig_Timescale"]
trigger_duration = None
for param in trig_dur_params:
if (param in VOEvent_params):
- trigger_duration = float(findParam(v, "", param).get_value())
+ trigger_duration = float(VOEvent_params.get(param).get('value'))
break
-
+
# Fermi GCNs (after the first one) often set Trig_Dur or Data_Integ
# to 0.000 (not sure why). We don't want to overwrite the currently
# existing value in the database with 0.000 if this has happened, so
@@ -657,6 +648,9 @@ def populateGrbEventFromVOEventFile(filename, event):
trigger_id_params = ['TrigID', 'Trans_Num', 'EventID']
for param in trigger_id_params:
if (param in VOEvent_params):
- trigger_id = findParam(v, "", param).get_value()
+ trigger_id = VOEvent_params.get(param).get('value')
break
event.trigger_id = trigger_id
+
+ # Save event
+ event.save()
--
GitLab
From 2f0f53f276e26ef16109fbe704fe1ebc9b7f29c1 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Wed, 18 Sep 2019 09:33:13 -0500
Subject: [PATCH 046/106] docker: convert containers to use Python 3
---
Dockerfile | 20 ++++++++++----------
docker/check_shibboleth_status | 2 +-
docker/cleanup | 6 +++---
docker/entrypoint | 2 +-
4 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index d349e3782..5ef4cf53d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
FROM ligo/base:stretch
LABEL name="LIGO GraceDB Django application" \
maintainer="tanner.prestegard@ligo.org" \
- date="20190430"
+ date="20190920"
ARG SETTINGS_MODULE="config.settings.container.dev"
COPY docker/SWITCHaai-swdistrib.gpg /etc/apt/trusted.gpg.d
@@ -24,11 +24,10 @@ RUN apt-get update && \
mariadb-client \
nodejs \
osg-ca-certs \
- python2.7 \
- python2.7-dev \
- python-libxml2 \
- python-pip \
- python-voeventlib \
+ python3.5 \
+ python3.5-dev \
+ python3-libxml2 \
+ python3-pip \
procps \
shibboleth \
supervisor \
@@ -61,11 +60,12 @@ ADD . /app/gracedb_project
# install gracedb application itself
WORKDIR /app/gracedb_project
RUN bower install --allow-root
-RUN pip install --upgrade setuptools wheel && \
- pip install -r requirements.txt
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools wheel && \
+ pip3 install -r requirements.txt
# Give pip-installed packages priority over distribution packages
-ENV PYTHONPATH /usr/local/lib/python2.7/dist-packages:$PYTHONPATH
+ENV PYTHONPATH /usr/local/lib/python3.5/dist-packages:$PYTHONPATH
ENV ENABLE_SHIBD false
ENV ENABLE_OVERSEER true
ENV VIRTUAL_ENV dummy
@@ -97,7 +97,7 @@ RUN DJANGO_SETTINGS_MODULE=${SETTINGS_MODULE} \
DJANGO_TWILIO_AUTH_TOKEN=fake_token \
AWS_SES_ACCESS_KEY_ID=fake_aws_id \
AWS_SES_SECRET_ACCESS_KEY=fake_aws_key \
- python manage.py collectstatic --noinput
+ python3 manage.py collectstatic --noinput
RUN rm -rf /app/logs/* /app/project_data/*
diff --git a/docker/check_shibboleth_status b/docker/check_shibboleth_status
index 0e30bfda2..71cfb0c0f 100644
--- a/docker/check_shibboleth_status
+++ b/docker/check_shibboleth_status
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
'''
Pulls Shibboleth status.sso page, checks for:
diff --git a/docker/cleanup b/docker/cleanup
index 5ccf6dc81..13e6994f8 100644
--- a/docker/cleanup
+++ b/docker/cleanup
@@ -1,5 +1,5 @@
#!/bin/sh
-python /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap people
-python /app/gracedb_project/manage.py remove_inactive_alerts
-python /app/gracedb_project/manage.py clearsessions
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap people
+python3 /app/gracedb_project/manage.py remove_inactive_alerts
+python3 /app/gracedb_project/manage.py clearsessions
diff --git a/docker/entrypoint b/docker/entrypoint
index 9d8be7681..ccc392808 100644
--- a/docker/entrypoint
+++ b/docker/entrypoint
@@ -1,4 +1,4 @@
#!/bin/bash
-export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python -c 'import uuid; print(uuid.uuid4().hex)')
+export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python3 -c 'import uuid; print(uuid.uuid4().hex)')
exec "$@"
--
GitLab
From 4c4f44262c5a1b243b80e3c631723b912dd184b4 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Wed, 18 Sep 2019 10:17:43 -0500
Subject: [PATCH 047/106] api.v1.events.views.EventList.get: bugfix integer
division calculation
---
gracedb/api/v1/events/views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index fbb41cde9..42f98171f 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -407,7 +407,7 @@ class EventList(InheritPermissionsAPIView):
d = {'error': 'Too many events.' }
return Response(d, status=status.HTTP_400_BAD_REQUEST)
- last = max(0, (numRows / count)) * count
+ last = max(0, (numRows // count)) * count
rv = {}
links = {}
rv['links'] = links
--
GitLab
From a343b132e6c149c81a5838f382430345b54a5ced Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 19 Sep 2019 15:10:33 -0500
Subject: [PATCH 048/106] Python 3: migrations
Not sure why these were generated. The models were not modified,
but maybe it was just because we are now using Python 3, and string
handling has changed compared to Python 2. I think all of the fields
that were touched have strings in them either as a default option
or as a set of choices.
---
.../migrations/0004_auto_20190919_1957.py | 27 ++++
.../migrations/0040_auto_20190919_1957.py | 138 ++++++++++++++++++
.../auth/0024_auto_20190919_1957.py | 23 +++
.../migrations/0005_auto_20190919_1957.py | 58 ++++++++
4 files changed, 246 insertions(+)
create mode 100644 gracedb/alerts/migrations/0004_auto_20190919_1957.py
create mode 100644 gracedb/events/migrations/0040_auto_20190919_1957.py
create mode 100644 gracedb/migrations/auth/0024_auto_20190919_1957.py
create mode 100644 gracedb/superevents/migrations/0005_auto_20190919_1957.py
diff --git a/gracedb/alerts/migrations/0004_auto_20190919_1957.py b/gracedb/alerts/migrations/0004_auto_20190919_1957.py
new file mode 100644
index 000000000..9dfbdf114
--- /dev/null
+++ b/gracedb/alerts/migrations/0004_auto_20190919_1957.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-09-19 19:57
+# This was auto-generated after moving to Python 3, with no changes to the
+# actual models. See the commit message for more details.
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('alerts', '0003_add_created_updated_time_fields_to_notification'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='contact',
+ name='phone_method',
+ field=models.CharField(blank=True, choices=[('C', 'Call'), ('T', 'Text'), ('B', 'Call and text')], default=None, max_length=1, null=True),
+ ),
+ migrations.AlterField(
+ model_name='notification',
+ name='category',
+ field=models.CharField(choices=[('E', 'Event'), ('S', 'Superevent')], default='S', max_length=1),
+ ),
+ ]
diff --git a/gracedb/events/migrations/0040_auto_20190919_1957.py b/gracedb/events/migrations/0040_auto_20190919_1957.py
new file mode 100644
index 000000000..c201cdb3e
--- /dev/null
+++ b/gracedb/events/migrations/0040_auto_20190919_1957.py
@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-09-19 19:57
+# This was auto-generated after moving to Python 3, with no changes to the
+# actual models. See the commit message for more details.
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('events', '0039_specify_pipeline_types'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='coincinspiralevent',
+ name='ifos',
+ field=models.CharField(default='', max_length=20),
+ ),
+ migrations.AlterField(
+ model_name='embbeventlog',
+ name='eel_status',
+ field=models.CharField(choices=[('FO', 'FOOTPRINT'), ('SO', 'SOURCE'), ('CO', 'COMMENT'), ('CI', 'CIRCULAR')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='embbeventlog',
+ name='obs_status',
+ field=models.CharField(choices=[('NA', 'NOT APPLICABLE'), ('OB', 'OBSERVATION'), ('TE', 'TEST'), ('PR', 'PREDICTION')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='embbeventlog',
+ name='waveband',
+ field=models.CharField(choices=[('em.gamma', 'Gamma rays part of the spectrum'), ('em.gamma.soft', 'Soft gamma ray (120 - 500 keV)'), ('em.gamma.hard', 'Hard gamma ray (>500 keV)'), ('em.X-ray', 'X-ray part of the spectrum'), ('em.X-ray.soft', 'Soft X-ray (0.12 - 2 keV)'), ('em.X-ray.medium', 'Medium X-ray (2 - 12 keV)'), ('em.X-ray.hard', 'Hard X-ray (12 - 120 keV)'), ('em.UV', 'Ultraviolet part of the spectrum'), ('em.UV.10-50nm', 'Ultraviolet between 10 and 50 nm'), ('em.UV.50-100nm', 'Ultraviolet between 50 and 100 nm'), ('em.UV.100-200nm', 'Ultraviolet between 100 and 200 nm'), ('em.UV.200-300nm', 'Ultraviolet between 200 and 300 nm'), ('em.UV.FUV', 'Far-Infrared, 30-100 microns'), ('em.opt', 'Optical part of the spectrum'), ('em.opt.U', 'Optical band between 300 and 400 nm'), ('em.opt.B', 'Optical band between 400 and 500 nm'), ('em.opt.V', 'Optical band between 500 and 600 nm'), ('em.opt.R', 'Optical band between 600 and 750 nm'), ('em.opt.I', 'Optical band between 750 and 1000 nm'), ('em.IR', 'Infrared part of the spectrum'), ('em.IR.NIR', 'Near-Infrared, 1-5 microns'), ('em.IR.J', 'Infrared between 1.0 and 1.5 micron'), ('em.IR.H', 'Infrared between 1.5 and 2 micron'), ('em.IR.K', 'Infrared between 2 and 3 micron'), ('em.IR.MIR', 'Medium-Infrared, 5-30 microns'), ('em.IR.3-4um', 'Infrared between 3 and 4 micron'), ('em.IR.4-8um', 'Infrared between 4 and 8 micron'), ('em.IR.8-15um', 'Infrared between 8 and 15 micron'), ('em.IR.15-30um', 'Infrared between 15 and 30 micron'), ('em.IR.30-60um', 'Infrared between 30 and 60 micron'), ('em.IR.60-100um', 'Infrared between 60 and 100 micron'), ('em.IR.FIR', 'Far-Infrared, 30-100 microns'), ('em.mm', 'Millimetric part of the spectrum'), ('em.mm.1500-3000GHz', 'Millimetric between 1500 and 3000 GHz'), ('em.mm.750-1500GHz', 'Millimetric between 750 and 1500 GHz'), ('em.mm.400-750GHz', 'Millimetric between 400 and 750 GHz'), ('em.mm.200-400GHz', 'Millimetric between 200 and 400 GHz'), ('em.mm.100-200GHz', 'Millimetric between 100 and 200 GHz'), ('em.mm.50-100GHz', 'Millimetric between 50 and 100 GHz'), ('em.mm.30-50GHz', 'Millimetric between 30 and 50 GHz'), ('em.radio', 'Radio part of the spectrum'), ('em.radio.12-30GHz', 'Radio between 12 and 30 GHz'), ('em.radio.6-12GHz', 'Radio between 6 and 12 GHz'), ('em.radio.3-6GHz', 'Radio between 3 and 6 GHz'), ('em.radio.1500-3000MHz', 'Radio between 1500 and 3000 MHz'), ('em.radio.750-1500MHz', 'Radio between 750 and 1500 MHz'), ('em.radio.400-750MHz', 'Radio between 400 and 750 MHz'), ('em.radio.200-400MHz', 'Radio between 200 and 400 MHz'), ('em.radio.100-200MHz', 'Radio between 100 and 200 MHz'), ('em.radio.20-100MHz', 'Radio between 20 and 100 MHz')], max_length=25),
+ ),
+ migrations.AlterField(
+ model_name='event',
+ name='instruments',
+ field=models.CharField(default='', max_length=20),
+ ),
+ migrations.AlterField(
+ model_name='eventlog',
+ name='filename',
+ field=models.CharField(blank=True, default='', max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='label',
+ name='defaultColor',
+ field=models.CharField(default='black', max_length=20),
+ ),
+ migrations.AlterField(
+ model_name='multiburstevent',
+ name='ifos',
+ field=models.CharField(default='', max_length=20),
+ ),
+ migrations.AlterField(
+ model_name='multiburstevent',
+ name='single_ifo_times',
+ field=models.CharField(default='', max_length=255),
+ ),
+ migrations.AlterField(
+ model_name='pipeline',
+ name='pipeline_type',
+ field=models.CharField(choices=[('E', 'external'), ('O', 'other'), ('SO', 'non-production search'), ('SP', 'production search')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='pipelinelog',
+ name='action',
+ field=models.CharField(choices=[('D', 'disable'), ('E', 'enable')], max_length=10),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='instrument',
+ field=models.CharField(blank=True, choices=[('H1', 'LHO'), ('L1', 'LLO'), ('V1', 'Virgo')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='signoff_type',
+ field=models.CharField(choices=[('OP', 'operator'), ('ADV', 'advocate')], max_length=3),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='status',
+ field=models.CharField(choices=[('OK', 'OKAY'), ('NO', 'NOT OKAY')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='destination_channel',
+ field=models.CharField(blank=True, default='', max_length=50, null=True),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='numrel_data',
+ field=models.CharField(blank=True, default='', max_length=50),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='source',
+ field=models.CharField(blank=True, default='', max_length=50),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='source_channel',
+ field=models.CharField(blank=True, default='', max_length=50, null=True),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='taper',
+ field=models.CharField(blank=True, default='', max_length=50),
+ ),
+ migrations.AlterField(
+ model_name='siminspiralevent',
+ name='waveform',
+ field=models.CharField(blank=True, default='', max_length=50),
+ ),
+ migrations.AlterField(
+ model_name='tag',
+ name='name',
+ field=models.CharField(max_length=100, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_tag_name', message='Tag names can only include [0-9a-zA-z_-]', regex='^[0-9a-zA-Z_\\-]*$')]),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='filename',
+ field=models.CharField(blank=True, default='', editable=False, max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=200),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='voevent_type',
+ field=models.CharField(choices=[('PR', 'preliminary'), ('IN', 'initial'), ('UP', 'update'), ('RE', 'retraction')], max_length=2),
+ ),
+ ]
diff --git a/gracedb/migrations/auth/0024_auto_20190919_1957.py b/gracedb/migrations/auth/0024_auto_20190919_1957.py
new file mode 100644
index 000000000..f05eb1405
--- /dev/null
+++ b/gracedb/migrations/auth/0024_auto_20190919_1957.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-09-19 19:57
+# This was auto-generated after moving to Python 3, with no changes to the
+# actual models. See the commit message for more details.
+from __future__ import unicode_literals
+
+import django.contrib.auth.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('auth', '0023_add_manage_pipeline_permissions'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='user',
+ name='username',
+ field=models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username'),
+ ),
+ ]
diff --git a/gracedb/superevents/migrations/0005_auto_20190919_1957.py b/gracedb/superevents/migrations/0005_auto_20190919_1957.py
new file mode 100644
index 000000000..42d87f724
--- /dev/null
+++ b/gracedb/superevents/migrations/0005_auto_20190919_1957.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-09-19 19:57
+# Note from Tanner:
+# This was auto-generated after moving to Python 3, with no changes to the
+# actual models. See the commit message for more details.
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('superevents', '0004_populate_voevent_fields'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='log',
+ name='filename',
+ field=models.CharField(blank=True, default='', max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='instrument',
+ field=models.CharField(blank=True, choices=[('H1', 'LHO'), ('L1', 'LLO'), ('V1', 'Virgo')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='signoff_type',
+ field=models.CharField(choices=[('OP', 'operator'), ('ADV', 'advocate')], max_length=3),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='status',
+ field=models.CharField(choices=[('OK', 'OKAY'), ('NO', 'NOT OKAY')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='superevent',
+ name='category',
+ field=models.CharField(choices=[('P', 'Production'), ('T', 'Test'), ('M', 'MDC')], default='P', max_length=1),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='filename',
+ field=models.CharField(blank=True, default='', editable=False, max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=200),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='voevent_type',
+ field=models.CharField(choices=[('PR', 'preliminary'), ('IN', 'initial'), ('UP', 'update'), ('RE', 'retraction')], max_length=2),
+ ),
+ ]
--
GitLab
From 1728bf54e1a9a243bd4843cb788f808aee961a8a Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Thu, 19 Sep 2019 15:12:29 -0500
Subject: [PATCH 049/106] alerts.xmpp: fix encoding for LVAlerts in Python 3
---
gracedb/alerts/xmpp.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/alerts/xmpp.py b/gracedb/alerts/xmpp.py
index 59355090e..09445b456 100644
--- a/gracedb/alerts/xmpp.py
+++ b/gracedb/alerts/xmpp.py
@@ -108,7 +108,7 @@ def issue_xmpp_alerts(event_or_superevent, alert_type, serialized_object,
for node_name in node_names:
# Calculate unique message_id and log
- message_id = sha1(node_name + msg).hexdigest()
+ message_id = sha1((node_name + msg).encode()).hexdigest()
# Log message
logger.info(("issue_xmpp_alerts: sending alert type {alert_type} "
--
GitLab
From f23bb98b545440617ee7c4791561c519376ba38f Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 20 Sep 2019 10:17:26 -0500
Subject: [PATCH 050/106] update user account getter to handle string
encoding/decoding for Python 3
---
.../update_user_accounts_from_ligo_ldap.py | 34 +++++++++++--------
1 file changed, 19 insertions(+), 15 deletions(-)
diff --git a/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py b/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py
index 2330e71a5..918b9880e 100644
--- a/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py
+++ b/gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py
@@ -33,13 +33,15 @@ class LdapPersonResultProcessor(object):
def extract_user_attributes(self):
if self.ldap_connection is None:
raise RuntimeError('LDAP connection not configured')
+ memberships = [group_name.decode('utf-8') for group_name in
+ self.ldap_result.get('isMemberOf', [])]
self.user_data = {
- 'first_name': str(self.ldap_result['givenName'][0], 'utf-8'),
- 'last_name': str(self.ldap_result['sn'][0], 'utf-8'),
- 'email': self.ldap_result['mail'][0],
- 'is_active': self.ldap_connection.lvc_group.ldap_name in
- self.ldap_result.get('isMemberOf', []),
- 'username': self.ldap_result['krbPrincipalName'][0],
+ 'first_name': self.ldap_result['givenName'][0].decode('utf-8'),
+ 'last_name': self.ldap_result['sn'][0].decode('utf-8'),
+ 'email': self.ldap_result['mail'][0].decode('utf-8'),
+ 'is_active': (self.ldap_connection.lvc_group.ldap_name in
+ memberships),
+ 'username': self.ldap_result['krbPrincipalName'][0].decode('utf-8'),
}
def check_situation(self, user_exists, l_user_exists):
@@ -143,7 +145,8 @@ class LdapPersonResultProcessor(object):
def update_user_groups(self):
# Get list of group names that the user belongs to from the LDAP result
- memberships = self.ldap_result.get('isMemberOf', [])
+ memberships = [group_name.decode('utf-8') for group_name in
+ self.ldap_result.get('isMemberOf', [])]
# Get groups which are listed for the user in the LDAP and whose
# membership is controlled by the LDAP
@@ -209,7 +212,8 @@ class LdapPersonResultProcessor(object):
# Get two lists of subjects as sets
db_x509_subjects = set(list(self.ligoldapuser.x509cert_set.values_list(
'subject', flat=True)))
- ldap_x509_subjects = set(self.ldap_result.get('gridX509subject', []))
+ ldap_x509_subjects = set([x.decode('utf-8') for x in
+ self.ldap_result.get('gridX509subject', [])])
# Get certs to add and remove
certs_to_add = ldap_x509_subjects.difference(db_x509_subjects)
@@ -239,14 +243,14 @@ class LdapRobotResultProcessor(LdapPersonResultProcessor):
def extract_user_attributes(self):
if self.ldap_connection is None:
raise RuntimeError('LDAP connection not configured')
+ memberships = [group_name.decode('utf-8') for group_name in
+ self.ldap_result.get('isMemberOf', [])]
self.user_data = {
- 'last_name': str(self.ldap_result['x-LIGO-TWikiName'][0],
- 'utf-8'),
- 'email': self.ldap_result['mail'][0],
- 'is_active': self.ldap_connection.groups.get(
- name='robot_accounts').name in self.ldap_result.get(
- 'isMemberOf', []),
- 'username': self.ldap_result['cn'][0],
+ 'last_name': self.ldap_result['x-LIGO-TWikiName'][0].decode('utf-8'),
+ 'email': self.ldap_result['mail'][0].decode('utf-8'),
+ 'is_active': (self.ldap_connection.groups.get(
+ name='robot_accounts').name in memberships),
+ 'username': self.ldap_result['cn'][0].decode('utf-8'),
}
def check_situation(self, user_exists, l_user_exists):
--
GitLab
From 799b051ddd2f2b4991b1af6839746852617be44e Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 20 Sep 2019 12:03:14 -0500
Subject: [PATCH 051/106] Update manage.py to activate the virtualenv with
Python 3 compatibility
---
manage.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/manage.py b/manage.py
index 1755bbdbd..bd91b43b9 100755
--- a/manage.py
+++ b/manage.py
@@ -18,7 +18,10 @@ if __name__ == '__main__':
if (exists(VENV_PATH) and 'VIRTUAL_ENV' not in os.environ):
VIRTUALENV_ACTIVATOR = abspath(join(VENV_PATH, 'bin',
'activate_this.py'))
- execfile(VIRTUALENV_ACTIVATOR, dict(__file__=VIRTUALENV_ACTIVATOR))
+ exec(
+ open(VIRTUALENV_ACTIVATOR).read(),
+ {'__file__': VIRTUALENV_ACTIVATOR}
+ )
# Set DJANGO_SETTINGS_MODULE environment variable
os.environ.setdefault('DJANGO_SETTINGS_MODULE', DEFAULT_SETTINGS_MODULE)
--
GitLab
From 25e355e12113a150861ed1cb0ec0265036d2a156 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Mon, 7 Oct 2019 19:39:23 -0500
Subject: [PATCH 052/106] Two migrations:
- 0052_update_gstlalcbc_O3b_cert.py: gstlalcbc was issued a new cert at CIT.
I manually migrated on production and playground, but i'm backing up the
migration here.
- 0005_add_coinc_far_and_em_type.py: snapshotting the migration for the new
`em_coinc` and `em_type` data fields. This will most likely change.
---
.../0052_update_gstlalcbc_O3b_cert.py | 47 +++++++++++++++++++
.../0005_add_coinc_far_and_em_type.py | 26 ++++++++++
2 files changed, 73 insertions(+)
create mode 100644 gracedb/ligoauth/migrations/0052_update_gstlalcbc_O3b_cert.py
create mode 100644 gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py
diff --git a/gracedb/ligoauth/migrations/0052_update_gstlalcbc_O3b_cert.py b/gracedb/ligoauth/migrations/0052_update_gstlalcbc_O3b_cert.py
new file mode 100644
index 000000000..cc2adc205
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0052_update_gstlalcbc_O3b_cert.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# The new cert and subject line came as a result of gstlalcbc's old
+# certificate expiring on Oct 1, 2019. The new cert was generated
+# on Oct 7, 2019 and expires Oct 7, 2020. So this one should get
+# through O3b.
+
+ACCOUNT = {
+ 'name': 'gstlalcbc',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=cbc.ligo.caltech.edu/CN=gstlalcbc/CN=Chad Hanna/CN=UID:chad.hanna.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('ligoauth', 'RobotUser')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0051_populate_grb_managers_authgroup'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
diff --git a/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py b/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py
new file mode 100644
index 000000000..71589d2d4
--- /dev/null
+++ b/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-15 18:07
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('superevents', '0004_populate_voevent_fields'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='Superevent',
+ name='coinc_far',
+ field=models.FloatField(blank=True, null=True),
+ ),
+ migrations.AddField(
+ model_name='Superevent',
+ name='em_type',
+ field=models.CharField(blank=True, default=b'', max_length=100),
+ ),
+ ]
--
GitLab
From faf19ffc9562439163c1484db89b1ff5fc35fde4 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Tue, 15 Oct 2019 08:24:40 -0500
Subject: [PATCH 053/106] em_type and coinc_far added to superevent model. next
step: add to superevent api requests
---
gracedb/superevents/models.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/gracedb/superevents/models.py b/gracedb/superevents/models.py
index 72548cff9..2e2e11901 100644
--- a/gracedb/superevents/models.py
+++ b/gracedb/superevents/models.py
@@ -112,6 +112,10 @@ class Superevent(CleanSaveModel, AutoIncrementModel):
# superevent, we are going to use a database field to track it.
is_exposed = models.BooleanField(default=False)
+ # New O3b fields for RAVEN:
+ coinc_far = models.FloatField(null=True, blank=True)
+ em_type = models.CharField(blank=True, default=b'', max_length=100)
+
# Meta class --------------------------------------------------------------
class Meta:
ordering = ["-id"]
--
GitLab
From 74c5cb81c2d793b176c3965cd8acd711f6010da4 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Tue, 15 Oct 2019 10:19:50 -0500
Subject: [PATCH 054/106] added em_type and coinc_far to api requests. next
step: modify client to update these fields, make sure logging and alerts are
active, and then modify tests.
---
gracedb/api/v1/superevents/serializers.py | 8 +++++---
gracedb/api/v1/superevents/views.py | 3 ++-
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index 8009b7911..da811e9b4 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -71,8 +71,9 @@ class SupereventSerializer(serializers.ModelSerializer):
class Meta:
model = Superevent
fields = ('superevent_id', 'gw_id', 'category', 'created', 'submitter',
- 'preferred_event', 'events', 't_start', 't_0', 't_end',
- 'gw_events', 'em_events', 'far', 'labels', 'links', 'user')
+ 'preferred_event', 'events', 'em_type', 't_start', 't_0', 't_end',
+ 'gw_events', 'em_events', 'far', 'coinc_far', 'labels', 'links',
+ 'user')
def validate(self, data):
data = super(SupereventSerializer, self).validate(data)
@@ -171,7 +172,8 @@ class SupereventUpdateSerializer(SupereventSerializer):
Used for updates ONLY (PUT/PATCH). Overrides validation which is needed
for object creation.
"""
- allowed_fields = ('t_start', 't_0', 't_end', 'preferred_event')
+ allowed_fields = ('t_start', 't_0', 't_end', 'preferred_event',
+ 'em_type', 'coinc_far')
def __init__(self, *args, **kwargs):
super(SupereventUpdateSerializer, self).__init__(*args, **kwargs)
diff --git a/gracedb/api/v1/superevents/views.py b/gracedb/api/v1/superevents/views.py
index 2575d53af..37b62c2fc 100644
--- a/gracedb/api/v1/superevents/views.py
+++ b/gracedb/api/v1/superevents/views.py
@@ -73,7 +73,8 @@ class SupereventViewSet(SafeCreateMixin, InheritDefaultPermissionsMixin,
SupereventSearchFilter, SupereventOrderingFilter,)
ordering_fields = ('created', 't_0', 't_start', 't_end',
'preferred_event__id', 't_0_date', 'is_gw', 'base_date_number',
- 'gw_date_number', 'category')
+ 'gw_date_number', 'category',
+ 'coinc_far','em_type')
def get_serializer_class(self):
"""Select a different serializer for updates"""
--
GitLab
From a0decede79b438388c5050fa9b385cf4d26c7f18 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Tue, 15 Oct 2019 11:09:07 -0500
Subject: [PATCH 055/106] default for em_type and coinc_far = null
---
.../superevents/migrations/0005_add_coinc_far_and_em_type.py | 2 +-
gracedb/superevents/models.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py b/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py
index 71589d2d4..744636056 100644
--- a/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py
+++ b/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py
@@ -21,6 +21,6 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='Superevent',
name='em_type',
- field=models.CharField(blank=True, default=b'', max_length=100),
+ field=models.CharField(null=True, blank=True, max_length=100),
),
]
diff --git a/gracedb/superevents/models.py b/gracedb/superevents/models.py
index 2e2e11901..baffbef11 100644
--- a/gracedb/superevents/models.py
+++ b/gracedb/superevents/models.py
@@ -113,8 +113,8 @@ class Superevent(CleanSaveModel, AutoIncrementModel):
is_exposed = models.BooleanField(default=False)
# New O3b fields for RAVEN:
- coinc_far = models.FloatField(null=True, blank=True)
- em_type = models.CharField(blank=True, default=b'', max_length=100)
+ coinc_far = models.FloatField(null=False, blank=True)
+ em_type = models.CharField(blank=True, null=True, max_length=100)
# Meta class --------------------------------------------------------------
class Meta:
--
GitLab
From 3700dcb9b35b9f7c8b080e97e07b321b86e6a866 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Tue, 15 Oct 2019 12:55:37 -0500
Subject: [PATCH 056/106] superevents/utils.py: one more edit for updating
superevents.
---
gracedb/superevents/utils.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/gracedb/superevents/utils.py b/gracedb/superevents/utils.py
index cec67f820..8722bec32 100644
--- a/gracedb/superevents/utils.py
+++ b/gracedb/superevents/utils.py
@@ -125,11 +125,13 @@ def update_superevent(superevent, updater, add_log_message=True,
issue_alert=True, **kwargs):
"""
kwargs which are used as superevent parameters:
- t_start, t_0, t_end, preferred_event
+ t_start, t_0, t_end, preferred_event,
+ em_type, coinc_far
"""
# Extract "updatable" superevent params from kwargs
- param_names = ['t_start', 't_0', 't_end', 'preferred_event']
+ param_names = ['t_start', 't_0', 't_end', 'preferred_event',
+ 'em_type','coinc_far']
new_params = {k: v for k,v in kwargs.iteritems() if k in param_names}
# Get old parameters
--
GitLab
From 81f86e3510b1ce21ba1e85fca7c41855e629149d Mon Sep 17 00:00:00 2001
From: Philippe Grassia <philippe.grassia@ligo.org>
Date: Wed, 16 Oct 2019 10:12:37 -0700
Subject: [PATCH 057/106] First Commit of using docker secrets
Using docker secrets if present to populate the sensitive environment
variables whose values we do not want in clear text in the repo
amend: fixed typo
---
docker/entrypoint | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/docker/entrypoint b/docker/entrypoint
index 9d8be7681..efbe06bac 100644
--- a/docker/entrypoint
+++ b/docker/entrypoint
@@ -1,4 +1,33 @@
#!/bin/bash
+
+## PGA: 2019-10-15: use certs from secrets for Shibboleth SP
+SHIB_SP_CERT=/run/secrets/gracedb_ligo_org_saml_cert
+SHIB_SP_KEY=/run/secrets/gracedb_ligo_org_saml_privkey
+if [ -f $SHIB_SP_CERT && -f $SHIB_SP_KEY ]
+then
+ echo "Using Shibboleth Cert from docker secrets over the image one"
+ cp -f $SHIB_SP_CERT /etc/shibboleth/sp-cert.pem
+ cp -f $SHIB_SP_KEY /etc/shibboleth/sp-key.pem
+ chown _shibd:_shibd /etc/shibboleth/sp-{cert,key}.pem
+ chmod 0600 /etc/shibboleth/sp-key.pem
+fi
+
+## PGA 2019-10-16: use secrets for sensitive environment variables
+LIST="aws_ses_access_key_id
+ aws_ses_secret_access_key
+ django_db_password
+ django_secret_key
+ django_twilio_account_sid
+ django_twilio_auth_token
+ lvalert_password"
+
+for SECRET in $LIST
+do
+ VARNAME=$( tr [:lower:] [:upper:] <<<$SECRET)
+ [ -f run/secrets/$SECRET ] && export $VARNAME=\$(< /run/secrets/$SECRET)
+done
+
export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python -c 'import uuid; print(uuid.uuid4().hex)')
exec "$@"
+
--
GitLab
From 2ab36e7104f249ac96c6c4a55f42928791c50807 Mon Sep 17 00:00:00 2001
From: Philippe Grassia <philippe.grassia@ligo.org>
Date: Wed, 16 Oct 2019 11:05:50 -0700
Subject: [PATCH 058/106] fixed bash tests [ ] => [[ ]] and protected content
of secret from bash interpolation
---
docker/entrypoint | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docker/entrypoint b/docker/entrypoint
index efbe06bac..e7adad3d3 100644
--- a/docker/entrypoint
+++ b/docker/entrypoint
@@ -4,7 +4,7 @@
## PGA: 2019-10-15: use certs from secrets for Shibboleth SP
SHIB_SP_CERT=/run/secrets/gracedb_ligo_org_saml_cert
SHIB_SP_KEY=/run/secrets/gracedb_ligo_org_saml_privkey
-if [ -f $SHIB_SP_CERT && -f $SHIB_SP_KEY ]
+if [[ -f $SHIB_SP_CERT && -f $SHIB_SP_KEY ]]
then
echo "Using Shibboleth Cert from docker secrets over the image one"
cp -f $SHIB_SP_CERT /etc/shibboleth/sp-cert.pem
@@ -25,7 +25,7 @@ LIST="aws_ses_access_key_id
for SECRET in $LIST
do
VARNAME=$( tr [:lower:] [:upper:] <<<$SECRET)
- [ -f run/secrets/$SECRET ] && export $VARNAME=\$(< /run/secrets/$SECRET)
+ [ -f /run/secrets/$SECRET ] && export $VARNAME="'$(< /run/secrets/$SECRET)'"
done
export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python -c 'import uuid; print(uuid.uuid4().hex)')
--
GitLab
From b32c435c99758d210b7ff1a1fbea9de5a73259ea Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Thu, 17 Oct 2019 13:33:51 -0400
Subject: [PATCH 059/106] Adding new settings modules for development,
playground, and testing instances on AWS
---
config/settings/container/dev.py | 19 +++++-
config/settings/container/playground.py | 52 +++++++++++++++
config/settings/container/test.py | 89 +++++++++++++++++++++++++
3 files changed, 159 insertions(+), 1 deletion(-)
create mode 100644 config/settings/container/playground.py
create mode 100644 config/settings/container/test.py
diff --git a/config/settings/container/dev.py b/config/settings/container/dev.py
index 65c467a37..f10443b40 100644
--- a/config/settings/container/dev.py
+++ b/config/settings/container/dev.py
@@ -1,7 +1,7 @@
# Settings for a test/dev GraceDB instance running in a container
from .base import *
-CONFIG_NAME = "TEST"
+CONFIG_NAME = "DEV"
# Debug settings
DEBUG = True
@@ -62,3 +62,20 @@ if sentry_dsn is not None:
# Turn off default admin error emails
LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Development Server'
+INSTANCE_INFO = """
+<h3>Development Instance</h3>
+<p>
+This GraceDB instance is designed for GraceDB maintainers to develop and
+test in the AWS cloud architecture. There is <b>no guarantee</b> that the
+behavior of this instance will mimic the production system at any time.
+Events and associated data may change or be removed at any time.
+</p>
+<ul>
+<li>Phone and e-mail alerts are turned off.</li>
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>LVAlert messages are sent to lvalert-dev.cgca.uwm.edu.</li>
+</ul>
+"""
diff --git a/config/settings/container/playground.py b/config/settings/container/playground.py
new file mode 100644
index 000000000..c6b3d8ca0
--- /dev/null
+++ b/config/settings/container/playground.py
@@ -0,0 +1,52 @@
+# Settings for a playground GraceDB instance (for user testing) running
+# in a container on AWS. These settings inherent from base.py)
+# and overrides or adds to them.
+from .base import *
+
+CONFIG_NAME = "USER TESTING"
+
+# Debug settings
+DEBUG = False
+
+# Override EMBB email address
+# TP (8 Aug 2017): not sure why?
+EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
+
+# Turn on XMPP alerts
+SEND_XMPP_ALERTS = True
+
+# Enforce that phone and email alerts are off
+SEND_PHONE_ALERTS = False
+SEND_EMAIL_ALERTS = False
+
+# Define correct LVAlert settings
+LVALERT_OVERSEER_INSTANCES = [
+ {
+ "lvalert_server": "lvalert-playground.cgca.uwm.edu",
+ "listen_port": 8001,
+ },
+]
+
+# Add testserver to ALLOWED_HOSTS
+ALLOWED_HOSTS += ['testserver']
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Playground'
+INSTANCE_INFO = """
+<h3>Playground instance</h3>
+<p>
+This GraceDB instance is designed for users to develop and test their own
+applications. It mimics the production instance in all but the following ways:
+</p>
+<ul>
+<li>Phone and e-mail alerts are turned off.</li>
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>LVAlert messages are sent to lvalert-playground.cgca.uwm.edu.</li>
+<li>Events and associated data will <b>not</b> be preserved indefinitely.
+A nightly cron job removes events older than 21 days.</li>
+</ul>
+"""
+
+# Safety check on debug mode for playground
+if (DEBUG == True):
+ raise RuntimeError("Turn off debug mode for playground")
diff --git a/config/settings/container/test.py b/config/settings/container/test.py
new file mode 100644
index 000000000..f703bb5f5
--- /dev/null
+++ b/config/settings/container/test.py
@@ -0,0 +1,89 @@
+# Settings for a test/dev GraceDB instance running in a container
+from .base import *
+
+CONFIG_NAME = "TEST"
+
+# Debug settings
+DEBUG = True
+
+# Override EMBB email address
+# TP (8 Aug 2017): not sure why?
+EMBB_MAIL_ADDRESS = 'gracedb@{fqdn}'.format(fqdn=SERVER_FQDN)
+
+# Add middleware
+debug_middleware = 'debug_toolbar.middleware.DebugToolbarMiddleware'
+MIDDLEWARE += [
+ debug_middleware,
+ #'silk.middleware.SilkyMiddleware',
+ #'core.middleware.profiling.ProfileMiddleware',
+ #'core.middleware.admin.AdminsOnlyMiddleware',
+]
+
+# Add to installed apps
+INSTALLED_APPS += [
+ 'debug_toolbar',
+ #'silk'
+]
+
+# Add testserver to ALLOWED_HOSTS
+ALLOWED_HOSTS += ['testserver']
+
+# Settings for django-silk profiler
+SILKY_AUTHENTICATION = True
+SILKY_AUTHORISATION = True
+if 'silk' in INSTALLED_APPS:
+ # Needed to prevent RequestDataTooBig for files > 2.5 MB
+ # when silk is being used. This setting is typically used to
+ # prevent DOS attacks, so should not be changed in production.
+ DATA_UPLOAD_MAX_MEMORY_SIZE = 20*(1024**2)
+
+# Tuple of IPs which are marked as internal, useful for debugging.
+# Tanner (5 Dec. 2017): DON'T CHANGE THIS! Django Debug Toolbar exposes
+# some headers which we want to keep hidden. So to be safe, we only allow
+# it to be used through this server. You need to configure a SOCKS proxy
+# on your local machine to use DJDT (see admin docs).
+INTERNAL_IPS = [
+ INTERNAL_IP_ADDRESS,
+]
+
+# Define correct LVAlert settings
+LVALERT_OVERSEER_INSTANCES = [
+ {
+ "lvalert_server": "lvalert-test.cgca.uwm.edu",
+ "listen_port": 8001,
+ },
+]
+
+# Set up Sentry for error logging
+sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
+if sentry_dsn is not None:
+ USE_SENTRY = True
+
+ # Set up Sentry
+ import sentry_sdk
+ from sentry_sdk.integrations.django import DjangoIntegration
+ sentry_sdk.init(
+ environment='test',
+ dsn=sentry_dsn,
+ integrations=[DjangoIntegration()]
+ )
+
+ # Turn off default admin error emails
+ LOGGING['loggers']['django.request']['handlers'] = []
+
+# Home page stuff
+INSTANCE_TITLE = 'GraceDB Testing Server'
+INSTANCE_INFO = """
+<h3>Testing Instance</h3>
+<p>
+This GraceDB instance is designed for Quality Assurance (QA) testing and
+validation for GraceDB and electromagnetic follow-up (EMFollow) developers.
+Software should meet QA milestones on the test instance before being moved
+to Playground or Production. Note, on this GraceDB instance:
+</p>
+<ul>
+<li>Phone and e-mail alerts are turned off.</li>
+<li>Only LIGO logins are provided (no login via InCommon or Google).</li>
+<li>LVAlert messages are sent to lvalert-test.cgca.uwm.edu.</li>
+</ul>
+"""
--
GitLab
From 3da7748851efcdf1c08fbd28d32b0a4a14ec0721 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Thu, 17 Oct 2019 13:58:03 -0400
Subject: [PATCH 060/106] Modifying docker entrypoint to change ownership and
permissions on /app/db_data
---
docker/entrypoint | 2 ++
1 file changed, 2 insertions(+)
diff --git a/docker/entrypoint b/docker/entrypoint
index 9d8be7681..0a50b5e44 100644
--- a/docker/entrypoint
+++ b/docker/entrypoint
@@ -1,4 +1,6 @@
#!/bin/bash
export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python -c 'import uuid; print(uuid.uuid4().hex)')
+chown gracedb:www-data /app/db_data
+chmod 755 /app/db_data
exec "$@"
--
GitLab
From c33995f9c9c1976598f4321b226df9d8637f0a9c Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Thu, 17 Oct 2019 14:59:56 -0400
Subject: [PATCH 061/106] modify entrypoint to accept docker secrets
---
docker/entrypoint | 31 +++++++++++++++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/docker/entrypoint b/docker/entrypoint
index 0a50b5e44..d90572827 100644
--- a/docker/entrypoint
+++ b/docker/entrypoint
@@ -1,6 +1,37 @@
#!/bin/bash
+# Export the required UUID resource for the lvalert_overseer
export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python -c 'import uuid; print(uuid.uuid4().hex)')
+
+# Change the file permissions and ownership on /app/db_data:
chown gracedb:www-data /app/db_data
chmod 755 /app/db_data
+
+## PGA: 2019-10-15: use certs from secrets for Shibboleth SP
+SHIB_SP_CERT=/run/secrets/gracedb_ligo_org_saml_cert
+SHIB_SP_KEY=/run/secrets/gracedb_ligo_org_saml_privkey
+if [[ -f $SHIB_SP_CERT && -f $SHIB_SP_KEY ]]
+then
+ echo "Using Shibboleth Cert from docker secrets over the image one"
+ cp -f $SHIB_SP_CERT /etc/shibboleth/sp-cert.pem
+ cp -f $SHIB_SP_KEY /etc/shibboleth/sp-key.pem
+ chown _shibd:_shibd /etc/shibboleth/sp-{cert,key}.pem
+ chmod 0600 /etc/shibboleth/sp-key.pem
+fi
+
+## PGA 2019-10-16: use secrets for sensitive environment variables
+LIST="aws_ses_access_key_id
+ aws_ses_secret_access_key
+ django_db_password
+ django_secret_key
+ django_twilio_account_sid
+ django_twilio_auth_token
+ lvalert_password"
+
+for SECRET in $LIST
+do
+ VARNAME=$( tr [:lower:] [:upper:] <<<$SECRET)
+ [ -f /run/secrets/$SECRET ] && export $VARNAME="'$(< /run/secrets/$SECRET)'"
+done
+
exec "$@"
--
GitLab
From a9cecb94c742f82ec4725ca7fdf3a437c957aaf0 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Thu, 17 Oct 2019 15:12:48 -0500
Subject: [PATCH 062/106] Update entrypoint: removing quotation marks that were
being set in the password
---
docker/entrypoint | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docker/entrypoint b/docker/entrypoint
index d90572827..23d16c304 100644
--- a/docker/entrypoint
+++ b/docker/entrypoint
@@ -31,7 +31,7 @@ LIST="aws_ses_access_key_id
for SECRET in $LIST
do
VARNAME=$( tr [:lower:] [:upper:] <<<$SECRET)
- [ -f /run/secrets/$SECRET ] && export $VARNAME="'$(< /run/secrets/$SECRET)'"
+ [ -f /run/secrets/$SECRET ] && export $VARNAME="$(< /run/secrets/$SECRET)"
done
exec "$@"
--
GitLab
From 9adb9cb0070f50f87f576478bc793c57776e4b06 Mon Sep 17 00:00:00 2001
From: GraceDB <gracedb@gracedb-dev2.ligo.uwm.edu>
Date: Fri, 18 Oct 2019 10:58:23 -0500
Subject: [PATCH 063/106] cleaning up migrations
---
.../migrations/0005_auto_20190919_1957.py | 58 -------------------
1 file changed, 58 deletions(-)
delete mode 100644 gracedb/superevents/migrations/0005_auto_20190919_1957.py
diff --git a/gracedb/superevents/migrations/0005_auto_20190919_1957.py b/gracedb/superevents/migrations/0005_auto_20190919_1957.py
deleted file mode 100644
index 42d87f724..000000000
--- a/gracedb/superevents/migrations/0005_auto_20190919_1957.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.23 on 2019-09-19 19:57
-# Note from Tanner:
-# This was auto-generated after moving to Python 3, with no changes to the
-# actual models. See the commit message for more details.
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('superevents', '0004_populate_voevent_fields'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='log',
- name='filename',
- field=models.CharField(blank=True, default='', max_length=100),
- ),
- migrations.AlterField(
- model_name='signoff',
- name='instrument',
- field=models.CharField(blank=True, choices=[('H1', 'LHO'), ('L1', 'LLO'), ('V1', 'Virgo')], max_length=2),
- ),
- migrations.AlterField(
- model_name='signoff',
- name='signoff_type',
- field=models.CharField(choices=[('OP', 'operator'), ('ADV', 'advocate')], max_length=3),
- ),
- migrations.AlterField(
- model_name='signoff',
- name='status',
- field=models.CharField(choices=[('OK', 'OKAY'), ('NO', 'NOT OKAY')], max_length=2),
- ),
- migrations.AlterField(
- model_name='superevent',
- name='category',
- field=models.CharField(choices=[('P', 'Production'), ('T', 'Test'), ('M', 'MDC')], default='P', max_length=1),
- ),
- migrations.AlterField(
- model_name='voevent',
- name='filename',
- field=models.CharField(blank=True, default='', editable=False, max_length=100),
- ),
- migrations.AlterField(
- model_name='voevent',
- name='ivorn',
- field=models.CharField(blank=True, default='', editable=False, max_length=200),
- ),
- migrations.AlterField(
- model_name='voevent',
- name='voevent_type',
- field=models.CharField(choices=[('PR', 'preliminary'), ('IN', 'initial'), ('UP', 'update'), ('RE', 'retraction')], max_length=2),
- ),
- ]
--
GitLab
From 752de50c25930b1e22c1790d722f067bec7ee488 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Mon, 21 Oct 2019 08:25:35 -0500
Subject: [PATCH 064/106] Update dev.py: turning on xmpp for dev
---
config/settings/container/dev.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/config/settings/container/dev.py b/config/settings/container/dev.py
index f10443b40..e705a2932 100644
--- a/config/settings/container/dev.py
+++ b/config/settings/container/dev.py
@@ -28,6 +28,14 @@ INSTALLED_APPS += [
# Add testserver to ALLOWED_HOSTS
ALLOWED_HOSTS += ['testserver']
+# Turn on XMPP alerts
+SEND_XMPP_ALERTS = True
+
+# Enforce that phone and email alerts are off
+SEND_PHONE_ALERTS = False
+SEND_EMAIL_ALERTS = False
+
+
# Settings for django-silk profiler
SILKY_AUTHENTICATION = True
SILKY_AUTHORISATION = True
--
GitLab
From 027dbd61528fe2c8b15ef7d7a49ec5164fc95459 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Tue, 22 Oct 2019 08:39:24 -0500
Subject: [PATCH 065/106] Update base.py: updating the version number to
"2.7.1-1". There's no functional differences, just a few settings for the new
AWS deployment. I'm also adding the "*-1" qualifier so that I and users can
tell that the new AWS deployment is in place and working.
---
config/settings/base.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/config/settings/base.py b/config/settings/base.py
index 855096930..662c4eac8 100644
--- a/config/settings/base.py
+++ b/config/settings/base.py
@@ -26,7 +26,7 @@ MAINTENANCE_MODE = False
MAINTENANCE_MODE_MESSAGE = None
# Version ---------------------------------------------------------------------
-PROJECT_VERSION = '2.7.1'
+PROJECT_VERSION = '2.7.1-1'
# Unauthenticated access ------------------------------------------------------
# This variable should eventually control whether unauthenticated access is
--
GitLab
From 78c931445f15bb87334497d16fdccec1ea016068 Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Fri, 20 Sep 2019 12:45:14 -0500
Subject: [PATCH 066/106] Add RAVEN VOEvent variant
---
gracedb/events/models.py | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index a36143cc7..7bc74192e 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -1008,6 +1008,23 @@ class VOEventBase(CleanSaveModel):
validators=[models.fields.validators.MinValueValidator(0.0),
models.fields.validators.MaxValueValidator(1.0)])
+ # Additional RAVEN Fields
+ ext_gcn = models.CharField(max_length=20, default="", blank=True,
+ editable=False)
+ ext_pipeline = models.CharField(max_length=20, default="", blank=True,
+ editable=False)
+ ext_search = models.CharField(max_length=20, default="", blank=True,
+ editable=False)
+ time_coinc_far = models.FloatField(null=True, default=None, blank=True,
+ validators=[models.fields.validators.MinValueValidator(0.0)])
+ space_coinc_far = models.FloatField(null=True, default=None, blank=True,
+ validators=[models.fields.validators.MinValueValidator(0.0)])
+ comb_skymap_filename = models.CharField(max_length=100, null=True,
+ default=None, blank=True)
+ delta_t = models.FloatField(null=True, default=None, blank=True,
+ validators=[models.fields.validators.MinValueValidator(-1000),
+ models.fields.validators.MaxValueValidator(1000)])
+
def fileurl(self):
# Override this method on derived classes
return NotImplemented
--
GitLab
From e2b06cb4b0905eec764a82691b4bec011544278b Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Fri, 20 Sep 2019 13:12:19 -0500
Subject: [PATCH 067/106] Add processing to the superevent API
---
gracedb/api/v1/superevents/serializers.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index e55a741b0..9f6815b60 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -577,6 +577,18 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
MassGap = serializers.FloatField(write_only=True, min_value=0,
max_value=1, required=False)
+ # Additional RAVEN fields
+ ext_gcn = serializers.CharField(required=False)
+ ext_pipeline = serializers.CharField(required=False)
+ ext_search = serializers.CharField(required=False)
+ time_coinc_far = serializers.FloatField(write_only=True, min_value=0,
+ max_value=1000, required=False)
+ space_coinc_far = serializers.FloatField(write_only=True, min_value=0,
+ max_value=1000, required=False)
+ comb_skymap_filename = serializers.CharField(required=False)
+ delta_t = serializers.FloatField(write_only=True, min_value=-1000,
+ max_value=1000, required=False)
+
class Meta:
model = VOEvent
fields = ('voevent_type', 'file_version', 'ivorn', 'created',
--
GitLab
From fd83907ef59d38061774a6c2825987c81ecc5601 Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Fri, 20 Sep 2019 13:47:35 -0500
Subject: [PATCH 068/106] Add processing and validation in the API
---
gracedb/api/v1/events/views.py | 23 +++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index 42f98171f..add6512f1 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -1689,6 +1689,15 @@ class VOEventList(InheritPermissionsAPIView):
Terrestrial = request.data.get('Terrestrial', None)
MassGap = request.data.get('MassGap', None)
+ # Get RAVEN data
+ ext_gcn = request.data.get('ext_gcn', None)
+ ext_pipeline = request.data.get('ext_pipeline', None)
+ ext_search = request.data.get('ext_search', None)
+ time_coinc_far = request.data.get('time_coinc_far', None)
+ space_coinc_far = request.data.get('space_coinc_far', None)
+ comb_skymap_filename = request.data.get('comb_skymap_filename', None)
+ delta_t = request.data.get('delta_t', None)
+
# Get VOEvent types as a dict (key = short form, value = long form)
VOEVENT_TYPE_DICT = dict(VOEvent.VOEVENT_TYPE_CHOICES)
@@ -1722,8 +1731,18 @@ class VOEventList(InheritPermissionsAPIView):
if not os.path.exists(skymap_file_path):
error = True
msg = "Skymap file {fname} does not exist".format(
- fname=skymap_filename)
-
+ fname=skymap_filenamei)
+ elif time_coinc_far or space_coinc_far:
+ if not ext_gcn:
+ error = True
+ msg = "External GCN ID not provided"
+ elif not ext_pipeline:
+ error = True
+ msg = "External Pipeline not provided"
+ elif not ext_search:
+ error = True
+ msg = "External Search not provided"
+
# If there's an error, return a 400 response
if error:
return Response({'error': msg}, status=status.HTTP_400_BAD_REQUEST)
--
GitLab
From 2f9d52214ade051ab69e6cfb88b09ae5d41f4fbe Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Fri, 20 Sep 2019 13:50:23 -0500
Subject: [PATCH 069/106] Fix bug accidently introduced
---
gracedb/api/v1/events/views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index add6512f1..ef52e5d03 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -1731,7 +1731,7 @@ class VOEventList(InheritPermissionsAPIView):
if not os.path.exists(skymap_file_path):
error = True
msg = "Skymap file {fname} does not exist".format(
- fname=skymap_filenamei)
+ fname=skymap_filename)
elif time_coinc_far or space_coinc_far:
if not ext_gcn:
error = True
--
GitLab
From aec2feb19e41ea7434dcdc8737f48f7ba5b19692 Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Fri, 20 Sep 2019 14:13:39 -0500
Subject: [PATCH 070/106] Question on getting data
---
gracedb/annotations/voevent_utils.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index 518334c31..dfe86b033 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -60,6 +60,7 @@ def construct_voevent_file(obj, voevent, request=None):
obj_is_superevent = True
event = obj.preferred_event
graceid = obj.default_superevent_id
+ # ext_event = obj.em_events ???
obj_view_name = "superevents:view"
fits_view_name = "api:default:superevents:superevent-file-detail"
else:
--
GitLab
From b6eb2c421b629c17b7048fbed060139b2a413b63 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Wed, 23 Oct 2019 15:46:12 -0500
Subject: [PATCH 071/106] adding raven voevent fields
---
config/settings/base.py | 1 -
gracedb/api/v1/events/views.py | 4 +-
gracedb/api/v1/superevents/serializers.py | 13 ++++-
gracedb/events/models.py | 3 +-
.../0006_add_raven_voevent_fields.py | 56 +++++++++++++++++++
gracedb/superevents/utils.py | 7 ++-
6 files changed, 77 insertions(+), 7 deletions(-)
create mode 100644 gracedb/superevents/migrations/0006_add_raven_voevent_fields.py
diff --git a/config/settings/base.py b/config/settings/base.py
index e605585da..1d463efde 100644
--- a/config/settings/base.py
+++ b/config/settings/base.py
@@ -51,7 +51,6 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
# MANAGERS defines who gets broken link notifications when
# BrokenLinkEmailsMiddleware is enabled
ADMINS = [
- ("Tanner Prestegard", "tanner.prestegard@ligo.org"),
("Alexander Pace", "alexander.pace@ligo.org"),
("Duncan Meacher", "duncan.meacher@ligo.org"),
]
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index ef52e5d03..1f259dccd 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -1695,8 +1695,10 @@ class VOEventList(InheritPermissionsAPIView):
ext_search = request.data.get('ext_search', None)
time_coinc_far = request.data.get('time_coinc_far', None)
space_coinc_far = request.data.get('space_coinc_far', None)
- comb_skymap_filename = request.data.get('comb_skymap_filename', None)
+ combined_skymap_filename = request.data.get('combined_skymap_filename',
+ None)
delta_t = request.data.get('delta_t', None)
+ raven_coinc = request.data.get('raven_coinc', None)
# Get VOEvent types as a dict (key = short form, value = long form)
VOEVENT_TYPE_DICT = dict(VOEvent.VOEVENT_TYPE_CHOICES)
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index 9f6815b60..3a2e82e5e 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -578,6 +578,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
max_value=1, required=False)
# Additional RAVEN fields
+ raven_coinc = serializers.BooleanField(default=False)
ext_gcn = serializers.CharField(required=False)
ext_pipeline = serializers.CharField(required=False)
ext_search = serializers.CharField(required=False)
@@ -585,7 +586,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
max_value=1000, required=False)
space_coinc_far = serializers.FloatField(write_only=True, min_value=0,
max_value=1000, required=False)
- comb_skymap_filename = serializers.CharField(required=False)
+ combined_skymap_filename = serializers.CharField(required=False)
delta_t = serializers.FloatField(write_only=True, min_value=-1000,
max_value=1000, required=False)
@@ -599,11 +600,19 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
'prob_has_remnant', 'prob_bns', 'prob_nsbh', 'prob_bbh',
'prob_terrestrial', 'prob_mass_gap', 'superevent', 'user')
+ raven_fields = ('raven_coinc','ext_gcn', 'ext_pipeline', 'ext_search',
+ 'time_coinc_far', 'space_coinc_far', 'combined_skymap_filename',
+ 'delta_t')
+
+ # Combine the fields:
+ fields = fields + raven_fields
+
def __init__(self, *args, **kwargs):
super(SupereventVOEventSerializer, self).__init__(*args, **kwargs)
read_only_fields = ['file_version', 'filename', 'ivorn',
'coinc_comment', 'prob_has_ns', 'prob_has_remnant', 'prob_bns',
- 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap']
+ 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap',
+ 'raven_coinc']
for f in read_only_fields:
self.fields.get(f).read_only = True
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index 7bc74192e..f41c303dd 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -1009,6 +1009,7 @@ class VOEventBase(CleanSaveModel):
models.fields.validators.MaxValueValidator(1.0)])
# Additional RAVEN Fields
+ raven_coinc = models.BooleanField(null=False, default=True, blank=True)
ext_gcn = models.CharField(max_length=20, default="", blank=True,
editable=False)
ext_pipeline = models.CharField(max_length=20, default="", blank=True,
@@ -1019,7 +1020,7 @@ class VOEventBase(CleanSaveModel):
validators=[models.fields.validators.MinValueValidator(0.0)])
space_coinc_far = models.FloatField(null=True, default=None, blank=True,
validators=[models.fields.validators.MinValueValidator(0.0)])
- comb_skymap_filename = models.CharField(max_length=100, null=True,
+ combined_skymap_filename = models.CharField(max_length=100, null=True,
default=None, blank=True)
delta_t = models.FloatField(null=True, default=None, blank=True,
validators=[models.fields.validators.MinValueValidator(-1000),
diff --git a/gracedb/superevents/migrations/0006_add_raven_voevent_fields.py b/gracedb/superevents/migrations/0006_add_raven_voevent_fields.py
new file mode 100644
index 000000000..7efed677d
--- /dev/null
+++ b/gracedb/superevents/migrations/0006_add_raven_voevent_fields.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-15 18:07
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('superevents', '0005_add_coinc_far_and_em_type'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='voevent',
+ name='raven_coinc',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='combined_skymap_filename',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_gcn',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_pipeline',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_search',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='time_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='space_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='delta_t',
+ field=models.FloatField(blank=True, default=None, null=True),
+ ),
+ ]
diff --git a/gracedb/superevents/utils.py b/gracedb/superevents/utils.py
index a662e9be0..277a7003f 100644
--- a/gracedb/superevents/utils.py
+++ b/gracedb/superevents/utils.py
@@ -594,7 +594,8 @@ def create_voevent_for_superevent(superevent, issuer, voevent_type,
skymap_type=None, skymap_filename=None, internal=True, open_alert=False,
hardware_inj=False, CoincComment=False, ProbHasNS=None,
ProbHasRemnant=None, BNS=None, NSBH=None, BBH=None, Terrestrial=None,
- MassGap=None, add_log_message=True, issue_alert=True):
+ MassGap=None, add_log_message=True, issue_alert=True,
+ combined_skymap_filename=None, raven_coinc=False):
# Instantiate VOEvent object
voevent = VOEvent.objects.create(superevent=superevent, issuer=issuer,
@@ -603,7 +604,9 @@ def create_voevent_for_superevent(superevent, issuer, voevent_type,
open_alert=open_alert, hardware_inj=hardware_inj,
coinc_comment=CoincComment, prob_has_ns=ProbHasNS,
prob_has_remnant=ProbHasRemnant, prob_bns=BNS, prob_nsbh=NSBH,
- prob_bbh=BBH, prob_terrestrial=Terrestrial, prob_mass_gap=MassGap)
+ prob_bbh=BBH, prob_terrestrial=Terrestrial, prob_mass_gap=MassGap,
+ combined_skymap_filename=combined_skymap_filename,
+ raven_coinc=raven_coinc)
# Construct VOEvent file text
voevent_text, ivorn = construct_voevent_file(superevent, voevent)
--
GitLab
From b94447981a1c287e4af5163ca48ff1590240586d Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Wed, 23 Oct 2019 16:35:34 -0500
Subject: [PATCH 072/106] fixed raven_coinc
---
gracedb/api/v1/superevents/serializers.py | 3 +--
gracedb/events/models.py | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index 3a2e82e5e..540e63698 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -611,8 +611,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
super(SupereventVOEventSerializer, self).__init__(*args, **kwargs)
read_only_fields = ['file_version', 'filename', 'ivorn',
'coinc_comment', 'prob_has_ns', 'prob_has_remnant', 'prob_bns',
- 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap',
- 'raven_coinc']
+ 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap', ]
for f in read_only_fields:
self.fields.get(f).read_only = True
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index f41c303dd..978a35d95 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -1009,7 +1009,7 @@ class VOEventBase(CleanSaveModel):
models.fields.validators.MaxValueValidator(1.0)])
# Additional RAVEN Fields
- raven_coinc = models.BooleanField(null=False, default=True, blank=True)
+ raven_coinc = models.BooleanField(null=False, default=False, blank=True)
ext_gcn = models.CharField(max_length=20, default="", blank=True,
editable=False)
ext_pipeline = models.CharField(max_length=20, default="", blank=True,
--
GitLab
From 08457eca7ef60aa5ea494655f6d8c79a5568009b Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Wed, 23 Oct 2019 22:49:10 -0500
Subject: [PATCH 073/106] Build xml file from RAVEN entries
---
gracedb/annotations/voevent_utils.py | 106 ++++++++++++++++++++++++++-
1 file changed, 105 insertions(+), 1 deletion(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index dfe86b033..b9505e62d 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -60,7 +60,6 @@ def construct_voevent_file(obj, voevent, request=None):
obj_is_superevent = True
event = obj.preferred_event
graceid = obj.default_superevent_id
- # ext_event = obj.em_events ???
obj_view_name = "superevents:view"
fits_view_name = "api:default:superevents:superevent-file-detail"
else:
@@ -288,6 +287,84 @@ def construct_voevent_file(obj, voevent, request=None):
p_search.Description = "Specific low-latency search"
v.What.append(p_search)
+ ## RAVEN specific entries
+ if (is_superevent(obj) and voevent.raven_coinc):
+ ext_id = obj.em_type
+ ext_event = ###???
+
+ ## External GCN ID
+ if ext_event.extra_attributes.GRB.trigger_id:
+ p_extid = vp.Param(
+ "External_GCN_Notice_Id",
+ value=ext_event.extra_attributes.GRB.trigger_id,
+ ucd="meta.id",
+ dataType="string"
+ )
+ p_search.Description = ("GCN trigger ID of external event")
+ v.What.append(p_extid)
+
+ ## External Pipeline
+ if ext_event.pipeline:
+ p_extpipeline = vp.Param(
+ "External_Alert_Type",
+ value=ext_event.pipeline,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_search.Description = ("External Observatory")
+ v.What.append(p_extpipeline)
+
+ ## External Search
+ if ext_event.search:
+ p_extsearch = vp.Param(
+ "External_Search",
+ value=ext_event.search,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_search.Description = ("External astrophysical search")
+ v.What.append(p_extpipeline)
+
+ ## Time Difference
+ if ext_event.gpstime and event.t_0:
+ deltat = round(ext_event.gpstime - event.t_0, 2)
+ p_extsearch = vp.Param(
+ "Time_Difference",
+ value=delta,
+ ucd="meta.code",
+ dataType="string"
+ )
+ p_search.Description = ("External astrophysical search")
+ v.What.append(p_extpipeline)
+
+ ## Temporal Coinc FAR
+ if obj.coinc_far:
+ p_coincfar = vp.Param(
+ "Time_Coincidence_FAR",
+ value=obj.coinc_far,
+ ucd="arith.rate;stat.falsealarm",
+ dataType="float",
+ unit="Hz"
+ )
+ p_search.Description = ("Estimated coincidence false alarm "
+ "rate in Hz using timing")
+ v.What.append(p_coincfar)
+
+ ## Spatial-Temporal Coinc FAR
+ ## FIXME: Find a way to supply this value
+ if False:
+ p_coincfar = vp.Param(
+ "Time_Sky_Position_Coincidence_FAR",
+ value=obj.coinc_far_space,
+ ucd="arith.rate;stat.falsealarm",
+ dataType="float",
+ unit="Hz"
+ )
+ p_search.Description = ("Estimated coincidence false alarm "
+ "rate in Hz using timing and sky "
+ "position")
+ v.What.append(p_coincfar_space)
+
# initial and update VOEvents must have a skymap.
# new feature (10/24/2016): preliminary VOEvents can have a skymap,
# but they don't have to.
@@ -318,6 +395,33 @@ def construct_voevent_file(obj, voevent, request=None):
### Add to What
v.What.append(skymap_group)
+ ## RAVEN combined sky map
+ if (voevent.combined_skymap_filename and voevent.raven_coinc):
+ ## Skymap group
+ ### fits skymap URL
+ fits_skymap_url_comb = build_absolute_uri(
+ reverse(fits_view_name, args=[graceid,
+ combined_skymap_filename]),
+ request
+ )
+ p_fits_url_comb = vp.Param(
+ "skymap_fits",
+ value=fits_skymap_url_comb,
+ ucd="meta.ref.url",
+ dataType="string"
+ )
+ p_fits_url_comb.Description = "Combined GW-External Sky Map FITS"
+
+ ### Create skymap group with params
+ skymap_group_comb = vp.Group(
+ [p_fits_url],
+ name="GW-External_SKYMAP",
+ type="GW-External_SKYMAP",
+ )
+
+ ### Add to What
+ v.What.append(skymap_group_comb)
+
## Analysis specific attributes
if voevent_type != 'retraction':
### Classification group (EM-Bright params; CBC only)
--
GitLab
From 732a5ee4cbcd03d45f19a415b1bcf744c8c3e404 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Thu, 24 Oct 2019 11:29:14 -0500
Subject: [PATCH 074/106] raven voevents in 'working' state.
---
gracedb/annotations/voevent_utils.py | 44 ++++++++++++++++------------
1 file changed, 25 insertions(+), 19 deletions(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index b9505e62d..a98a85f64 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -13,7 +13,7 @@ from django.urls import reverse
from core.time_utils import gpsToUtc
from core.urls import build_absolute_uri
-from events.models import VOEventBase
+from events.models import VOEventBase, Event
from events.models import CoincInspiralEvent, MultiBurstEvent, \
LalInferenceBurstEvent
from superevents.shortcuts import is_superevent
@@ -290,13 +290,13 @@ def construct_voevent_file(obj, voevent, request=None):
## RAVEN specific entries
if (is_superevent(obj) and voevent.raven_coinc):
ext_id = obj.em_type
- ext_event = ###???
+ ext_event = Event.getByGraceid(ext_id)
## External GCN ID
- if ext_event.extra_attributes.GRB.trigger_id:
+ if ext_event.trigger_id:
p_extid = vp.Param(
"External_GCN_Notice_Id",
- value=ext_event.extra_attributes.GRB.trigger_id,
+ value=ext_event.trigger_id,
ucd="meta.id",
dataType="string"
)
@@ -307,7 +307,7 @@ def construct_voevent_file(obj, voevent, request=None):
if ext_event.pipeline:
p_extpipeline = vp.Param(
"External_Alert_Type",
- value=ext_event.pipeline,
+ value=ext_event.pipeline.name,
ucd="meta.code",
dataType="string"
)
@@ -318,24 +318,26 @@ def construct_voevent_file(obj, voevent, request=None):
if ext_event.search:
p_extsearch = vp.Param(
"External_Search",
- value=ext_event.search,
+ value=ext_event.search.name,
ucd="meta.code",
dataType="string"
)
p_search.Description = ("External astrophysical search")
v.What.append(p_extpipeline)
- ## Time Difference
- if ext_event.gpstime and event.t_0:
- deltat = round(ext_event.gpstime - event.t_0, 2)
- p_extsearch = vp.Param(
- "Time_Difference",
- value=delta,
- ucd="meta.code",
- dataType="string"
- )
- p_search.Description = ("External astrophysical search")
- v.What.append(p_extpipeline)
+ ## Time Difference
+ if ext_event.gpstime and obj.t_0:
+ deltat = round(ext_event.gpstime - obj.t_0, 2)
+ p_extsearch = vp.Param(
+ "Time_Difference",
+ value=float(deltat),
+ ucd="meta.code",
+ #dataType="float"
+ #AEP--> figure this out
+ ac=True,
+ )
+ p_search.Description = ("External astrophysical search")
+ v.What.append(p_extpipeline)
## Temporal Coinc FAR
if obj.coinc_far:
@@ -343,7 +345,9 @@ def construct_voevent_file(obj, voevent, request=None):
"Time_Coincidence_FAR",
value=obj.coinc_far,
ucd="arith.rate;stat.falsealarm",
- dataType="float",
+ #dataType="float",
+ #AEP--> figure this out
+ ac=True,
unit="Hz"
)
p_search.Description = ("Estimated coincidence false alarm "
@@ -357,7 +361,9 @@ def construct_voevent_file(obj, voevent, request=None):
"Time_Sky_Position_Coincidence_FAR",
value=obj.coinc_far_space,
ucd="arith.rate;stat.falsealarm",
- dataType="float",
+ #dataType="float",
+ #AEP--> figure this out
+ ac=True,
unit="Hz"
)
p_search.Description = ("Estimated coincidence false alarm "
--
GitLab
From 88a440b5a4c5eb0dc2ffb662d13eb5b4a6fbf4e9 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Thu, 24 Oct 2019 12:22:16 -0500
Subject: [PATCH 075/106] fixing voevent_utils.py
There were some copy/paste errors that I cleaned up. Interestingly though,
voeventlib-parse will just do crazy things with the xml nesting instead
of throwing an error.
---
gracedb/annotations/voevent_utils.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index a98a85f64..904e2d309 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -300,7 +300,7 @@ def construct_voevent_file(obj, voevent, request=None):
ucd="meta.id",
dataType="string"
)
- p_search.Description = ("GCN trigger ID of external event")
+ p_extid.Description = ("GCN trigger ID of external event")
v.What.append(p_extid)
## External Pipeline
@@ -311,7 +311,7 @@ def construct_voevent_file(obj, voevent, request=None):
ucd="meta.code",
dataType="string"
)
- p_search.Description = ("External Observatory")
+ p_extpipeline.Description = ("External Observatory")
v.What.append(p_extpipeline)
## External Search
@@ -322,13 +322,13 @@ def construct_voevent_file(obj, voevent, request=None):
ucd="meta.code",
dataType="string"
)
- p_search.Description = ("External astrophysical search")
- v.What.append(p_extpipeline)
+ p_extsearch.Description = ("External astrophysical search")
+ v.What.append(p_extsearch)
## Time Difference
if ext_event.gpstime and obj.t_0:
deltat = round(ext_event.gpstime - obj.t_0, 2)
- p_extsearch = vp.Param(
+ p_deltat = vp.Param(
"Time_Difference",
value=float(deltat),
ucd="meta.code",
@@ -336,8 +336,8 @@ def construct_voevent_file(obj, voevent, request=None):
#AEP--> figure this out
ac=True,
)
- p_search.Description = ("External astrophysical search")
- v.What.append(p_extpipeline)
+ p_deltat.Description = ("External astrophysical search")
+ v.What.append(p_deltat)
## Temporal Coinc FAR
if obj.coinc_far:
@@ -350,14 +350,14 @@ def construct_voevent_file(obj, voevent, request=None):
ac=True,
unit="Hz"
)
- p_search.Description = ("Estimated coincidence false alarm "
+ p_coincfar.Description = ("Estimated coincidence false alarm "
"rate in Hz using timing")
v.What.append(p_coincfar)
## Spatial-Temporal Coinc FAR
## FIXME: Find a way to supply this value
if False:
- p_coincfar = vp.Param(
+ p_coincfar_space = vp.Param(
"Time_Sky_Position_Coincidence_FAR",
value=obj.coinc_far_space,
ucd="arith.rate;stat.falsealarm",
@@ -366,7 +366,7 @@ def construct_voevent_file(obj, voevent, request=None):
ac=True,
unit="Hz"
)
- p_search.Description = ("Estimated coincidence false alarm "
+ p_coincfar_space.Description = ("Estimated coincidence false alarm "
"rate in Hz using timing and sky "
"position")
v.What.append(p_coincfar_space)
--
GitLab
From 21cdd867bbd82d0bdbf42c00fa5447dd4fb5eb52 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Thu, 24 Oct 2019 15:02:59 -0500
Subject: [PATCH 076/106] updating virgo_detchar's cert
---
.../0053_update_virgodetchar_cert.py | 46 +++++++++++++++++++
1 file changed, 46 insertions(+)
create mode 100644 gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py
diff --git a/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py b/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py
new file mode 100644
index 000000000..6766b1514
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# The new cert and subject line came as a result of virgo detchar's
+# cert expiring. I (Alex) was contained by Nicolas Arnaud on 10/24/2019
+# via email.
+
+ACCOUNT = {
+ 'name': 'virgo_detchar',
+ 'new_cert': 'DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=lscgw.virgo.infn.it/CN=Virgodetchar/CN=Nicolas Arnaud/CN=UID:nicolas.arnaud.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('ligoauth', 'RobotUser')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0052_update_gstlalcbc_O3b_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
--
GitLab
From b6fe8319ef8d07cbb8cf04796a2e4214f970f3c1 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Sat, 26 Oct 2019 19:11:58 -0500
Subject: [PATCH 077/106] Update 0053_update_virgodetchar_cert.py: missing
slash
---
gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py b/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py
index 6766b1514..d791b80e0 100644
--- a/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py
+++ b/gracedb/ligoauth/migrations/0053_update_virgodetchar_cert.py
@@ -10,7 +10,7 @@ from django.db import migrations
ACCOUNT = {
'name': 'virgo_detchar',
- 'new_cert': 'DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=lscgw.virgo.infn.it/CN=Virgodetchar/CN=Nicolas Arnaud/CN=UID:nicolas.arnaud.robot',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=lscgw.virgo.infn.it/CN=Virgodetchar/CN=Nicolas Arnaud/CN=UID:nicolas.arnaud.robot',
}
--
GitLab
From 0c893aeb5157a3416142c8f8a07b50f2a9f3ee7b Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Fri, 1 Nov 2019 15:29:53 -0500
Subject: [PATCH 078/106] Adding latency to basic info table
---
gracedb/templates/gracedb/event_detail.html | 2 ++
1 file changed, 2 insertions(+)
diff --git a/gracedb/templates/gracedb/event_detail.html b/gracedb/templates/gracedb/event_detail.html
index aeb55bdab..363a9a81b 100644
--- a/gracedb/templates/gracedb/event_detail.html
+++ b/gracedb/templates/gracedb/event_detail.html
@@ -192,6 +192,7 @@
</th>
<th>FAR (Hz)</th>
<th>FAR (yr<sup>-1</sup>)</th>
+ <th> Latency (s) </th>
<th>Links</th>
<th>
<div id="basic_info_created_ts"></div>
@@ -219,6 +220,7 @@
{# NOTE: XXX Using event_far so it can be floored for external users. #}
<td>{% if far_is_upper_limit %} < {% endif %}{{ display_far|scientific }}</td>
<td>{% if far_is_upper_limit %} < {% endif %}{{ display_far_yr }}</td>
+ <td>{{ object.reportingLatency }} </td>
<td><a href="{{ object.weburl }}">Data</a></td>
<td>{{ object.created|multiTime:"created" }}</td>
{% if object.superevent %}
--
GitLab
From 88590b964ab08f6a77fd7180e85ceb81d979bdf9 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Mon, 4 Nov 2019 11:21:26 -0600
Subject: [PATCH 079/106] Adding a migration to update pycbclive's cert for
O3b.
---
.../migrations/0054_update_pycbclive_cert.py | 43 +++++++++++++++++++
1 file changed, 43 insertions(+)
create mode 100644 gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py
diff --git a/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py b/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py
new file mode 100644
index 000000000..cea9b3630
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+ACCOUNT = {
+ 'name': 'virgo_detchar',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=ldas-grid.ligo.caltech.edu/CN=pycbclive/CN=Tito Canton/CN=UID:tito.canton.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0053_update_virgodetchar_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
\ No newline at end of file
--
GitLab
From e343c488e48bc7dbf0128cf9813d1faebcb18c1c Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Thu, 7 Nov 2019 18:48:14 -0600
Subject: [PATCH 080/106] Update 0054_update_pycbclive_cert.py
---
gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py b/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py
index cea9b3630..31cfa20a6 100644
--- a/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py
+++ b/gracedb/ligoauth/migrations/0054_update_pycbclive_cert.py
@@ -6,7 +6,7 @@ from django.db import migrations
ACCOUNT = {
- 'name': 'virgo_detchar',
+ 'name': 'pycbclive',
'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=ldas-grid.ligo.caltech.edu/CN=pycbclive/CN=Tito Canton/CN=UID:tito.canton.robot',
}
--
GitLab
From 9088dea076550a00764f776fec8a49693245d238 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Thu, 7 Nov 2019 19:12:24 -0600
Subject: [PATCH 081/106] Adding detchar-la certificate to detchar's account
---
.../migrations/0055_update_detchar_cert.py | 44 +++++++++++++++++++
1 file changed, 44 insertions(+)
create mode 100644 gracedb/ligoauth/migrations/0055_update_detchar_cert.py
diff --git a/gracedb/ligoauth/migrations/0055_update_detchar_cert.py b/gracedb/ligoauth/migrations/0055_update_detchar_cert.py
new file mode 100644
index 000000000..e63bfa062
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0055_update_detchar_cert.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# Note: the CN in the cert is for detchar-la, not detchar.
+
+ACCOUNT = {
+ 'name': 'detchar',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=detchar.ligo-la.caltech.edu/CN=detchar-la/CN=Alexander Urban/CN=UID:alexander.urban.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0054_update_pycbclive_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
\ No newline at end of file
--
GitLab
From 8f527f9100c81038aba66f415bfa56e8b619a6f3 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Tue, 12 Nov 2019 13:14:31 -0500
Subject: [PATCH 082/106] Cleaning up python3 migrations.
I think when I inadvertently removed the 0005-* automatic migration
that broke the tests and subsequent migrations. I'm putting everything
back into place in a form athat should restore the tests.
---
.../migrations/0005_auto_20190919_1957.py | 58 +++++++++++++++++++
.../0006_add_coinc_far_and_em_type.py | 26 +++++++++
.../0007_add_raven_voevent_fields.py | 56 ++++++++++++++++++
3 files changed, 140 insertions(+)
create mode 100644 gracedb/superevents/migrations/0005_auto_20190919_1957.py
create mode 100644 gracedb/superevents/migrations/0006_add_coinc_far_and_em_type.py
create mode 100644 gracedb/superevents/migrations/0007_add_raven_voevent_fields.py
diff --git a/gracedb/superevents/migrations/0005_auto_20190919_1957.py b/gracedb/superevents/migrations/0005_auto_20190919_1957.py
new file mode 100644
index 000000000..42d87f724
--- /dev/null
+++ b/gracedb/superevents/migrations/0005_auto_20190919_1957.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-09-19 19:57
+# Note from Tanner:
+# This was auto-generated after moving to Python 3, with no changes to the
+# actual models. See the commit message for more details.
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('superevents', '0004_populate_voevent_fields'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='log',
+ name='filename',
+ field=models.CharField(blank=True, default='', max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='instrument',
+ field=models.CharField(blank=True, choices=[('H1', 'LHO'), ('L1', 'LLO'), ('V1', 'Virgo')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='signoff_type',
+ field=models.CharField(choices=[('OP', 'operator'), ('ADV', 'advocate')], max_length=3),
+ ),
+ migrations.AlterField(
+ model_name='signoff',
+ name='status',
+ field=models.CharField(choices=[('OK', 'OKAY'), ('NO', 'NOT OKAY')], max_length=2),
+ ),
+ migrations.AlterField(
+ model_name='superevent',
+ name='category',
+ field=models.CharField(choices=[('P', 'Production'), ('T', 'Test'), ('M', 'MDC')], default='P', max_length=1),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='filename',
+ field=models.CharField(blank=True, default='', editable=False, max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=200),
+ ),
+ migrations.AlterField(
+ model_name='voevent',
+ name='voevent_type',
+ field=models.CharField(choices=[('PR', 'preliminary'), ('IN', 'initial'), ('UP', 'update'), ('RE', 'retraction')], max_length=2),
+ ),
+ ]
diff --git a/gracedb/superevents/migrations/0006_add_coinc_far_and_em_type.py b/gracedb/superevents/migrations/0006_add_coinc_far_and_em_type.py
new file mode 100644
index 000000000..90a73790a
--- /dev/null
+++ b/gracedb/superevents/migrations/0006_add_coinc_far_and_em_type.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-15 18:07
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('superevents', '0005_auto_20190919_1957'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='Superevent',
+ name='coinc_far',
+ field=models.FloatField(blank=True, null=True),
+ ),
+ migrations.AddField(
+ model_name='Superevent',
+ name='em_type',
+ field=models.CharField(null=True, blank=True, max_length=100),
+ ),
+ ]
diff --git a/gracedb/superevents/migrations/0007_add_raven_voevent_fields.py b/gracedb/superevents/migrations/0007_add_raven_voevent_fields.py
new file mode 100644
index 000000000..176f4f5cc
--- /dev/null
+++ b/gracedb/superevents/migrations/0007_add_raven_voevent_fields.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-15 18:07
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('superevents', '0006_add_coinc_far_and_em_type'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='voevent',
+ name='raven_coinc',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='combined_skymap_filename',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_gcn',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_pipeline',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_search',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='time_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='space_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='delta_t',
+ field=models.FloatField(blank=True, default=None, null=True),
+ ),
+ ]
--
GitLab
From 62fa1f25e02a6a1ef604ec80da30a0c0d9fbec26 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Tue, 12 Nov 2019 13:16:44 -0600
Subject: [PATCH 083/106] Adding confirmation for making superevent public.
Adding a javascript 'confirm' dialog box that comes up before
exposing/hiding superevents from the public.
---
gracedb/superevents/mixins.py | 5 +++++
gracedb/templates/superevents/detail.html | 12 ++++++++++--
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/gracedb/superevents/mixins.py b/gracedb/superevents/mixins.py
index a7cf26adb..db645c467 100644
--- a/gracedb/superevents/mixins.py
+++ b/gracedb/superevents/mixins.py
@@ -178,18 +178,23 @@ class ExposeHideMixin(ContextMixin):
# Object is hidden and user can expose
can_modify_permissions = True
button_text = 'Make this superevent publicly visible'
+ confirmation_text = 'Warning: You are attempting to make this \
+ event publicly visible. Continue?'
action = 'expose'
elif (self.request.user.has_perm(self.hide_perm_name) and
self.object.is_exposed):
# Object is visible and user can hide
can_modify_permissions = True
button_text = 'Make this superevent internal-only'
+ confirmation_text = 'Warning: You are attempting to make this \
+ event internal only. Continue?'
action = 'hide'
# Update context
context['can_modify_permissions'] = can_modify_permissions
if can_modify_permissions:
context['permissions_form_button_text'] = button_text
+ context['confirmation_dialog_text'] = confirmation_text
context['permissions_action'] = action
return context
diff --git a/gracedb/templates/superevents/detail.html b/gracedb/templates/superevents/detail.html
index e529337d7..8da74e0d1 100644
--- a/gracedb/templates/superevents/detail.html
+++ b/gracedb/templates/superevents/detail.html
@@ -35,6 +35,12 @@
{% include "superevents/superevent_detail_script.js" %}
</script>
+<script>
+ function clickConfirm() {
+ return confirm(" {{confirmation_dialog_text}} ");
+}
+</script>
+
{% endblock %}
{% block content %}
@@ -68,9 +74,11 @@
{#-- XXX This next bit is super hacky. #}
{% if can_modify_permissions %}
<div class="content-area">
-<form action="{% url "legacy_apiweb:default:superevents:superevent-permission-modify" superevent.superevent_id %}" method="POST" id="permissions_form">
+<form action="{% url "legacy_apiweb:default:superevents:superevent-permission-modify" superevent.superevent_id %}"
+ method="POST"
+ id="permissions_form">
<input type="hidden" name="action" value="{{ permissions_action }}">
- <input type="submit" value="{{ permissions_form_button_text }}" class="permButtonClass" disabled>
+ <input type="submit" value="{{ permissions_form_button_text }}" class="permButtonClass" id="permissions_submit_link" onclick="return clickConfirm()">
</form>
</div>
{% endif %}
--
GitLab
From 221aab7f748f6e975abc14495f02bf86162b09b6 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Tue, 12 Nov 2019 14:29:13 -0600
Subject: [PATCH 084/106] Finally cleaning up migrations
Squashing errors in the gitlab pipeline.
---
.../0005_add_coinc_far_and_em_type.py | 26 ---------
.../0006_add_coinc_far_and_em_type.py | 26 ---------
...t_fields.py => 0006_auto_20191112_2004.py} | 37 +++++++-----
.../0007_add_raven_voevent_fields.py | 56 -------------------
4 files changed, 24 insertions(+), 121 deletions(-)
delete mode 100644 gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py
delete mode 100644 gracedb/superevents/migrations/0006_add_coinc_far_and_em_type.py
rename gracedb/superevents/migrations/{0006_add_raven_voevent_fields.py => 0006_auto_20191112_2004.py} (58%)
delete mode 100644 gracedb/superevents/migrations/0007_add_raven_voevent_fields.py
diff --git a/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py b/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py
deleted file mode 100644
index 744636056..000000000
--- a/gracedb/superevents/migrations/0005_add_coinc_far_and_em_type.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.20 on 2019-05-15 18:07
-from __future__ import unicode_literals
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('superevents', '0004_populate_voevent_fields'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='Superevent',
- name='coinc_far',
- field=models.FloatField(blank=True, null=True),
- ),
- migrations.AddField(
- model_name='Superevent',
- name='em_type',
- field=models.CharField(null=True, blank=True, max_length=100),
- ),
- ]
diff --git a/gracedb/superevents/migrations/0006_add_coinc_far_and_em_type.py b/gracedb/superevents/migrations/0006_add_coinc_far_and_em_type.py
deleted file mode 100644
index 90a73790a..000000000
--- a/gracedb/superevents/migrations/0006_add_coinc_far_and_em_type.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.20 on 2019-05-15 18:07
-from __future__ import unicode_literals
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('superevents', '0005_auto_20190919_1957'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='Superevent',
- name='coinc_far',
- field=models.FloatField(blank=True, null=True),
- ),
- migrations.AddField(
- model_name='Superevent',
- name='em_type',
- field=models.CharField(null=True, blank=True, max_length=100),
- ),
- ]
diff --git a/gracedb/superevents/migrations/0006_add_raven_voevent_fields.py b/gracedb/superevents/migrations/0006_auto_20191112_2004.py
similarity index 58%
rename from gracedb/superevents/migrations/0006_add_raven_voevent_fields.py
rename to gracedb/superevents/migrations/0006_auto_20191112_2004.py
index 7efed677d..02bcd1e69 100644
--- a/gracedb/superevents/migrations/0006_add_raven_voevent_fields.py
+++ b/gracedb/superevents/migrations/0006_auto_20191112_2004.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Generated by Django 1.11.20 on 2019-05-15 18:07
+# Generated by Django 1.11.23 on 2019-11-12 20:04
from __future__ import unicode_literals
import django.core.validators
@@ -9,48 +9,59 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
- ('superevents', '0005_add_coinc_far_and_em_type'),
+ ('superevents', '0005_auto_20190919_1957'),
]
operations = [
migrations.AddField(
- model_name='voevent',
- name='raven_coinc',
- field=models.BooleanField(default=False),
+ model_name='superevent',
+ name='coinc_far',
+ field=models.FloatField(blank=True, null=True),
+ preserve_default=False,
+ ),
+ migrations.AddField(
+ model_name='superevent',
+ name='em_type',
+ field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AddField(
model_name='voevent',
name='combined_skymap_filename',
field=models.CharField(blank=True, default=None, max_length=100, null=True),
),
+ migrations.AddField(
+ model_name='voevent',
+ name='delta_t',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(-1000), django.core.validators.MaxValueValidator(1000)]),
+ ),
migrations.AddField(
model_name='voevent',
name='ext_gcn',
- field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ field=models.CharField(blank=True, default='', editable=False, max_length=100),
),
migrations.AddField(
model_name='voevent',
name='ext_pipeline',
- field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ field=models.CharField(blank=True, default='', editable=False, max_length=100),
),
migrations.AddField(
model_name='voevent',
name='ext_search',
- field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ field=models.CharField(blank=True, default='', editable=False, max_length=100),
),
migrations.AddField(
model_name='voevent',
- name='time_coinc_far',
- field=models.FloatField(blank=True, default=None, null=True),
+ name='raven_coinc',
+ field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='voevent',
name='space_coinc_far',
- field=models.FloatField(blank=True, default=None, null=True),
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
),
migrations.AddField(
model_name='voevent',
- name='delta_t',
- field=models.FloatField(blank=True, default=None, null=True),
+ name='time_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
),
]
diff --git a/gracedb/superevents/migrations/0007_add_raven_voevent_fields.py b/gracedb/superevents/migrations/0007_add_raven_voevent_fields.py
deleted file mode 100644
index 176f4f5cc..000000000
--- a/gracedb/superevents/migrations/0007_add_raven_voevent_fields.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.20 on 2019-05-15 18:07
-from __future__ import unicode_literals
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('superevents', '0006_add_coinc_far_and_em_type'),
- ]
-
- operations = [
- migrations.AddField(
- model_name='voevent',
- name='raven_coinc',
- field=models.BooleanField(default=False),
- ),
- migrations.AddField(
- model_name='voevent',
- name='combined_skymap_filename',
- field=models.CharField(blank=True, default=None, max_length=100, null=True),
- ),
- migrations.AddField(
- model_name='voevent',
- name='ext_gcn',
- field=models.CharField(blank=True, default=None, max_length=100, null=True),
- ),
- migrations.AddField(
- model_name='voevent',
- name='ext_pipeline',
- field=models.CharField(blank=True, default=None, max_length=100, null=True),
- ),
- migrations.AddField(
- model_name='voevent',
- name='ext_search',
- field=models.CharField(blank=True, default=None, max_length=100, null=True),
- ),
- migrations.AddField(
- model_name='voevent',
- name='time_coinc_far',
- field=models.FloatField(blank=True, default=None, null=True),
- ),
- migrations.AddField(
- model_name='voevent',
- name='space_coinc_far',
- field=models.FloatField(blank=True, default=None, null=True),
- ),
- migrations.AddField(
- model_name='voevent',
- name='delta_t',
- field=models.FloatField(blank=True, default=None, null=True),
- ),
- ]
--
GitLab
From 7e86682e844616caaf6c53ce8ca44d29adde0a1d Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Tue, 12 Nov 2019 15:26:30 -0600
Subject: [PATCH 085/106] This is the last migrations commit.
Neglecting to create and apply the voevents migration to the 'events'
model would cause gracedb's tests to fail.
---
.../0041_add_raven_voevent_fields.py | 56 +++++++++++++++++++
...> 0006_add_raven_voevent_sevent_fields.py} | 9 ++-
gracedb/superevents/models.py | 2 +-
3 files changed, 61 insertions(+), 6 deletions(-)
create mode 100644 gracedb/events/migrations/0041_add_raven_voevent_fields.py
rename gracedb/superevents/migrations/{0006_auto_20191112_2004.py => 0006_add_raven_voevent_sevent_fields.py} (93%)
diff --git a/gracedb/events/migrations/0041_add_raven_voevent_fields.py b/gracedb/events/migrations/0041_add_raven_voevent_fields.py
new file mode 100644
index 000000000..c81bfdbc9
--- /dev/null
+++ b/gracedb/events/migrations/0041_add_raven_voevent_fields.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-11-12 21:18
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('events', '0040_auto_20190919_1957'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='voevent',
+ name='combined_skymap_filename',
+ field=models.CharField(blank=True, default=None, max_length=100, null=True),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='delta_t',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(-1000), django.core.validators.MaxValueValidator(1000)]),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_gcn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_pipeline',
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='ext_search',
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='raven_coinc',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='space_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
+ ),
+ migrations.AddField(
+ model_name='voevent',
+ name='time_coinc_far',
+ field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
+ ),
+ ]
diff --git a/gracedb/superevents/migrations/0006_auto_20191112_2004.py b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
similarity index 93%
rename from gracedb/superevents/migrations/0006_auto_20191112_2004.py
rename to gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
index 02bcd1e69..1a8e6b6a8 100644
--- a/gracedb/superevents/migrations/0006_auto_20191112_2004.py
+++ b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Generated by Django 1.11.23 on 2019-11-12 20:04
+# Generated by Django 1.11.23 on 2019-11-12 21:18
from __future__ import unicode_literals
import django.core.validators
@@ -17,7 +17,6 @@ class Migration(migrations.Migration):
model_name='superevent',
name='coinc_far',
field=models.FloatField(blank=True, null=True),
- preserve_default=False,
),
migrations.AddField(
model_name='superevent',
@@ -37,17 +36,17 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='voevent',
name='ext_gcn',
- field=models.CharField(blank=True, default='', editable=False, max_length=100),
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
),
migrations.AddField(
model_name='voevent',
name='ext_pipeline',
- field=models.CharField(blank=True, default='', editable=False, max_length=100),
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
),
migrations.AddField(
model_name='voevent',
name='ext_search',
- field=models.CharField(blank=True, default='', editable=False, max_length=100),
+ field=models.CharField(blank=True, default='', editable=False, max_length=20),
),
migrations.AddField(
model_name='voevent',
diff --git a/gracedb/superevents/models.py b/gracedb/superevents/models.py
index f4b45edcb..ca3bdff0c 100644
--- a/gracedb/superevents/models.py
+++ b/gracedb/superevents/models.py
@@ -119,7 +119,7 @@ class Superevent(CleanSaveModel, AutoIncrementModel):
is_exposed = models.BooleanField(default=False)
# New O3b fields for RAVEN:
- coinc_far = models.FloatField(null=False, blank=True)
+ coinc_far = models.FloatField(null=True, blank=True)
em_type = models.CharField(blank=True, null=True, max_length=100)
# Meta class --------------------------------------------------------------
--
GitLab
From 306de2031f5f9755e1351f41092189cef0664c7e Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Thu, 14 Nov 2019 12:51:46 -0600
Subject: [PATCH 086/106] Three new migrations
New certificates for nagios, gstlalcbc, and detchar
---
.../migrations/0056_update_dashboard_cert.py | 44 +++++++++++++++++
.../migrations/0057_gstlalcbc_luigi_cert.py | 43 +++++++++++++++++
.../migrations/0058_add_more_detchar_certs.py | 47 +++++++++++++++++++
3 files changed, 134 insertions(+)
create mode 100644 gracedb/ligoauth/migrations/0056_update_dashboard_cert.py
create mode 100644 gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py
create mode 100644 gracedb/ligoauth/migrations/0058_add_more_detchar_certs.py
diff --git a/gracedb/ligoauth/migrations/0056_update_dashboard_cert.py b/gracedb/ligoauth/migrations/0056_update_dashboard_cert.py
new file mode 100644
index 000000000..76de8da6d
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0056_update_dashboard_cert.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# Note: the CN in the cert is for detchar-la, not detchar.
+
+ACCOUNT = {
+ 'name': 'nagios',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=dashboard.ligo.org/CN=NagiosShibScraper/CN=Shawn Kwang/CN=UID:shawn.kwang.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0055_update_detchar_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
diff --git a/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py b/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py
new file mode 100644
index 000000000..f1bc0af53
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py
@@ -0,0 +1,43 @@
+ngo 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# Note: the CN in the cert is for detchar-la, not detchar.
+
+ACCOUNT = {
+ 'name': 'gstlalcbc',
+ 'new_cert': '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=submit.nemo.uwm.edu/CN=gstlal_online_luigi/CN=Duncan Meacher/CN=UID:duncan.meacher.robot',
+}
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Create new certificate
+ user.x509cert_set.create(subject=ACCOUNT['new_cert'])
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=ACCOUNT['name'])
+
+ # Delete new certificate
+ cert = user.x509cert_set.get(subject=ACCOUNT['new_cert'])
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0056_update_dashboard_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
diff --git a/gracedb/ligoauth/migrations/0058_add_more_detchar_certs.py b/gracedb/ligoauth/migrations/0058_add_more_detchar_certs.py
new file mode 100644
index 000000000..dfb8b8e55
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0058_add_more_detchar_certs.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# detchar is on a tear getting new certs, so I'm doing three
+# at once.
+
+gracedb_account = 'detchar'
+new_certs = ['/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=detchar.ligo-wa.caltech.edu/CN=detchar_ligo-wa/CN=Alexander Urban/CN=UID:alexander.urban.robot',
+ '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=detchar.ligo.caltech.edu/CN=detchar_cit/CN=Alexander Urban/CN=UID:alexander.urban.robot',
+ '/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=ldas-pcdev1.ligo-la.caltech.edu/CN=detchar_ldas-pcdev1_ligo-la/CN=Michael Thomas/CN=UID:michael.thomas.robot']
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Create new certificates
+ for cert in new_certs:
+ user.x509cert_set.create(subject=cert)
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Delete new certificates
+ for cert in new_certs:
+ cert = user.x509cert_set.get(subject=cert)
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0057_gstlalcbc_luigi_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
--
GitLab
From 87d01ac36a74dc67d0972e9ede901a60a987c912 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Thu, 14 Nov 2019 13:12:17 -0600
Subject: [PATCH 087/106] Fix copy/paste error on gstlal migration
---
gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py b/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py
index f1bc0af53..67459917b 100644
--- a/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py
+++ b/gracedb/ligoauth/migrations/0057_gstlalcbc_luigi_cert.py
@@ -1,4 +1,5 @@
-ngo 1.11.20 on 2019-06-03 20:10
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
from __future__ import unicode_literals
from django.db import migrations
--
GitLab
From 4f508978c148c1c15de231ee8b5583cada77c150 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Fri, 15 Nov 2019 12:49:22 -0600
Subject: [PATCH 088/106] Fixed decoding error in cert subjects
API requests on AWS were returning a 500 Server Error, traced it
down to a bytes vs string conversion when retrieving the certificate
subject
---
gracedb/api/backends.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/gracedb/api/backends.py b/gracedb/api/backends.py
index 9274ae56d..52252964f 100644
--- a/gracedb/api/backends.py
+++ b/gracedb/api/backends.py
@@ -221,6 +221,7 @@ class GraceDbX509FullCertAuthentication(GraceDbX509Authentication):
def authenticate(self, request):
+ raise ValueError(request)
# Make sure this request is directed to the API
if self.api_only and not is_api_request(request.path):
return None
@@ -298,19 +299,22 @@ class GraceDbX509FullCertAuthentication(GraceDbX509Authentication):
@staticmethod
def get_certificate_subject_string(certificate):
subject = certificate.get_subject()
+ subject_decoded = [[word.decode("utf8") for word in sets]
+ for sets in subject.get_components()]
subject_string = '/' + "/".join(["=".join(c) for c in
- subject.get_components()])
+ subject_decoded])
return subject_string
@staticmethod
def get_certificate_issuer_string(certificate):
issuer = certificate.get_issuer()
+ issuer_decoded = [[word.decode("utf8") for word in sets]
+ for sets in issuer.get_components()]
issuer_string = '/' + "/".join(["=".join(c) for c in
- issuer.get_components()])
+ issuer_decoded])
return issuer_string
-
class GraceDbAuthenticatedAuthentication(authentication.BaseAuthentication):
"""
If user is already authenticated by the main Django middleware,
--
GitLab
From 1bd6c67e7e9f23dd485d901a41b88a63a573afc7 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Fri, 15 Nov 2019 13:21:34 -0600
Subject: [PATCH 089/106] Removed raised ValueError from debugging.
Yeah, save it.
---
gracedb/api/backends.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/gracedb/api/backends.py b/gracedb/api/backends.py
index 52252964f..3766ecdb5 100644
--- a/gracedb/api/backends.py
+++ b/gracedb/api/backends.py
@@ -221,7 +221,6 @@ class GraceDbX509FullCertAuthentication(GraceDbX509Authentication):
def authenticate(self, request):
- raise ValueError(request)
# Make sure this request is directed to the API
if self.api_only and not is_api_request(request.path):
return None
--
GitLab
From 72339c15400d41de89aadf09c880a74163a6c934 Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Thu, 14 Nov 2019 14:57:53 -0600
Subject: [PATCH 090/106] Modify RAVEN VOEvent template
---
gracedb/annotations/voevent_utils.py | 77 +++++++++++++++-------------
1 file changed, 40 insertions(+), 37 deletions(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index 904e2d309..6e58190a5 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -301,7 +301,7 @@ def construct_voevent_file(obj, voevent, request=None):
dataType="string"
)
p_extid.Description = ("GCN trigger ID of external event")
- v.What.append(p_extid)
+ emcoinc_params.append(p_extid)
## External Pipeline
if ext_event.pipeline:
@@ -312,7 +312,7 @@ def construct_voevent_file(obj, voevent, request=None):
dataType="string"
)
p_extpipeline.Description = ("External Observatory")
- v.What.append(p_extpipeline)
+ emcoinc_params.append(p_extpipeline)
## External Search
if ext_event.search:
@@ -323,7 +323,7 @@ def construct_voevent_file(obj, voevent, request=None):
dataType="string"
)
p_extsearch.Description = ("External astrophysical search")
- v.What.append(p_extsearch)
+ emcoinc_params.append(p_extsearch)
## Time Difference
if ext_event.gpstime and obj.t_0:
@@ -336,8 +336,10 @@ def construct_voevent_file(obj, voevent, request=None):
#AEP--> figure this out
ac=True,
)
- p_deltat.Description = ("External astrophysical search")
- v.What.append(p_deltat)
+ p_deltat.Description = ("Time difference between GW candidate "
+ "and external event, centered on the "
+ "GW candidate")
+ emcoinc_params.append(p_deltat)
## Temporal Coinc FAR
if obj.coinc_far:
@@ -351,8 +353,8 @@ def construct_voevent_file(obj, voevent, request=None):
unit="Hz"
)
p_coincfar.Description = ("Estimated coincidence false alarm "
- "rate in Hz using timing")
- v.What.append(p_coincfar)
+ "rate in Hz using timing")
+ emcoinc_params.append(p_coincfar)
## Spatial-Temporal Coinc FAR
## FIXME: Find a way to supply this value
@@ -367,9 +369,37 @@ def construct_voevent_file(obj, voevent, request=None):
unit="Hz"
)
p_coincfar_space.Description = ("Estimated coincidence false alarm "
- "rate in Hz using timing and sky "
- "position")
- v.What.append(p_coincfar_space)
+ "rate in Hz using timing and sky "
+ "position")
+ emcoinc_params.append(p_coincfar_space)
+
+ ## RAVEN combined sky map
+ if voevent.combined_skymap_filename:
+ ## Skymap group
+ ### fits skymap URL
+ fits_skymap_url_comb = build_absolute_uri(
+ reverse(fits_view_name, args=[graceid,
+ combined_skymap_filename]),
+ request
+ )
+ p_fits_url_comb = vp.Param(
+ "joint_skymap_fits",
+ value=fits_skymap_url_comb,
+ ucd="meta.ref.url",
+ dataType="string"
+ )
+ p_fits_url_comb.Description = "Combined GW-External Sky Map FITS"
+ emcoinc_params.append(p_fits_url_comb)
+
+ ## Create EMCOINC group
+ emcoinc_group = vp.Group(
+ emcoinc_params,
+ name='External Coincidence',
+ type='External Coincidence' # keep this only for backwards compatibility
+ )
+ emcoinc_group.Description = \
+ ("Properties of joint coincidence found by RAVEN")
+ v.What.append(emcoinc_group)
# initial and update VOEvents must have a skymap.
# new feature (10/24/2016): preliminary VOEvents can have a skymap,
@@ -401,33 +431,6 @@ def construct_voevent_file(obj, voevent, request=None):
### Add to What
v.What.append(skymap_group)
- ## RAVEN combined sky map
- if (voevent.combined_skymap_filename and voevent.raven_coinc):
- ## Skymap group
- ### fits skymap URL
- fits_skymap_url_comb = build_absolute_uri(
- reverse(fits_view_name, args=[graceid,
- combined_skymap_filename]),
- request
- )
- p_fits_url_comb = vp.Param(
- "skymap_fits",
- value=fits_skymap_url_comb,
- ucd="meta.ref.url",
- dataType="string"
- )
- p_fits_url_comb.Description = "Combined GW-External Sky Map FITS"
-
- ### Create skymap group with params
- skymap_group_comb = vp.Group(
- [p_fits_url],
- name="GW-External_SKYMAP",
- type="GW-External_SKYMAP",
- )
-
- ### Add to What
- v.What.append(skymap_group_comb)
-
## Analysis specific attributes
if voevent_type != 'retraction':
### Classification group (EM-Bright params; CBC only)
--
GitLab
From fc38446d636c6d7da13611743040d266acafdbb2 Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Mon, 18 Nov 2019 13:14:15 -0600
Subject: [PATCH 091/106] Add ivorn field
---
gracedb/annotations/voevent_utils.py | 11 +++++++++++
gracedb/api/v1/superevents/serializers.py | 5 +++--
gracedb/events/models.py | 3 +++
.../0006_add_raven_voevent_sevent_fields.py | 5 +++++
4 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index 6e58190a5..b675dddd6 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -303,6 +303,17 @@ def construct_voevent_file(obj, voevent, request=None):
p_extid.Description = ("GCN trigger ID of external event")
emcoinc_params.append(p_extid)
+ ## External IVORN
+ if voevent.ivorn:
+ p_extivorn = vp.Param(
+ "External_Ivorn",
+ value=ext_event.trigger_id,
+ ucd="meta.id",
+ dataType="string"
+ )
+ p_extivorn.Description = ("IVORN of external event")
+ emcoinc_params.append(p_extivorn)
+
## External Pipeline
if ext_event.pipeline:
p_extpipeline = vp.Param(
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index 540e63698..ca5534233 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -589,6 +589,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
combined_skymap_filename = serializers.CharField(required=False)
delta_t = serializers.FloatField(write_only=True, min_value=-1000,
max_value=1000, required=False)
+ ivorn = serializers.CharField(required=False)
class Meta:
model = VOEvent
@@ -602,7 +603,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
raven_fields = ('raven_coinc','ext_gcn', 'ext_pipeline', 'ext_search',
'time_coinc_far', 'space_coinc_far', 'combined_skymap_filename',
- 'delta_t')
+ 'delta_t', 'ivorn')
# Combine the fields:
fields = fields + raven_fields
@@ -611,7 +612,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
super(SupereventVOEventSerializer, self).__init__(*args, **kwargs)
read_only_fields = ['file_version', 'filename', 'ivorn',
'coinc_comment', 'prob_has_ns', 'prob_has_remnant', 'prob_bns',
- 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap', ]
+ 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap']
for f in read_only_fields:
self.fields.get(f).read_only = True
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index 978a35d95..6413e548d 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -1025,6 +1025,9 @@ class VOEventBase(CleanSaveModel):
delta_t = models.FloatField(null=True, default=None, blank=True,
validators=[models.fields.validators.MinValueValidator(-1000),
models.fields.validators.MaxValueValidator(1000)])
+ ivorn = models.CharField(max_length=300, default="", blank=True,
+ editable=False)
+
def fileurl(self):
# Override this method on derived classes
diff --git a/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
index 1a8e6b6a8..34342c173 100644
--- a/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
+++ b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
@@ -63,4 +63,9 @@ class Migration(migrations.Migration):
name='time_coinc_far',
field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
),
+ migrations.AddField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=300),
+ )
]
--
GitLab
From 07cec262b45a3e6e79b692522bdfe93af4cb58b9 Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Mon, 18 Nov 2019 15:58:45 -0600
Subject: [PATCH 092/106] Create emcoinc_params before called
---
gracedb/annotations/voevent_utils.py | 1 +
gracedb/api/v1/superevents/serializers.py | 2 +-
.../migrations/0006_add_raven_voevent_sevent_fields.py | 2 +-
3 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index b675dddd6..6c0f5a8c1 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -291,6 +291,7 @@ def construct_voevent_file(obj, voevent, request=None):
if (is_superevent(obj) and voevent.raven_coinc):
ext_id = obj.em_type
ext_event = Event.getByGraceid(ext_id)
+ emcoinc_params = []
## External GCN ID
if ext_event.trigger_id:
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index ca5534233..352a7a829 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -612,7 +612,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
super(SupereventVOEventSerializer, self).__init__(*args, **kwargs)
read_only_fields = ['file_version', 'filename', 'ivorn',
'coinc_comment', 'prob_has_ns', 'prob_has_remnant', 'prob_bns',
- 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap']
+ 'prob_nsbh', 'prob_bbh', 'prob_terrestrial', 'prob_mass_gap', ]
for f in read_only_fields:
self.fields.get(f).read_only = True
diff --git a/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
index 34342c173..bbeaac50d 100644
--- a/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
+++ b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
@@ -67,5 +67,5 @@ class Migration(migrations.Migration):
model_name='voevent',
name='ivorn',
field=models.CharField(blank=True, default='', editable=False, max_length=300),
- )
+ ),
]
--
GitLab
From 2db875c8dccb591e7ac83e0eb05b0dd7b361fe7b Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Mon, 18 Nov 2019 17:29:08 -0600
Subject: [PATCH 093/106] Try getting ivorn from event
---
gracedb/annotations/voevent_utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index 6c0f5a8c1..f91068305 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -305,7 +305,7 @@ def construct_voevent_file(obj, voevent, request=None):
emcoinc_params.append(p_extid)
## External IVORN
- if voevent.ivorn:
+ if event.ivorn:
p_extivorn = vp.Param(
"External_Ivorn",
value=ext_event.trigger_id,
--
GitLab
From dc45a85e472e7446d49e8643dc540d8415e95ea5 Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Mon, 18 Nov 2019 20:24:43 -0600
Subject: [PATCH 094/106] Fixing event to ext_event
---
gracedb/annotations/voevent_utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index f91068305..217926b88 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -305,7 +305,7 @@ def construct_voevent_file(obj, voevent, request=None):
emcoinc_params.append(p_extid)
## External IVORN
- if event.ivorn:
+ if ext_event.ivorn:
p_extivorn = vp.Param(
"External_Ivorn",
value=ext_event.trigger_id,
--
GitLab
From 1cab55eb47c975de2ba21ca94bcca207857665bd Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Mon, 18 Nov 2019 20:33:58 -0600
Subject: [PATCH 095/106] Fix the ivorn value
---
gracedb/annotations/voevent_utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index 217926b88..974ffa7ba 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -308,7 +308,7 @@ def construct_voevent_file(obj, voevent, request=None):
if ext_event.ivorn:
p_extivorn = vp.Param(
"External_Ivorn",
- value=ext_event.trigger_id,
+ value=ext_event.ivorn,
ucd="meta.id",
dataType="string"
)
--
GitLab
From 7400ca64268493a3f61149d745844ef11439a1c7 Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Mon, 18 Nov 2019 20:45:40 -0600
Subject: [PATCH 096/106] Fix combined skymap filename value
---
gracedb/annotations/voevent_utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index 974ffa7ba..1cd0397dc 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -391,7 +391,7 @@ def construct_voevent_file(obj, voevent, request=None):
### fits skymap URL
fits_skymap_url_comb = build_absolute_uri(
reverse(fits_view_name, args=[graceid,
- combined_skymap_filename]),
+ voevent.combined_skymap_filename]),
request
)
p_fits_url_comb = vp.Param(
--
GitLab
From 8dc9eb5dbff7e6a5553e283e65733d905c71dbdf Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Tue, 19 Nov 2019 08:24:25 -0600
Subject: [PATCH 097/106] Add new file
---
.../migrations/0059_grb_exttrig_cert.py | 45 +++++++++++++++++++
1 file changed, 45 insertions(+)
create mode 100644 gracedb/ligoauth/migrations/0059_grb_exttrig_cert.py
diff --git a/gracedb/ligoauth/migrations/0059_grb_exttrig_cert.py b/gracedb/ligoauth/migrations/0059_grb_exttrig_cert.py
new file mode 100644
index 000000000..b27c286f2
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0059_grb_exttrig_cert.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# detchar is on a tear getting new certs, so I'm doing three
+# at once.
+
+gracedb_account = 'grb.exttrig'
+new_certs = ['/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=ldas-pcdev1.ligo.caltech.edu/CN=grb.exttrig/CN=Ryan Fisher/CN=UID:ryan.fisher.robot',]
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Create new certificates
+ for cert in new_certs:
+ user.x509cert_set.create(subject=cert)
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Delete new certificates
+ for cert in new_certs:
+ cert = user.x509cert_set.get(subject=cert)
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0058_add_more_detchar_certs'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
\ No newline at end of file
--
GitLab
From 447d63b0fef5ce68b8e80157db6e5a2987c704a1 Mon Sep 17 00:00:00 2001
From: Alexander Pace <alexander.pace@ligo.org>
Date: Tue, 19 Nov 2019 10:03:58 -0600
Subject: [PATCH 098/106] updated cert for emfollow-test
---
.../migrations/0060_add_emfollow-test_cert.py | 45 +++++++++++++++++++
1 file changed, 45 insertions(+)
create mode 100644 gracedb/ligoauth/migrations/0060_add_emfollow-test_cert.py
diff --git a/gracedb/ligoauth/migrations/0060_add_emfollow-test_cert.py b/gracedb/ligoauth/migrations/0060_add_emfollow-test_cert.py
new file mode 100644
index 000000000..ca6b795a2
--- /dev/null
+++ b/gracedb/ligoauth/migrations/0060_add_emfollow-test_cert.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-03 20:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+# detchar is on a tear getting new certs, so I'm doing three
+# at once.
+
+gracedb_account = 'emfollow'
+new_certs = ['/DC=org/DC=cilogon/C=US/O=LIGO/OU=Robots/CN=emfollow-test.ligo.caltech.edu/CN=emfollow-test/CN=Leo Singer/CN=UID:leo.singer.robot',]
+
+
+def add_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Create new certificates
+ for cert in new_certs:
+ user.x509cert_set.create(subject=cert)
+
+
+def delete_cert(apps, schema_editor):
+ RobotUser = apps.get_model('auth', 'User')
+
+ # Get user
+ user = RobotUser.objects.get(username=gracedb_account)
+
+ # Delete new certificates
+ for cert in new_certs:
+ cert = user.x509cert_set.get(subject=cert)
+ cert.delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ligoauth', '0059_grb_exttrig_cert'),
+ ]
+
+ operations = [
+ migrations.RunPython(add_cert, delete_cert),
+ ]
\ No newline at end of file
--
GitLab
From f14d9d2750de86ba621848e3a2c1a880adc5dbd1 Mon Sep 17 00:00:00 2001
From: Brandon Piotrzkowski <piotrzk3@uwm.edu>
Date: Thu, 14 Nov 2019 14:57:53 -0600
Subject: [PATCH 099/106] Modify RAVEN VOEvent template
---
gracedb/annotations/voevent_utils.py | 91 +++++++++++--------
gracedb/api/v1/superevents/serializers.py | 3 +-
gracedb/events/models.py | 3 +
.../0006_add_raven_voevent_sevent_fields.py | 5 +
4 files changed, 63 insertions(+), 39 deletions(-)
diff --git a/gracedb/annotations/voevent_utils.py b/gracedb/annotations/voevent_utils.py
index 904e2d309..9541c0b39 100644
--- a/gracedb/annotations/voevent_utils.py
+++ b/gracedb/annotations/voevent_utils.py
@@ -291,6 +291,7 @@ def construct_voevent_file(obj, voevent, request=None):
if (is_superevent(obj) and voevent.raven_coinc):
ext_id = obj.em_type
ext_event = Event.getByGraceid(ext_id)
+ emcoinc_params = []
## External GCN ID
if ext_event.trigger_id:
@@ -301,18 +302,29 @@ def construct_voevent_file(obj, voevent, request=None):
dataType="string"
)
p_extid.Description = ("GCN trigger ID of external event")
- v.What.append(p_extid)
+ emcoinc_params.append(p_extid)
+
+ ## External IVORN
+ if ext_event.ivorn:
+ p_extivorn = vp.Param(
+ "External_Ivorn",
+ value=ext_event.ivorn,
+ ucd="meta.id",
+ dataType="string"
+ )
+ p_extivorn.Description = ("IVORN of external event")
+ emcoinc_params.append(p_extivorn)
## External Pipeline
if ext_event.pipeline:
p_extpipeline = vp.Param(
- "External_Alert_Type",
+ "External_Observatory",
value=ext_event.pipeline.name,
ucd="meta.code",
dataType="string"
)
p_extpipeline.Description = ("External Observatory")
- v.What.append(p_extpipeline)
+ emcoinc_params.append(p_extpipeline)
## External Search
if ext_event.search:
@@ -323,7 +335,7 @@ def construct_voevent_file(obj, voevent, request=None):
dataType="string"
)
p_extsearch.Description = ("External astrophysical search")
- v.What.append(p_extsearch)
+ emcoinc_params.append(p_extsearch)
## Time Difference
if ext_event.gpstime and obj.t_0:
@@ -336,8 +348,10 @@ def construct_voevent_file(obj, voevent, request=None):
#AEP--> figure this out
ac=True,
)
- p_deltat.Description = ("External astrophysical search")
- v.What.append(p_deltat)
+ p_deltat.Description = ("Time difference between GW candidate "
+ "and external event, centered on the "
+ "GW candidate")
+ emcoinc_params.append(p_deltat)
## Temporal Coinc FAR
if obj.coinc_far:
@@ -351,8 +365,8 @@ def construct_voevent_file(obj, voevent, request=None):
unit="Hz"
)
p_coincfar.Description = ("Estimated coincidence false alarm "
- "rate in Hz using timing")
- v.What.append(p_coincfar)
+ "rate in Hz using timing")
+ emcoinc_params.append(p_coincfar)
## Spatial-Temporal Coinc FAR
## FIXME: Find a way to supply this value
@@ -367,9 +381,37 @@ def construct_voevent_file(obj, voevent, request=None):
unit="Hz"
)
p_coincfar_space.Description = ("Estimated coincidence false alarm "
- "rate in Hz using timing and sky "
- "position")
- v.What.append(p_coincfar_space)
+ "rate in Hz using timing and sky "
+ "position")
+ emcoinc_params.append(p_coincfar_space)
+
+ ## RAVEN combined sky map
+ if voevent.combined_skymap_filename:
+ ## Skymap group
+ ### fits skymap URL
+ fits_skymap_url_comb = build_absolute_uri(
+ reverse(fits_view_name, args=[graceid,
+ voevent.combined_skymap_filename]),
+ request
+ )
+ p_fits_url_comb = vp.Param(
+ "joint_skymap_fits",
+ value=fits_skymap_url_comb,
+ ucd="meta.ref.url",
+ dataType="string"
+ )
+ p_fits_url_comb.Description = "Combined GW-External Sky Map FITS"
+ emcoinc_params.append(p_fits_url_comb)
+
+ ## Create EMCOINC group
+ emcoinc_group = vp.Group(
+ emcoinc_params,
+ name='External Coincidence',
+ type='External Coincidence' # keep this only for backwards compatibility
+ )
+ emcoinc_group.Description = \
+ ("Properties of joint coincidence found by RAVEN")
+ v.What.append(emcoinc_group)
# initial and update VOEvents must have a skymap.
# new feature (10/24/2016): preliminary VOEvents can have a skymap,
@@ -401,33 +443,6 @@ def construct_voevent_file(obj, voevent, request=None):
### Add to What
v.What.append(skymap_group)
- ## RAVEN combined sky map
- if (voevent.combined_skymap_filename and voevent.raven_coinc):
- ## Skymap group
- ### fits skymap URL
- fits_skymap_url_comb = build_absolute_uri(
- reverse(fits_view_name, args=[graceid,
- combined_skymap_filename]),
- request
- )
- p_fits_url_comb = vp.Param(
- "skymap_fits",
- value=fits_skymap_url_comb,
- ucd="meta.ref.url",
- dataType="string"
- )
- p_fits_url_comb.Description = "Combined GW-External Sky Map FITS"
-
- ### Create skymap group with params
- skymap_group_comb = vp.Group(
- [p_fits_url],
- name="GW-External_SKYMAP",
- type="GW-External_SKYMAP",
- )
-
- ### Add to What
- v.What.append(skymap_group_comb)
-
## Analysis specific attributes
if voevent_type != 'retraction':
### Classification group (EM-Bright params; CBC only)
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index 540e63698..352a7a829 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -589,6 +589,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
combined_skymap_filename = serializers.CharField(required=False)
delta_t = serializers.FloatField(write_only=True, min_value=-1000,
max_value=1000, required=False)
+ ivorn = serializers.CharField(required=False)
class Meta:
model = VOEvent
@@ -602,7 +603,7 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
raven_fields = ('raven_coinc','ext_gcn', 'ext_pipeline', 'ext_search',
'time_coinc_far', 'space_coinc_far', 'combined_skymap_filename',
- 'delta_t')
+ 'delta_t', 'ivorn')
# Combine the fields:
fields = fields + raven_fields
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index 978a35d95..6413e548d 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -1025,6 +1025,9 @@ class VOEventBase(CleanSaveModel):
delta_t = models.FloatField(null=True, default=None, blank=True,
validators=[models.fields.validators.MinValueValidator(-1000),
models.fields.validators.MaxValueValidator(1000)])
+ ivorn = models.CharField(max_length=300, default="", blank=True,
+ editable=False)
+
def fileurl(self):
# Override this method on derived classes
diff --git a/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
index 1a8e6b6a8..bbeaac50d 100644
--- a/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
+++ b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
@@ -63,4 +63,9 @@ class Migration(migrations.Migration):
name='time_coinc_far',
field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
),
+ migrations.AddField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=300),
+ ),
]
--
GitLab
From b04e88c2d98c4e38737f08d383698423afadd05f Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Tue, 19 Nov 2019 12:46:54 -0600
Subject: [PATCH 100/106] Two more migrations for RAVEN
For some reason, django makemigration wasn't recognizing that the
voevent ivorn table actually existed, so it was trying to recreate
that table and the pytests were running into an issue.
---
.../migrations/0042_auto_20191119_1730.py | 20 +++++++++++++++++++
.../0006_add_raven_voevent_sevent_fields.py | 4 ++--
2 files changed, 22 insertions(+), 2 deletions(-)
create mode 100644 gracedb/events/migrations/0042_auto_20191119_1730.py
diff --git a/gracedb/events/migrations/0042_auto_20191119_1730.py b/gracedb/events/migrations/0042_auto_20191119_1730.py
new file mode 100644
index 000000000..084106971
--- /dev/null
+++ b/gracedb/events/migrations/0042_auto_20191119_1730.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-11-19 17:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('events', '0041_add_raven_voevent_fields'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='voevent',
+ name='ivorn',
+ field=models.CharField(blank=True, default='', editable=False, max_length=300),
+ ),
+ ]
diff --git a/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
index bbeaac50d..695631b34 100644
--- a/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
+++ b/gracedb/superevents/migrations/0006_add_raven_voevent_sevent_fields.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Generated by Django 1.11.23 on 2019-11-12 21:18
+# Generated by Django 1.11.23 on 2019-11-19 17:39
from __future__ import unicode_literals
import django.core.validators
@@ -63,7 +63,7 @@ class Migration(migrations.Migration):
name='time_coinc_far',
field=models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0)]),
),
- migrations.AddField(
+ migrations.AlterField(
model_name='voevent',
name='ivorn',
field=models.CharField(blank=True, default='', editable=False, max_length=300),
--
GitLab
From 21efc41ff555d84bde61173bcb1ea250e1e6b334 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Tue, 19 Nov 2019 15:32:53 -0600
Subject: [PATCH 101/106] Added API traps for RAVEN Superevents.
The server will now return a 400 "bad request" error for RAVEN
createVOEvent calls if:
1. em_type for the associated superevent is None
2. em_type for the associated superevent is not a valid graceid
3. the event associated with em_type cannot be found
4. the combined_skymap_filename is set, but the file could not
be found.
---
gracedb/api/v1/superevents/serializers.py | 26 +++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/gracedb/api/v1/superevents/serializers.py b/gracedb/api/v1/superevents/serializers.py
index 352a7a829..c38e2e260 100644
--- a/gracedb/api/v1/superevents/serializers.py
+++ b/gracedb/api/v1/superevents/serializers.py
@@ -540,6 +540,12 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
'superevent.'),
'skymap_image_not_found': _('Skymap image file {filename} not found '
'for this superevent.'),
+ 'em_type_none': _('em_type for superevent {s_event} not set (is None)'),
+ 'invalid_em_type': _('em_type for superevent {s_event} is not a valid '
+ 'graceid'),
+ 'em_type_not_found': _('event for em_type={em_type} not found'),
+ 'comb_skymap_not_found': _('Combined skymap file {filename} not found '
+ 'for this superevent.'),
}
# Read only fields
issuer = serializers.SlugRelatedField(slug_field='username',
@@ -640,6 +646,9 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
voevent_type = data.get('voevent_type')
skymap_filename = data.get('skymap_filename', None)
skymap_type = data.get('skymap_type', None)
+ raven_coinc = data.get('raven_coinc')
+ combined_smfn = data.get('combined_skymap_filename',None)
+
# Checks to do:
# Preferred event must have gpstime
@@ -665,6 +674,23 @@ class SupereventVOEventSerializer(serializers.ModelSerializer):
if not os.path.exists(full_skymap_path):
self.fail('skymap_not_found', filename=skymap_filename)
+ if raven_coinc:
+ if superevent.em_type is None:
+ self.fail('em_type_none', s_event=superevent.graceid)
+ else:
+ try:
+ em_event = Event.getByGraceid(superevent.em_type)
+ except ValueError:
+ self.fail('invalid_em_type', s_event=superevent.graceid)
+ except Event.DoesNotExist:
+ self.fail('em_type_not_found',em_type=superevent.em_type)
+ if combined_smfn != None:
+ comb_skymap_path = os.path.join(superevent.datadir,
+ combined_smfn)
+ if not os.path.exists(comb_skymap_path):
+ self.fail('comb_skymap_not_found', filename=combined_smfn)
+
+
return data
def create(self, validated_data):
--
GitLab
From 15d2dc7a0a6931ba7963ccf5e27f44edb9396101 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Wed, 20 Nov 2019 12:17:00 -0600
Subject: [PATCH 102/106] Update label documentation
Reflects new usage of EM_READY and EM_Selected labels in
superevent 2.0.
---
docs/user_docs/source/labels.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/user_docs/source/labels.rst b/docs/user_docs/source/labels.rst
index 9d6fb2570..3fa3ea690 100644
--- a/docs/user_docs/source/labels.rst
+++ b/docs/user_docs/source/labels.rst
@@ -29,9 +29,9 @@ Here is a table showing the currently available labels and their meanings.
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
| EM_COINC | Signifies that a coincidence was found between gravitational-wave candidates and External triggers. |
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
-| EM_READY | Has been processed by GDB Processor. Skymaps have been produced. |
+| EM_READY | Indicates data products associated with a Superevent's preferred event are complete |
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
-| EM_Selected | GraceID automatically chosen as the most promising candidate out of a set of entries thought to correspond to the same physical event. |
+| EM_Selected | Indicates Superevent has been selected to be sent out as a public alert and freezes the preferred event from updates |
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
| EM_SENT | Has been sent to MOU partners. |
+-----------------+----------------------------------------------------------------------------------------------------------------------------------------+
--
GitLab
From 89bd4d683ecf83592801982931c12a0430c6cf23 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Wed, 20 Nov 2019 14:51:27 -0600
Subject: [PATCH 103/106] Turning on XMPP Alerts on gracedb-test.
---
config/settings/container/test.py | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/config/settings/container/test.py b/config/settings/container/test.py
index f703bb5f5..42cb4698d 100644
--- a/config/settings/container/test.py
+++ b/config/settings/container/test.py
@@ -46,6 +46,21 @@ INTERNAL_IPS = [
INTERNAL_IP_ADDRESS,
]
+# Turn on XMPP alerts
+SEND_XMPP_ALERTS = True
+
+# Enforce that phone and email alerts are off
+SEND_PHONE_ALERTS = False
+SEND_EMAIL_ALERTS = False
+
+# Define correct LVAlert settings
+LVALERT_OVERSEER_INSTANCES = [
+ {
+ "lvalert_server": "lvalert-playground.cgca.uwm.edu",
+ "listen_port": 8001,
+ },
+]
+
# Define correct LVAlert settings
LVALERT_OVERSEER_INSTANCES = [
{
--
GitLab
From 30a19b9403a010aa1b5043e52cdf80312d49092e Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Fri, 22 Nov 2019 13:35:26 -0600
Subject: [PATCH 104/106] Two changes:
Fixed LVAlert server for "test" container, and fixed 500 server error
when sorting p_astro list for the public event page. I think this
https://stackoverflow.com/questions/27776026/lambda-arguments-unpack-error
fixes it.
---
config/settings/container/test.py | 7 -------
gracedb/superevents/views.py | 2 +-
2 files changed, 1 insertion(+), 8 deletions(-)
diff --git a/config/settings/container/test.py b/config/settings/container/test.py
index 42cb4698d..199239b85 100644
--- a/config/settings/container/test.py
+++ b/config/settings/container/test.py
@@ -53,13 +53,6 @@ SEND_XMPP_ALERTS = True
SEND_PHONE_ALERTS = False
SEND_EMAIL_ALERTS = False
-# Define correct LVAlert settings
-LVALERT_OVERSEER_INSTANCES = [
- {
- "lvalert_server": "lvalert-playground.cgca.uwm.edu",
- "listen_port": 8001,
- },
-]
# Define correct LVAlert settings
LVALERT_OVERSEER_INSTANCES = [
diff --git a/gracedb/superevents/views.py b/gracedb/superevents/views.py
index 7b7f8ac31..7015c8623 100644
--- a/gracedb/superevents/views.py
+++ b/gracedb/superevents/views.py
@@ -233,7 +233,7 @@ class SupereventPublic(DisplayFarMixin, ListView):
("BBH", voe.prob_bbh),
("Terrestrial", voe.prob_terrestrial),
("MassGap", voe.prob_mass_gap)]
- pastro_values.sort(reverse=True, key=lambda a, b: b)
+ pastro_values.sort(reverse=True, key=lambda p_a: p_a[1])
sourcelist = []
for key, value in pastro_values:
if value > 0.01:
--
GitLab
From 0e04e30c67507335a06f816c341f165aef9afb15 Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Sun, 24 Nov 2019 10:10:27 -0600
Subject: [PATCH 105/106] Two changes:
1) Fixed a 500 Internal Server error when removing labels from events
that do not exist. Basically the ValueError.message method was
depreciated in python3; replaced it with a str(error).
2) Bumped version to 2.8.0
---
config/settings/base.py | 2 +-
gracedb/api/v1/events/views.py | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/config/settings/base.py b/config/settings/base.py
index 4b6f11031..4f26af00f 100644
--- a/config/settings/base.py
+++ b/config/settings/base.py
@@ -26,7 +26,7 @@ MAINTENANCE_MODE = False
MAINTENANCE_MODE_MESSAGE = None
# Version ---------------------------------------------------------------------
-PROJECT_VERSION = '2.7.1-1'
+PROJECT_VERSION = '2.8.0'
# Unauthenticated access ------------------------------------------------------
# This variable should eventually control whether unauthenticated access is
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index 1f259dccd..c9b292c9b 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -798,7 +798,7 @@ class EventLabel(InheritPermissionsAPIView):
try:
rv, label_created = create_label(event, request, label)
except (ValueError, Label.ProtectedLabelError) as e:
- return Response(e.message,
+ return Response(str(e),
status=status.HTTP_400_BAD_REQUEST)
# Return response and status code
@@ -813,9 +813,9 @@ class EventLabel(InheritPermissionsAPIView):
try:
rv = delete_label(event, request, label)
except Labelling.DoesNotExist as e:
- return Response(e.message, status=status.HTTP_404_NOT_FOUND)
+ return Response(str(e), status=status.HTTP_404_NOT_FOUND)
except (ValueError, Label.ProtectedLabelError) as e:
- return Response(e.message, status=status.HTTP_400_BAD_REQUEST)
+ return Response(str(e), status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_204_NO_CONTENT)
--
GitLab
From e0e1fa03c530fed2ecc26a0a9ec00b0ec59c654e Mon Sep 17 00:00:00 2001
From: "alexander.pace@ligo.org" <alexander.pace@ligo.org>
Date: Sun, 24 Nov 2019 13:33:09 -0600
Subject: [PATCH 106/106] Settings change
Don't override LVALERT_OVERSEER_INSTANCE settings from
envrionment variables.
---
config/settings/container/playground.py | 8 --------
config/settings/container/test.py | 8 --------
2 files changed, 16 deletions(-)
diff --git a/config/settings/container/playground.py b/config/settings/container/playground.py
index c6b3d8ca0..354f25b08 100644
--- a/config/settings/container/playground.py
+++ b/config/settings/container/playground.py
@@ -19,14 +19,6 @@ SEND_XMPP_ALERTS = True
SEND_PHONE_ALERTS = False
SEND_EMAIL_ALERTS = False
-# Define correct LVAlert settings
-LVALERT_OVERSEER_INSTANCES = [
- {
- "lvalert_server": "lvalert-playground.cgca.uwm.edu",
- "listen_port": 8001,
- },
-]
-
# Add testserver to ALLOWED_HOSTS
ALLOWED_HOSTS += ['testserver']
diff --git a/config/settings/container/test.py b/config/settings/container/test.py
index 199239b85..0bdbe4838 100644
--- a/config/settings/container/test.py
+++ b/config/settings/container/test.py
@@ -54,14 +54,6 @@ SEND_PHONE_ALERTS = False
SEND_EMAIL_ALERTS = False
-# Define correct LVAlert settings
-LVALERT_OVERSEER_INSTANCES = [
- {
- "lvalert_server": "lvalert-test.cgca.uwm.edu",
- "listen_port": 8001,
- },
-]
-
# Set up Sentry for error logging
sentry_dsn = get_from_env('DJANGO_SENTRY_DSN', fail_if_not_found=False)
if sentry_dsn is not None:
--
GitLab