Skip to content

GitLab

Commit 6f1a8fec authored by Branson Stephens's avatar Branson Stephens
Browse files

Fixed some bugs resulting from the merger.

parent 0174775a
Hide whitespace changes
Inline Side-by-side
Showing with 326 additions and 40 deletions
ligo/gracedb/rest.py
View file @ 6f1a8fec
# Copyright (C) 2012 LIGO Scientific Collaboration
# -*- coding: utf-8 -*-
# Copyright (C) Brian Moe, Branson Stephens (2015)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
# This file is part of gracedb
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
import httplib, socket, ssl
import mimetypes
import urllib
import os, sys
import json
from urlparse import urlparse
from ecp_client import EcpRest
from base64 import b64encode
import netrc
from ecp_client import EcpRest
DEFAULT_SERVICE_URL = "https://gracedb.ligo.org/apiweb/"
DEFAULT_BASIC_SERVICE_URL = "https://gracedb.ligo.org/apibasic/"
DEFAULT_SP_SESSION_ENDPOINT = "https://gracedb.ligo.org/Shibboleth.sso/Session"
KNOWN_TEST_HOSTS = ['moe.phys.uwm.edu', 'embb-dev.ligo.caltech.edu', 'simdb.phys.uwm.edu',]
#---------------------------------------------------------------------
# This monkey patch forces TLSv1 if the python version is 2.6.6.
# It was introduced because clients connection from CIT *occasionally*
# try to use SSLv3. See:
# http://stackoverflow.com/questions/18669457/python-httplib-ssl23-get-server-hellounknown-protocol
#---------------------------------------------------------------------
if sys.hexversion < 0x20709f0:
wrap_socket_orig = ssl.wrap_socket
def wrap_socket_patched(sock, keyfile=None, certfile=None,
server_side=False, cert_reqs=ssl.CERT_NONE,
ssl_version=ssl.PROTOCOL_TLSv1, ca_certs=None,
do_handshake_on_connect=True,
suppress_ragged_eofs=True):
return wrap_socket_orig(sock, keyfile, certfile, server_side,
cert_reqs, ssl_version, ca_certs,
do_handshake_on_connect,
suppress_ragged_eofs)
ssl.wrap_socket = wrap_socket_patched
#-----------------------------------------------------------------
# Utilities
# XXX It would be nice to get rid of this if we can.
# It seems that you can pass python lists via the requests package.
# That would make putting our lists into comma-separated strings
# unnecessary.
def cleanListInput(list_arg):
stringified_list = list_arg
if isinstance(list_arg, float) or isinstance(list_arg, int):
stringified_value = str(list_arg)
return stringified_value
if not isinstance(list_arg, basestring):
stringified_list = ','.join(map(str,list_arg))
return stringified_list
# The following are used to check whether a user has tried to use
# an expired certificate.
# A utility for writing out an error message to the user and then stopping
# execution. This seems to behave sensibly in both the interpreter and in
# a script.
def output_and_die(msg):
sys.stderr.write(msg)
sys.exit(1)
# Given an HTTPResponse object, try to read it's content and interpret as
# JSON--or die trying.
def load_json_or_die(response):
# First check that the response object actually exists.
if not response:
msg = "ERROR: no response object. \n\n"
output_and_die(msg)
# Next, try to read the content of the response.
try:
response_content = response.read()
except Exception, e:
msg = "ERROR: problem reading response. \n\n"
output_and_die(msg)
# Finally, try to create a dict by decoding the response as JSON.
rdict = None
try:
rdict = json.loads(response_content)
except Exception, e:
msg = "ERROR: got unexpected content from the server:\n\n"
msg += response_content + "\n\n"
output_and_die(msg)
return rdict
#-----------------------------------------------------------------
# HTTP/S Proxy classes
# Taken from: http://code.activestate.com/recipes/456195/
class ProxyHTTPConnection(httplib.HTTPConnection):
_ports = {'http' : 80, 'https' : 443}
def request(self, method, url, body=None, headers={}):
#request is called before connect, so can interpret url and get
#real host/port to be used to make CONNECT request to proxy
o = urlparse(url)
proto = o.scheme
port = o.port
host = o.hostname
if proto is None:
raise ValueError, "unknown URL type: %s" % url
if port is None:
try:
port = self._ports[proto]
except KeyError:
raise ValueError, "unknown protocol for: %s" % url
self._real_host = host
self._real_port = port
httplib.HTTPConnection.request(self, method, url, body, headers)
def connect(self):
httplib.HTTPConnection.connect(self)
#send proxy CONNECT request
self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self._real_host, self._real_port))
#expect a HTTP/1.0 200 Connection established
response = self.response_class(self.sock, strict=self.strict, method=self._method)
(version, code, message) = response._read_status()
#probably here we can handle auth requests...
if code != 200:
#proxy returned and error, abort connection, and raise exception
self.close()
raise socket.error, "Proxy connection failed: %d %s" % (code, message.strip())
#eat up header block from proxy....
while True:
#should not use directly fp probably
line = response.fp.readline()
if line == '\r\n': break
class ProxyHTTPSConnection(ProxyHTTPConnection):
default_port = 443
def __init__(self, host, port = None, key_file = None, cert_file = None,
strict = None, context = None):
ProxyHTTPConnection.__init__(self, host, port)
self.key_file = key_file
self.cert_file = cert_file
self.context = context
def connect(self):
ProxyHTTPConnection.connect(self)
#make the sock ssl-aware
if sys.hexversion < 0x20709f0:
ssl = socket.ssl(self.sock, self.key_file, self.cert_file)
self.sock = httplib.FakeSocket(self.sock, ssl)
else:
self.sock = self.context.wrap_socket(self.sock)
#------------------------------------------------------------------
# GraceDB
......@@ -67,7 +207,8 @@ class GraceDb(EcpRest):
@property
def service_info(self):
if not self._service_info:
self._service_info = self.request("GET", self.service_url).json()
r = self.request("GET", self.service_url)
self._service_info = r.json()
return self._service_info
@property
......@@ -130,7 +271,7 @@ class GraceDb(EcpRest):
if name == input_value][0]
# Search and filecontents are optional when creating an event.
def createEvent(self, group, pipeline, filename, search=None, filecontents=None):
def createEvent(self, group, pipeline, filename, search=None, filecontents=None, **kwargs):
"""Create a new GraceDB event
Required args: group, pipeline, filename
......@@ -171,6 +312,11 @@ class GraceDb(EcpRest):
]
if search:
fields.append(('search', search))
# Update fields with additional keyword arguments
for key, value in kwargs.iteritems():
fields.append((key, value))
files = [('eventFile', filename, filecontents)]
# Python httplib bug? unicode link
uri = str(self.links['events'])
......@@ -458,6 +604,64 @@ class GraceDb(EcpRest):
body.update(**kwargs)
return self.post(uri, body=body)
def emobservations(self, graceid):
"""Given a GraceID, get a list of EM observation entries
Example:
>>> g = GraceDb()
>>> r = g.emobserations('T101383')
>>> full_dictionary = r.json() # Convert the response to a dictionary
>>> emo_list = full_dictionary['observations'] # Pull out a list of EMO dicts
"""
template = self.templates['emobservation-list-template']
uri = template.format(graceid=graceid)
return self.get(uri)
def writeEMObservation(self, graceid, group, raList, raWidthList,
decList, decWidthList, startTimeList, durationList, comment=None):
"""Write an EM observation entry
Required args: graceid, group, raList, decList, raWidthList,
decWidthList, startTimeList, durationList
The various lists arguments should contain Python lists or
comma-separated values (or a single value). Start times are
in ISO 8601 UTC. Durations are in seconds.
(Note that 'group' here is the name of the EM MOU group, not
the LVC data analysis group responsible for the original detection.)
"""
# validate facility, waveband, eel_status, and obs_status
if not group in self.em_groups:
raise ValueError("group must be one of %s" % self.em_groups)
# NOTE: One could do validation of the various list inputs here
# rather than relying on the server.
# These arguments can consist of a single element, a python
# list, or a string. Transform the list args to csv (unless they
# already are)
raList, raWidthList, decList, decWidthList, startTimeList, durationList = map(
cleanListInput,
[raList, raWidthList, decList, decWidthList, startTimeList, durationList])
template = self.templates['emobservation-list-template']
uri = template.format(graceid=graceid)
body = {
'group' : group,
'raList' : raList,
'raWidthList': raWidthList,
'decList' : decList,
'decWidthList': decWidthList,
'startTimeList': startTimeList,
'durationList': durationList,
'comment': comment,
}
return self.post(uri, body=body)
def labels(self, graceid, label=""):
"""Get a list of labels for an event
......@@ -714,6 +918,33 @@ class GraceDbBasic(GraceDb):
self.service_url = service_url
self._service_info = None
# When there is a problem with the SSL connection or authentication,
# either conn.request() or conn.getresponse() will throw an exception.
def get_response(self, conn):
try:
return conn.getresponse()
except ssl.SSLError, e:
msg = "\nERROR \n\n"
msg += "Problem establishing secure connection: %s \n\n" % str(e)
output_and_die(msg)
except Exception, e:
msg = "\nERROR \n\n"
msg += "%s \n\n" % str(e)
output_and_die(msg)
# A wrapper for making the request.
def make_request(self, conn, *args, **kwargs):
try:
conn.request(*args, **kwargs)
except ssl.SSLError, e:
msg = "\nERROR \n\n"
msg += "Problem establishing secure connection: %s \n\n" % str(e)
output_and_die(msg)
except Exception, e:
msg = "\nERROR \n\n"
msg += "%s \n\n" % str(e)
output_and_die(msg)
def request(self, method, url, body=None, headers=None):
# Bug in Python (versions < 2.7.1 (?))
# http://bugs.python.org/issue11898
......@@ -727,11 +958,21 @@ class GraceDbBasic(GraceDb):
conn = self.getConnection()
headers = headers or {}
headers.update(self.authn_header)
conn.request(method, url, body, headers)
response = conn.getresponse()
self.make_request(conn, method, url, body, headers)
response = self.get_response(conn)
# Catch the 401 unauthorized response before sending to adjust
# response. Effectively, the 401 response will have special status.
if response.status == 401:
try:
msg = "\nERROR: %s \n\n" % json.loads(response.read())['error']
except:
msg = "\nERROR: \n\n"
msg += "Please check the username/password in your .netrc file. \n"
msg += "Note: If your password is more than a year old, you will \n"
msg += "need to use the web interface to generate a new one. \n\n"
output_and_die(msg)
return self.adjustResponse(response)
#-----------------------------------------------------------------
# HTTP upload encoding
......
ligo/gracedb/test/test.py
View file @ 6f1a8fec
# -*- coding: utf-8 -*-
# Copyright (C) Brian Moe, Branson Stephens (2015)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
import unittest
import random
import os
from datetime import datetime
import time
from ligo.gracedb.rest import GraceDb, GraceDbBasic
......@@ -36,8 +55,9 @@ from ligo.gracedb.rest import GraceDb, GraceDbBasic
# X509_USER_KEY
TEST_SERVICE = "https://moe.phys.uwm.edu/branson/apiweb/"
TEST_SP_SESSION_ENDPOINT = "https://moe.phys.uwm.edu/Shibboleth.sso/Session"
TEST_SERVICE = "https://gracedb-test.ligo.org/apiweb/"
TEST_SP_SESSION_ENDPOINT = "https://gracedb-test.ligo.org/Shibboleth.sso/Session"
SLEEP_TIME = 1
class TestGracedb(unittest.TestCase):
"""
......@@ -81,22 +101,31 @@ class TestGracedb(unittest.TestCase):
self.assertTrue('numRows' in logs)
pass
def test_create_embb_log(self):
"""Create an EMBB log entry."""
def test_create_emobservation(self):
"""Create an EM observation entry."""
comment = "Message is {0}".format(random.random())
resp = gracedb.writeEel(eventId, 'Test', 'em.gamma',
'FO', 'TE', comment=comment, instrument='Test')
# Let's put in some made-up values
raList = [1.0,1.0,1.0]
raWidthList = 1.0
decList = [1.0,1.0,1.0]
decWidthList = 1.0
dt = datetime(1900,1,1,1,1,1)
startTimeList = [dt.isoformat() for i in range(3)]
durationList = 1.0
resp = gracedb.writeEMObservation(eventId, 'Test',
raList, raWidthList, decList, decWidthList,
startTimeList, durationList, comment)
self.assertEqual(resp.status, 201)
new_embb_log_uri = resp.info().getheader('Location')
new_embb_log = resp.json()
self.assertEqual(new_embb_log_uri, new_embb_log['self'])
check_new_embb_log = gracedb.get(new_embb_log_uri).json()
self.assertEqual(check_new_embb_log['comment'], comment)
new_emobservation_uri = resp.info().getheader('Location')
new_emobservation = resp.json()
self.assertEqual(new_emobservation_uri, new_emobservation['self'])
check_new_emobservation = gracedb.get(new_emobservation_uri).json()
self.assertEqual(check_new_emobservation['comment'], comment)
def test_get_embb_log(self):
"""Retrieve EMBB event log"""
eels = gracedb.eels(eventId).json()
self.assertTrue('numRows' in eels)
def test_get_emobservations(self):
"""Retrieve EM Observation List"""
emos = gracedb.emobservations(eventId).json()
self.assertTrue('numRows' in emos)
def test_upload_large_file(self):
"""Upload a large file. Issue https://bugs.ligo.org/redmine/issues/951"""
......@@ -176,13 +205,14 @@ class TestGracedb(unittest.TestCase):
def test_create_cwb(self):
"""Create a CWB event"""
"""burst-cwb.txt"""
time.sleep(SLEEP_TIME)
eventFile = os.path.join(testdatadir, "burst-cwb.txt")
r = gracedb.createEvent("Test", "CWB", eventFile)
self.assertEqual(r.status, 201) # CREATED
cwb_event = r.json()
self.assertEqual(cwb_event['group'], "Test")
self.assertEqual(cwb_event['pipeline'], "CWB")
self.assertEqual(cwb_event['gpstime'], 1042312876)
self.assertEqual(float(cwb_event['gpstime']), 1042312876.5090)
def test_create_lowmass(self):
"""Create a Low Mass event"""
......@@ -193,21 +223,36 @@ class TestGracedb(unittest.TestCase):
def test_create_mbta(self):
"""Create an MBTA event"""
"""cbc-mbta.xml"""
time.sleep(SLEEP_TIME)
eventFile = os.path.join(testdatadir, "cbc-mbta.xml")
mbta_event = gracedb.createEvent(
"Test", "MBTAOnline", eventFile).json()
self.assertEqual(mbta_event['group'], "Test")
self.assertEqual(mbta_event['pipeline'], "MBTAOnline")
self.assertEqual(mbta_event['gpstime'], 1078903329)
self.assertEqual(float(mbta_event['gpstime']), 1078903329.421037)
self.assertEqual(mbta_event['far'], 4.006953918826065e-7)
def test_create_hardwareinjection(self):
"""Create a HardwareInjection event"""
"""sim-inj.xml"""
time.sleep(SLEEP_TIME)
eventFile = os.path.join(testdatadir, "sim-inj.xml")
hardwareinjection_event = gracedb.createEvent(
"Test", "HardwareInjection", eventFile,
instrument="H1", source_channel="",
destination_channel="").json()
self.assertEqual(hardwareinjection_event['group'], "Test")
self.assertEqual(hardwareinjection_event['pipeline'], "HardwareInjection")
self.assertEqual(hardwareinjection_event['instruments'], "H1")
def test_replace_event(self):
time.sleep(SLEEP_TIME)
graceid = eventId
old_event = gracedb.event(graceid).json()
self.assertEqual(old_event['group'], "Test")
self.assertEqual(old_event['search'], "LowMass")
self.assertEqual(old_event['gpstime'], 971609248)
self.assertEqual(float(old_event['gpstime']), 971609248.151741)
replacementFile = os.path.join(testdatadir, "cbc-lm2.xml")
......@@ -217,7 +262,7 @@ class TestGracedb(unittest.TestCase):
new_event = gracedb.event(graceid).json()
self.assertEqual(new_event['group'], "Test")
self.assertEqual(new_event['search'], "LowMass")
self.assertEqual(new_event['gpstime'], 971609249)
self.assertEqual(float(new_event['gpstime']), 971609249.151741)
def test_upload_binary(self):
"""
......@@ -271,7 +316,7 @@ class TestGracedb(unittest.TestCase):
def test_gittag(self):
# try to make sure GIT_TAG is set properly.
import errno
version = "1.17"
version = "1.20"
try:
# If we are in the source dir (setup.py is available)
# make sure the version above agrees.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment