Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Duncan Macleod
gracedb-client
Commits
6f1a8fec
Commit
6f1a8fec
authored
Apr 13, 2016
by
Branson Stephens
Browse files
Fixed some bugs resulting from the merger.
parent
0174775a
Changes
2
Hide whitespace changes
Inline
Side-by-side
ligo/gracedb/rest.py
View file @
6f1a8fec
# Copyright (C)
2012 LIGO Scientific Collaboration
# -*- coding: utf-8 -*-
# Copyright (C)
Brian Moe, Branson Stephens (2015)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
# This file is part of gracedb
#
#
This program is
distribute
d
i
n the hope that it will be useful, but
#
WITHOUT ANY WARRANTY; without even the implied warranty of
#
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
#
Public License for more details
.
#
gracedb is free software: you can re
distribute i
t and/or modify
#
it under the terms of the GNU General Public License as published by
#
the Free Software Foundation, either version 3 of the License, or
#
(at your option) any later version
.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
import
httplib
,
socket
,
ssl
import
mimetypes
import
urllib
import
os
,
sys
import
json
from
urlparse
import
urlparse
from
ecp_client
import
EcpRest
from
base64
import
b64encode
import
netrc
from
ecp_client
import
EcpRest
DEFAULT_SERVICE_URL
=
"https://gracedb.ligo.org/apiweb/"
DEFAULT_BASIC_SERVICE_URL
=
"https://gracedb.ligo.org/apibasic/"
DEFAULT_SP_SESSION_ENDPOINT
=
"https://gracedb.ligo.org/Shibboleth.sso/Session"
KNOWN_TEST_HOSTS
=
[
'moe.phys.uwm.edu'
,
'embb-dev.ligo.caltech.edu'
,
'simdb.phys.uwm.edu'
,]
#---------------------------------------------------------------------
# This monkey patch forces TLSv1 if the python version is 2.6.6.
# It was introduced because clients connection from CIT *occasionally*
# try to use SSLv3. See:
# http://stackoverflow.com/questions/18669457/python-httplib-ssl23-get-server-hellounknown-protocol
#---------------------------------------------------------------------
if
sys
.
hexversion
<
0x20709f0
:
wrap_socket_orig
=
ssl
.
wrap_socket
def
wrap_socket_patched
(
sock
,
keyfile
=
None
,
certfile
=
None
,
server_side
=
False
,
cert_reqs
=
ssl
.
CERT_NONE
,
ssl_version
=
ssl
.
PROTOCOL_TLSv1
,
ca_certs
=
None
,
do_handshake_on_connect
=
True
,
suppress_ragged_eofs
=
True
):
return
wrap_socket_orig
(
sock
,
keyfile
,
certfile
,
server_side
,
cert_reqs
,
ssl_version
,
ca_certs
,
do_handshake_on_connect
,
suppress_ragged_eofs
)
ssl
.
wrap_socket
=
wrap_socket_patched
#-----------------------------------------------------------------
# Utilities
# XXX It would be nice to get rid of this if we can.
# It seems that you can pass python lists via the requests package.
# That would make putting our lists into comma-separated strings
# unnecessary.
def
cleanListInput
(
list_arg
):
stringified_list
=
list_arg
if
isinstance
(
list_arg
,
float
)
or
isinstance
(
list_arg
,
int
):
stringified_value
=
str
(
list_arg
)
return
stringified_value
if
not
isinstance
(
list_arg
,
basestring
):
stringified_list
=
','
.
join
(
map
(
str
,
list_arg
))
return
stringified_list
# The following are used to check whether a user has tried to use
# an expired certificate.
# A utility for writing out an error message to the user and then stopping
# execution. This seems to behave sensibly in both the interpreter and in
# a script.
def
output_and_die
(
msg
):
sys
.
stderr
.
write
(
msg
)
sys
.
exit
(
1
)
# Given an HTTPResponse object, try to read it's content and interpret as
# JSON--or die trying.
def
load_json_or_die
(
response
):
# First check that the response object actually exists.
if
not
response
:
msg
=
"ERROR: no response object.
\n\n
"
output_and_die
(
msg
)
# Next, try to read the content of the response.
try
:
response_content
=
response
.
read
()
except
Exception
,
e
:
msg
=
"ERROR: problem reading response.
\n\n
"
output_and_die
(
msg
)
# Finally, try to create a dict by decoding the response as JSON.
rdict
=
None
try
:
rdict
=
json
.
loads
(
response_content
)
except
Exception
,
e
:
msg
=
"ERROR: got unexpected content from the server:
\n\n
"
msg
+=
response_content
+
"
\n\n
"
output_and_die
(
msg
)
return
rdict
#-----------------------------------------------------------------
# HTTP/S Proxy classes
# Taken from: http://code.activestate.com/recipes/456195/
class
ProxyHTTPConnection
(
httplib
.
HTTPConnection
):
_ports
=
{
'http'
:
80
,
'https'
:
443
}
def
request
(
self
,
method
,
url
,
body
=
None
,
headers
=
{}):
#request is called before connect, so can interpret url and get
#real host/port to be used to make CONNECT request to proxy
o
=
urlparse
(
url
)
proto
=
o
.
scheme
port
=
o
.
port
host
=
o
.
hostname
if
proto
is
None
:
raise
ValueError
,
"unknown URL type: %s"
%
url
if
port
is
None
:
try
:
port
=
self
.
_ports
[
proto
]
except
KeyError
:
raise
ValueError
,
"unknown protocol for: %s"
%
url
self
.
_real_host
=
host
self
.
_real_port
=
port
httplib
.
HTTPConnection
.
request
(
self
,
method
,
url
,
body
,
headers
)
def
connect
(
self
):
httplib
.
HTTPConnection
.
connect
(
self
)
#send proxy CONNECT request
self
.
send
(
"CONNECT %s:%d HTTP/1.0
\r\n\r\n
"
%
(
self
.
_real_host
,
self
.
_real_port
))
#expect a HTTP/1.0 200 Connection established
response
=
self
.
response_class
(
self
.
sock
,
strict
=
self
.
strict
,
method
=
self
.
_method
)
(
version
,
code
,
message
)
=
response
.
_read_status
()
#probably here we can handle auth requests...
if
code
!=
200
:
#proxy returned and error, abort connection, and raise exception
self
.
close
()
raise
socket
.
error
,
"Proxy connection failed: %d %s"
%
(
code
,
message
.
strip
())
#eat up header block from proxy....
while
True
:
#should not use directly fp probably
line
=
response
.
fp
.
readline
()
if
line
==
'
\r\n
'
:
break
class
ProxyHTTPSConnection
(
ProxyHTTPConnection
):
default_port
=
443
def
__init__
(
self
,
host
,
port
=
None
,
key_file
=
None
,
cert_file
=
None
,
strict
=
None
,
context
=
None
):
ProxyHTTPConnection
.
__init__
(
self
,
host
,
port
)
self
.
key_file
=
key_file
self
.
cert_file
=
cert_file
self
.
context
=
context
def
connect
(
self
):
ProxyHTTPConnection
.
connect
(
self
)
#make the sock ssl-aware
if
sys
.
hexversion
<
0x20709f0
:
ssl
=
socket
.
ssl
(
self
.
sock
,
self
.
key_file
,
self
.
cert_file
)
self
.
sock
=
httplib
.
FakeSocket
(
self
.
sock
,
ssl
)
else
:
self
.
sock
=
self
.
context
.
wrap_socket
(
self
.
sock
)
#------------------------------------------------------------------
# GraceDB
...
...
@@ -67,7 +207,8 @@ class GraceDb(EcpRest):
@
property
def
service_info
(
self
):
if
not
self
.
_service_info
:
self
.
_service_info
=
self
.
request
(
"GET"
,
self
.
service_url
).
json
()
r
=
self
.
request
(
"GET"
,
self
.
service_url
)
self
.
_service_info
=
r
.
json
()
return
self
.
_service_info
@
property
...
...
@@ -130,7 +271,7 @@ class GraceDb(EcpRest):
if
name
==
input_value
][
0
]
# Search and filecontents are optional when creating an event.
def
createEvent
(
self
,
group
,
pipeline
,
filename
,
search
=
None
,
filecontents
=
None
):
def
createEvent
(
self
,
group
,
pipeline
,
filename
,
search
=
None
,
filecontents
=
None
,
**
kwargs
):
"""Create a new GraceDB event
Required args: group, pipeline, filename
...
...
@@ -171,6 +312,11 @@ class GraceDb(EcpRest):
]
if
search
:
fields
.
append
((
'search'
,
search
))
# Update fields with additional keyword arguments
for
key
,
value
in
kwargs
.
iteritems
():
fields
.
append
((
key
,
value
))
files
=
[(
'eventFile'
,
filename
,
filecontents
)]
# Python httplib bug? unicode link
uri
=
str
(
self
.
links
[
'events'
])
...
...
@@ -458,6 +604,64 @@ class GraceDb(EcpRest):
body
.
update
(
**
kwargs
)
return
self
.
post
(
uri
,
body
=
body
)
def
emobservations
(
self
,
graceid
):
"""Given a GraceID, get a list of EM observation entries
Example:
>>> g = GraceDb()
>>> r = g.emobserations('T101383')
>>> full_dictionary = r.json() # Convert the response to a dictionary
>>> emo_list = full_dictionary['observations'] # Pull out a list of EMO dicts
"""
template
=
self
.
templates
[
'emobservation-list-template'
]
uri
=
template
.
format
(
graceid
=
graceid
)
return
self
.
get
(
uri
)
def
writeEMObservation
(
self
,
graceid
,
group
,
raList
,
raWidthList
,
decList
,
decWidthList
,
startTimeList
,
durationList
,
comment
=
None
):
"""Write an EM observation entry
Required args: graceid, group, raList, decList, raWidthList,
decWidthList, startTimeList, durationList
The various lists arguments should contain Python lists or
comma-separated values (or a single value). Start times are
in ISO 8601 UTC. Durations are in seconds.
(Note that 'group' here is the name of the EM MOU group, not
the LVC data analysis group responsible for the original detection.)
"""
# validate facility, waveband, eel_status, and obs_status
if
not
group
in
self
.
em_groups
:
raise
ValueError
(
"group must be one of %s"
%
self
.
em_groups
)
# NOTE: One could do validation of the various list inputs here
# rather than relying on the server.
# These arguments can consist of a single element, a python
# list, or a string. Transform the list args to csv (unless they
# already are)
raList
,
raWidthList
,
decList
,
decWidthList
,
startTimeList
,
durationList
=
map
(
cleanListInput
,
[
raList
,
raWidthList
,
decList
,
decWidthList
,
startTimeList
,
durationList
])
template
=
self
.
templates
[
'emobservation-list-template'
]
uri
=
template
.
format
(
graceid
=
graceid
)
body
=
{
'group'
:
group
,
'raList'
:
raList
,
'raWidthList'
:
raWidthList
,
'decList'
:
decList
,
'decWidthList'
:
decWidthList
,
'startTimeList'
:
startTimeList
,
'durationList'
:
durationList
,
'comment'
:
comment
,
}
return
self
.
post
(
uri
,
body
=
body
)
def
labels
(
self
,
graceid
,
label
=
""
):
"""Get a list of labels for an event
...
...
@@ -714,6 +918,33 @@ class GraceDbBasic(GraceDb):
self
.
service_url
=
service_url
self
.
_service_info
=
None
# When there is a problem with the SSL connection or authentication,
# either conn.request() or conn.getresponse() will throw an exception.
def
get_response
(
self
,
conn
):
try
:
return
conn
.
getresponse
()
except
ssl
.
SSLError
,
e
:
msg
=
"
\n
ERROR
\n\n
"
msg
+=
"Problem establishing secure connection: %s
\n\n
"
%
str
(
e
)
output_and_die
(
msg
)
except
Exception
,
e
:
msg
=
"
\n
ERROR
\n\n
"
msg
+=
"%s
\n\n
"
%
str
(
e
)
output_and_die
(
msg
)
# A wrapper for making the request.
def
make_request
(
self
,
conn
,
*
args
,
**
kwargs
):
try
:
conn
.
request
(
*
args
,
**
kwargs
)
except
ssl
.
SSLError
,
e
:
msg
=
"
\n
ERROR
\n\n
"
msg
+=
"Problem establishing secure connection: %s
\n\n
"
%
str
(
e
)
output_and_die
(
msg
)
except
Exception
,
e
:
msg
=
"
\n
ERROR
\n\n
"
msg
+=
"%s
\n\n
"
%
str
(
e
)
output_and_die
(
msg
)
def
request
(
self
,
method
,
url
,
body
=
None
,
headers
=
None
):
# Bug in Python (versions < 2.7.1 (?))
# http://bugs.python.org/issue11898
...
...
@@ -727,11 +958,21 @@ class GraceDbBasic(GraceDb):
conn
=
self
.
getConnection
()
headers
=
headers
or
{}
headers
.
update
(
self
.
authn_header
)
conn
.
request
(
method
,
url
,
body
,
headers
)
response
=
conn
.
getresponse
()
self
.
make_request
(
conn
,
method
,
url
,
body
,
headers
)
response
=
self
.
get_response
(
conn
)
# Catch the 401 unauthorized response before sending to adjust
# response. Effectively, the 401 response will have special status.
if
response
.
status
==
401
:
try
:
msg
=
"
\n
ERROR: %s
\n\n
"
%
json
.
loads
(
response
.
read
())[
'error'
]
except
:
msg
=
"
\n
ERROR:
\n\n
"
msg
+=
"Please check the username/password in your .netrc file.
\n
"
msg
+=
"Note: If your password is more than a year old, you will
\n
"
msg
+=
"need to use the web interface to generate a new one.
\n\n
"
output_and_die
(
msg
)
return
self
.
adjustResponse
(
response
)
#-----------------------------------------------------------------
# HTTP upload encoding
...
...
ligo/gracedb/test/test.py
View file @
6f1a8fec
# -*- coding: utf-8 -*-
# Copyright (C) Brian Moe, Branson Stephens (2015)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
import
unittest
import
random
import
os
from
datetime
import
datetime
import
time
from
ligo.gracedb.rest
import
GraceDb
,
GraceDbBasic
...
...
@@ -36,8 +55,9 @@ from ligo.gracedb.rest import GraceDb, GraceDbBasic
# X509_USER_KEY
TEST_SERVICE
=
"https://moe.phys.uwm.edu/branson/apiweb/"
TEST_SP_SESSION_ENDPOINT
=
"https://moe.phys.uwm.edu/Shibboleth.sso/Session"
TEST_SERVICE
=
"https://gracedb-test.ligo.org/apiweb/"
TEST_SP_SESSION_ENDPOINT
=
"https://gracedb-test.ligo.org/Shibboleth.sso/Session"
SLEEP_TIME
=
1
class
TestGracedb
(
unittest
.
TestCase
):
"""
...
...
@@ -81,22 +101,31 @@ class TestGracedb(unittest.TestCase):
self
.
assertTrue
(
'numRows'
in
logs
)
pass
def
test_create_em
bb_log
(
self
):
"""Create an EM
BB log
entry."""
def
test_create_em
observation
(
self
):
"""Create an EM
observation
entry."""
comment
=
"Message is {0}"
.
format
(
random
.
random
())
resp
=
gracedb
.
writeEel
(
eventId
,
'Test'
,
'em.gamma'
,
'FO'
,
'TE'
,
comment
=
comment
,
instrument
=
'Test'
)
# Let's put in some made-up values
raList
=
[
1.0
,
1.0
,
1.0
]
raWidthList
=
1.0
decList
=
[
1.0
,
1.0
,
1.0
]
decWidthList
=
1.0
dt
=
datetime
(
1900
,
1
,
1
,
1
,
1
,
1
)
startTimeList
=
[
dt
.
isoformat
()
for
i
in
range
(
3
)]
durationList
=
1.0
resp
=
gracedb
.
writeEMObservation
(
eventId
,
'Test'
,
raList
,
raWidthList
,
decList
,
decWidthList
,
startTimeList
,
durationList
,
comment
)
self
.
assertEqual
(
resp
.
status
,
201
)
new_em
bb_log
_uri
=
resp
.
info
().
getheader
(
'Location'
)
new_em
bb_log
=
resp
.
json
()
self
.
assertEqual
(
new_em
bb_log
_uri
,
new_em
bb_log
[
'self'
])
check_new_em
bb_log
=
gracedb
.
get
(
new_em
bb_log
_uri
).
json
()
self
.
assertEqual
(
check_new_em
bb_log
[
'comment'
],
comment
)
new_em
observation
_uri
=
resp
.
info
().
getheader
(
'Location'
)
new_em
observation
=
resp
.
json
()
self
.
assertEqual
(
new_em
observation
_uri
,
new_em
observation
[
'self'
])
check_new_em
observation
=
gracedb
.
get
(
new_em
observation
_uri
).
json
()
self
.
assertEqual
(
check_new_em
observation
[
'comment'
],
comment
)
def
test_get_em
bb_log
(
self
):
"""Retrieve EM
BB event log
"""
e
el
s
=
gracedb
.
e
el
s
(
eventId
).
json
()
self
.
assertTrue
(
'numRows'
in
e
el
s
)
def
test_get_em
observations
(
self
):
"""Retrieve EM
Observation List
"""
e
mo
s
=
gracedb
.
e
mobservation
s
(
eventId
).
json
()
self
.
assertTrue
(
'numRows'
in
e
mo
s
)
def
test_upload_large_file
(
self
):
"""Upload a large file. Issue https://bugs.ligo.org/redmine/issues/951"""
...
...
@@ -176,13 +205,14 @@ class TestGracedb(unittest.TestCase):
def
test_create_cwb
(
self
):
"""Create a CWB event"""
"""burst-cwb.txt"""
time
.
sleep
(
SLEEP_TIME
)
eventFile
=
os
.
path
.
join
(
testdatadir
,
"burst-cwb.txt"
)
r
=
gracedb
.
createEvent
(
"Test"
,
"CWB"
,
eventFile
)
self
.
assertEqual
(
r
.
status
,
201
)
# CREATED
cwb_event
=
r
.
json
()
self
.
assertEqual
(
cwb_event
[
'group'
],
"Test"
)
self
.
assertEqual
(
cwb_event
[
'pipeline'
],
"CWB"
)
self
.
assertEqual
(
cwb_event
[
'gpstime'
],
1042312876
)
self
.
assertEqual
(
float
(
cwb_event
[
'gpstime'
]
)
,
1042312876
.5090
)
def
test_create_lowmass
(
self
):
"""Create a Low Mass event"""
...
...
@@ -193,21 +223,36 @@ class TestGracedb(unittest.TestCase):
def
test_create_mbta
(
self
):
"""Create an MBTA event"""
"""cbc-mbta.xml"""
time
.
sleep
(
SLEEP_TIME
)
eventFile
=
os
.
path
.
join
(
testdatadir
,
"cbc-mbta.xml"
)
mbta_event
=
gracedb
.
createEvent
(
"Test"
,
"MBTAOnline"
,
eventFile
).
json
()
self
.
assertEqual
(
mbta_event
[
'group'
],
"Test"
)
self
.
assertEqual
(
mbta_event
[
'pipeline'
],
"MBTAOnline"
)
self
.
assertEqual
(
mbta_event
[
'gpstime'
],
1078903329
)
self
.
assertEqual
(
float
(
mbta_event
[
'gpstime'
]
)
,
1078903329
.421037
)
self
.
assertEqual
(
mbta_event
[
'far'
],
4.006953918826065e-7
)
def
test_create_hardwareinjection
(
self
):
"""Create a HardwareInjection event"""
"""sim-inj.xml"""
time
.
sleep
(
SLEEP_TIME
)
eventFile
=
os
.
path
.
join
(
testdatadir
,
"sim-inj.xml"
)
hardwareinjection_event
=
gracedb
.
createEvent
(
"Test"
,
"HardwareInjection"
,
eventFile
,
instrument
=
"H1"
,
source_channel
=
""
,
destination_channel
=
""
).
json
()
self
.
assertEqual
(
hardwareinjection_event
[
'group'
],
"Test"
)
self
.
assertEqual
(
hardwareinjection_event
[
'pipeline'
],
"HardwareInjection"
)
self
.
assertEqual
(
hardwareinjection_event
[
'instruments'
],
"H1"
)
def
test_replace_event
(
self
):
time
.
sleep
(
SLEEP_TIME
)
graceid
=
eventId
old_event
=
gracedb
.
event
(
graceid
).
json
()
self
.
assertEqual
(
old_event
[
'group'
],
"Test"
)
self
.
assertEqual
(
old_event
[
'search'
],
"LowMass"
)
self
.
assertEqual
(
old_event
[
'gpstime'
],
971609248
)
self
.
assertEqual
(
float
(
old_event
[
'gpstime'
]
)
,
971609248
.151741
)
replacementFile
=
os
.
path
.
join
(
testdatadir
,
"cbc-lm2.xml"
)
...
...
@@ -217,7 +262,7 @@ class TestGracedb(unittest.TestCase):
new_event
=
gracedb
.
event
(
graceid
).
json
()
self
.
assertEqual
(
new_event
[
'group'
],
"Test"
)
self
.
assertEqual
(
new_event
[
'search'
],
"LowMass"
)
self
.
assertEqual
(
new_event
[
'gpstime'
],
971609249
)
self
.
assertEqual
(
float
(
new_event
[
'gpstime'
]
)
,
971609249
.151741
)
def
test_upload_binary
(
self
):
"""
...
...
@@ -271,7 +316,7 @@ class TestGracedb(unittest.TestCase):
def
test_gittag
(
self
):
# try to make sure GIT_TAG is set properly.
import
errno
version
=
"1.
17
"
version
=
"1.
20
"
try
:
# If we are in the source dir (setup.py is available)
# make sure the version above agrees.
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment