Commit 0174775a authored by Branson Stephens's avatar Branson Stephens

Merged in some changes from master.

parents 476d1bdf e1b9ae94
......@@ -2,3 +2,5 @@
MANIFEST
build
dist
install.sh
*.egg-info
recursive-include debian *
recursive-include ligo/gracedb/test *
include ligo-gracedb.spec
ligo-gracedb (1.20-1) unstable; urgency=low
* Improved error handling for expired or missing credentials
* Improved error handling when server returns non-JSON response
* Added --use-basic-auth option to command-line client
-- Branson Stephens <branson.stephens@ligo.org> Thu, 11 Feb 2016 15:00:00 -0600
ligo-gracedb (1.19.1-1) unstable; urgency=low
* Force TLSv1 for Python versions less than 2.7.9
* Changed adjustResponse to put retry-after in the JSON response for 429s
* Changed test-service to gracedb-test.ligo.org
* Introduced wait time to test suite
-- Branson Stephens <branson.stephens@ligo.org> Wed, 21 Oct 2015 11:00:00 -0500
ligo-gracedb (1.19-1) unstable; urgency=low
* bug fixes (comma separated strings for EMObservations, cli createLog call)
* capture of additional kwargs to facilitate HardwareInjection event upload
* packaging improvements (ligo as namespace package)
-- Branson Stephens <branson.stephens@ligo.org> Wed, 29 Jul 2015 14:00:00 -0500
ligo-gracedb (1.18-1) unstable; urgency=low
* Added comment to EM Observation record
* Allow python lists as arguments to writeEMObservation
* Changed ligo to namespace package
-- Branson Stephens <branson.stephens@ligo.org> Wed, 13 May 2015 14:00:00 -0500
ligo-gracedb (1.18.dev0-1) unstable; urgency=low
* New features for robotic basic auth
* New features for EM observation records
-- Branson Stephens <branson.stephens@ligo.org> Mon, 20 Apr 2015 16:30:00 -0500
ligo-gracedb (1.17-1) unstable; urgency=low
* Bugfix for python version incompatibility in 1.16
* New methods/tests for creating and retrieving VOEvents
* Bugfix for gittag test
-- Branson Stephens <branson.stephens@ligo.org> Wed, 25 Mar 2015 09:30:00 -0500
ligo-gracedb (1.16-1) unstable; urgency=low
* Fixes for glue 1.47 to the command line client
* Use SSLContext and explicitly turn off client-side server verification
......
......@@ -2,13 +2,13 @@ Source: ligo-gracedb
Maintainer: Branson Stephens <branson.stephens@ligo.org>
Section: python
Priority: optional
Build-Depends: debhelper (>= 7), python-all-dev
Build-Depends: debhelper (>= 7), python-all-dev,python-setuptools
Standards-Version: 3.8.4
X-Python-Version: >=2.6
Package: python-ligo-gracedb
Architecture: all
Depends: ${misc:Depends}, ${python:Depends}, python-m2crypto, python-cjson, python-ligo-common
Depends: ${misc:Depends}, ${python:Depends}, python-m2crypto, python-cjson, python-ligo-common, python-setuptools
XB-Python-Version: ${python:Versions}
Provides: ${python:Provides}
Description: Gravity Wave Candidate Event Database
......
%define name ligo-gracedb
%define version 1.16
%define unmangled_version 1.16
%define version 1.20
%define unmangled_version 1.20
%define release 1
Summary: Gravity Wave Candidate Event Database
......@@ -15,6 +15,7 @@ Prefix: %{_prefix}
BuildArch: noarch
Vendor: Branson Stephens <branson.stephens@ligo.org>
Requires: ligo-common m2crypto
BuildRequires: python-setuptools
Url: http://www.lsc-group.phys.uwm.edu/daswg/gracedb.html
%description
......
# -*- coding: utf-8 -*-
# Copyright (C) Brian Moe, Branson Stephens (2015)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
__import__('pkg_resources').declare_namespace(__name__)
# -*- coding: utf-8 -*-
# Copyright (C) Brian Moe, Branson Stephens (2015)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
__all__ = ["cli", "rest"]
GIT_TAG = 'gracedb-1.16-1'
GIT_TAG = 'gracedb-1.20-1'
# issue 717. Required for backward compatibility -- make sure "from ligo import gracedb"
# works as it used to.
......
#!/usr/bin/env python
# Copyright (C) 2012 LIGO Scientific Collaboration
# Copyright (C) Brian Moe, Branson Stephens (2015)
#
# This file is part of gracedb
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
import os, sys, shutil, urllib
import json
from ligo.gracedb.rest import GraceDb
from ligo.gracedb.rest import GraceDb, load_json_or_die, GraceDbBasic
DEFAULT_SERVICE_URL = "https://gracedb.ligo.org/gracedb/api"
GIT_TAG = 'gracedb-1.16-1'
DEFAULT_BASIC_URL = "https://gracedb.ligo.org/apibasic/"
GIT_TAG = 'gracedb-1.20-1'
DEFAULT_COLUMNS = "graceid,labels,group,pipeline,search,far,gpstime,created,dataurl"
......@@ -74,46 +76,50 @@ typeCodeMap = {
}
validTypes = typeCodeMap.keys()
#-----------------------------------------------------------------
# Web Service Client
class Client(GraceDb):
def __init__(self,
url=DEFAULT_SERVICE_URL,
proxy_host=None, proxy_port=3128,
credentials=None,
*args, **kwargs):
if (url[-1] != '/'):
url += '/'
self.url = url
super(Client, self).__init__(url, proxy_host, proxy_port,
credentials, *args, **kwargs)
def download(self, graceid, filename, destfile):
# Check that we *could* write the file before we
# go to the trouble of getting it. Also, try not
# to open a file until we know we have data.
if not isinstance(destfile, file) and destfile != "-":
if not os.access(os.path.dirname(os.path.abspath(destfile)), os.W_OK):
raise IOError("%s: Permission denied" % destfile)
response = self.files(graceid, filename)
if response.status == 200:
if not isinstance(destfile, file):
if destfile == '-':
destfile = sys.stdout
else:
destfile = open(destfile, "w")
shutil.copyfileobj(response, destfile)
return 0
else:
return "Error. (%d) %s" % (response.status, response.reason)
# Hamstring 'adjustResponse' from the example REST client.
# We don't want it messing with the response from the server.
def adjustResponse(self, response):
response.json = lambda: json.loads(response.read())
return response
# This is a factory for client classes.
# Given a base client class with the correct properties, derive
# one suitable for use with this command-line tool. In practice,
# the base class here will either be the X509 auth GraceDb class
# or the basic auth GraceDbBasic class.
def derive_client(ClientBase=GraceDb):
class client(ClientBase):
def __init__(self, url=DEFAULT_SERVICE_URL, *args, **kwargs):
if (url[-1] != '/'):
url += '/'
self.url = url
super(client, self).__init__(url, *args, **kwargs)
def download(self, graceid, filename, destfile):
# Check that we *could* write the file before we
# go to the trouble of getting it. Also, try not
# to open a file until we know we have data.
if not isinstance(destfile, file) and destfile != "-":
if not os.access(os.path.dirname(os.path.abspath(destfile)), os.W_OK):
raise IOError("%s: Permission denied" % destfile)
response = self.files(graceid, filename)
if response.status == 200:
if not isinstance(destfile, file):
if destfile == '-':
destfile = sys.stdout
else:
destfile = open(destfile, "w")
shutil.copyfileobj(response, destfile)
return 0
else:
return "Error. (%d) %s" % (response.status, response.reason)
# Hamstring 'adjustResponse' from the example REST client.
# We don't want it messing with the response from the server.
def adjustResponse(self, response):
response.json = lambda: load_json_or_die(response)
return response
return client
# Get X509 client in global scope
# This is required for passing the unit tests.
Client = derive_client()
#-----------------------------------------------------------------
# Main
......@@ -181,6 +187,9 @@ def main():
analysis type(s), labels, etc. Note that text is case insensitive
Example: %%prog search G0100..G0200 mbta LUMIN_GO
%%prog version
Display version information.
Environment Variables:
GRACEDB_SERVICE_URL (can be overridden by --service-url)
HTTP_PROXY (can be overridden by --proxy)
......@@ -232,6 +241,10 @@ Longer strings will be truncated.""" % {
help="tag display name (ignored for existing tags)",
default=None
)
op.add_option("-b", "--use-basic-auth", dest="use_basic_auth",
help="Use basic auth with a .netrc file. Available to non-LVC members only.",
action="store_true", default=False
)
options, args = op.parse_args()
......@@ -254,6 +267,23 @@ Longer strings will be truncated.""" % {
os.environ.get('GRACEDB_SERVICE_URL', None) or \
DEFAULT_SERVICE_URL
# If the user requested a specific service, but also wants basic auth,
# then the service had better be a basic auth endpoint. Otherwise die.
# On the other hand, if the user did not specify a service url, then we
# will use the default basic URL if basic auth was requested.
if options.use_basic_auth:
if options.service or os.environ.get('GRACEDB_SERVICE_URL', None):
if 'basic' not in service:
error("To use the basic auth client, specify a basic auth service URL or use the default.")
exit(1)
else:
service = DEFAULT_BASIC_URL
# Client subclass according to preferred auth method.
global Client
if options.use_basic_auth:
Client = derive_client(GraceDbBasic)
if options.alert is not None:
warning("alert option is deprecated. Alerts are now sent by default.")
......@@ -273,6 +303,11 @@ Longer strings will be truncated.""" % {
if len(args) < 1:
op.error("not enough arguments")
elif args[0] == 'version':
import pkg_resources
version = pkg_resources.require("ligo-gracedb")[0].version
print "GraceDB Client v. %s" % version
exit(0)
elif args[0] == 'ping':
response = client.ping()
output("Client groups: %s" % client.groups)
......@@ -333,7 +368,7 @@ Longer strings will be truncated.""" % {
op.error("not enough arguments for log")
graceid = args[1]
message = " ".join(args[2:])
response = client.writeLog(graceid, message, options.tagName, options.tagDispName)
response = client.writeLog(graceid, message, tagname=options.tagName, displayName=options.tagDispName)
elif args[0] == 'tag':
if options.tagName:
if len(args) != 2:
......
# Copyright (C) 2012 LIGO Scientific Collaboration
# -*- coding: utf-8 -*-
# Copyright (C) Leo Singer, Brian Moe, Branson Stephens (2015)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
# This file is part of gracedb
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import
"""
Some convenience logging classes courtesy of Leo Singer, provided as is.
......@@ -37,13 +39,8 @@ log.addHandler(ligo.gracedb.logging.GraceDbLogHandler(gracedb, graceid))
#
log.warn("this is a warning")
"""
# Perform explicit absolute import of Python standard library logging module.
# 'import logging' would not work here, because it would be interpreted as this
# module itself.
# module itself.g
logging = __import__('logging', level=0)
import logging
class GraceDbLogStream(object):
def __init__(self, gracedb, graceid):
......
......@@ -22,10 +22,12 @@ import os, sys
import json
from urlparse import urlparse
from ecp_client import EcpRest
from base64 import b64encode
import netrc
DEFAULT_SERVICE_URL = "https://gracedb.ligo.org/api/"
DEFAULT_SERVICE_URL = "https://gracedb.ligo.org/apiweb/"
DEFAULT_BASIC_SERVICE_URL = "https://gracedb.ligo.org/apibasic/"
DEFAULT_SP_SESSION_ENDPOINT = "https://gracedb.ligo.org/Shibboleth.sso/Session"
KNOWN_TEST_HOSTS = ['moe.phys.uwm.edu', 'embb-dev.ligo.caltech.ed', 'simdb.phys.uwm.edu',]
#------------------------------------------------------------------
# GraceDB
......@@ -626,6 +628,112 @@ class GraceDb(EcpRest):
#-----------------------------------------------------------------
# TBD
# Media Types
# Root
# Collection
# Event
# Log
# File
# Label
#-----------------------------------------------------------------
# Basic auth for the LV-EM users
class GraceDbBasic(GraceDb):
"""Example GraceDb REST client with basic auth
The GraceDB service URL may be passed to the constructor
if an alternate GraceDb instance is desired:
>>> g = GraceDb("https://alternate.gracedb.edu/api/")
>>> r = g.ping()
The proxy_host and proxy_port may also be passed in if accessing
GraceDB behind a proxy. For other kwargs accepted by the constructor,
consult the source code.
"""
def __init__(self, service_url=DEFAULT_SERVICE_URL,
proxy_host=None, proxy_port=3128, username=None, password=None,
*args, **kwargs):
o = urlparse(service_url)
port = o.port
host = o.hostname
port = port or 443
if not username or not password:
try:
username, account, password = netrc.netrc().authenticators(host)
except:
pass
if not username or not password:
msg = "Could not find user credentials. "
msg +="Please use a .netrc file or provide username and password."
raise ValueError(msg)
# Construct authorization header
userAndPass = b64encode(b"%s:%s" % (username, password)).decode("ascii")
self.authn_header = { 'Authorization' : 'Basic %s' % userAndPass }
# Versions of Python earlier than 2.7.9 don't use SSL Context
# objects for this purpose, and do not do any server cert verification.
ssl_context = None
if sys.hexversion >= 0x20709f0:
# Use the new method with SSL Context
# Prepare SSL context
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
# Generally speaking, test boxes use cheap/free certs from the LIGO CA.
# These cannot be verified by the client.
if host in KNOWN_TEST_HOSTS:
ssl_context.verify_mode = ssl.CERT_NONE
else:
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.check_hostname = True
# Find the various CA cert bundles stored on the system
ssl_context.load_default_certs()
if proxy_host:
self.connector = lambda: ProxyHTTPSConnection(proxy_host, proxy_port, context=ssl_context)
else:
self.connector = lambda: httplib.HTTPSConnection(host, port, context=ssl_context)
else:
# Using and older version of python. We'll pass in the cert and key files.
if proxy_host:
self.connector = lambda: ProxyHTTPSConnection(proxy_host, proxy_port)
else:
self.connector = lambda: httplib.HTTPSConnection(host, port)
self.service_url = service_url
self._service_info = None
def request(self, method, url, body=None, headers=None):
# Bug in Python (versions < 2.7.1 (?))
# http://bugs.python.org/issue11898
# if the URL is unicode and the body of a request is binary,
# the POST/PUT action fails because it tries to concatenate
# the two which fails due to encoding problems.
# Workaround is to cast all URLs to str.
# This is probably bad in general,
# but for our purposes, today, this will do.
url = url and str(url)
conn = self.getConnection()
headers = headers or {}
headers.update(self.authn_header)
conn.request(method, url, body, headers)
response = conn.getresponse()
return self.adjustResponse(response)
#-----------------------------------------------------------------
# HTTP upload encoding
# Taken from http://code.activestate.com/recipes/146306/
......
# -*- coding: utf-8 -*-
# Copyright (C) Brian Moe, Branson Stephens (2015)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
<?xml version='1.0' encoding='utf-8'?>
<!DOCTYPE LIGO_LW SYSTEM "http://ldas-sw.ligo.caltech.edu/doc/ligolwAPI/html/ligolw_dtd.txt">
<LIGO_LW>
<Table Name="process:table">
<Column Type="lstring" Name="process:comment"/>
<Column Type="lstring" Name="process:node"/>
<Column Type="lstring" Name="process:domain"/>
<Column Type="int_4s" Name="process:unix_procid"/>
<Column Type="int_4s" Name="process:start_time"/>
<Column Type="ilwd:char" Name="process:process_id"/>
<Column Type="int_4s" Name="process:is_online"/>
<Column Type="lstring" Name="process:ifos"/>
<Column Type="int_4s" Name="process:jobid"/>
<Column Type="lstring" Name="process:username"/>
<Column Type="lstring" Name="process:program"/>
<Column Type="int_4s" Name="process:end_time"/>
<Column Type="lstring" Name="process:version"/>
<Column Type="lstring" Name="process:cvs_repository"/>
<Column Type="int_4s" Name="process:cvs_entry_time"/>
<Stream Delimiter="," Type="Local" Name="process:table">
,"soapbox.cgca.uwm.edu",,6486,1118462084,"process:process_id:1182",0,,0,"lluser","gstinjector",,,,,
</Stream>
</Table>
<Table Name="process_params:table">
<Column Type="ilwd:char" Name="process_params:process_id"/>
<Column Type="lstring" Name="process_params:program"/>
<Column Type="lstring" Name="process_params:type"/>
<Column Type="lstring" Name="process_params:value"/>
<Column Type="lstring" Name="process_params:param"/>
<Stream Delimiter="," Type="Local" Name="process_params:table">
"process:process_id:1182","gstinjector",,,"--verbose",
"process:process_id:1182","gstinjector","lstring","127.0.0.1","--server"
</Stream>
</Table>
<Table Name="sim_inspiral:table">
<Column Type="real_4" Name="sim_inspiral:theta0"/>
<Column Type="int_4s" Name="sim_inspiral:geocent_end_time_ns"/>
<Column Type="int_4s" Name="sim_inspiral:amp_order"/>
<Column Type="real_8" Name="sim_inspiral:end_time_gmst"/>
<Column Type="real_4" Name="sim_inspiral:coa_phase"/>
<Column Type="real_4" Name="sim_inspiral:mchirp"/>
<Column Type="int_4s" Name="sim_inspiral:numrel_mode_min"/>
<Column Type="int_4s" Name="sim_inspiral:numrel_mode_max"/>
<Column Type="lstring" Name="sim_inspiral:source"/>
<Column Type="real_4" Name="sim_inspiral:latitude"/>
<Column Type="lstring" Name="sim_inspiral:numrel_data"/>
<Column Type="int_4s" Name="sim_inspiral:geocent_end_time"/>
<Column Type="real_4" Name="sim_inspiral:spin2x"/>
<Column Type="real_4" Name="sim_inspiral:spin2y"/>
<Column Type="real_4" Name="sim_inspiral:spin2z"/>
<Column Type="ilwd:char" Name="sim_inspiral:process_id"/>
<Column Type="int_4s" Name="sim_inspiral:h_end_time"/>
<Column Type="real_4" Name="sim_inspiral:distance"/>
<Column Type="int_4s" Name="sim_inspiral:t_end_time"/>
<Column Type="lstring" Name="sim_inspiral:taper"/>
<Column Type="real_4" Name="sim_inspiral:longitude"/>
<Column Type="int_4s" Name="sim_inspiral:v_end_time_ns"/>
<Column Type="int_4s" Name="sim_inspiral:bandpass"/>
<Column Type="real_4" Name="sim_inspiral:eff_dist_l"/>
<Column Type="real_4" Name="sim_inspiral:eff_dist_h"/>
<Column Type="real_4" Name="sim_inspiral:eff_dist_g"/>
<Column Type="int_4s" Name="sim_inspiral:t_end_time_ns"/>
<Column Type="real_4" Name="sim_inspiral:spin1y"/>
<Column Type="real_4" Name="sim_inspiral:spin1x"/>
<Column Type="real_4" Name="sim_inspiral:spin1z"/>
<Column Type="int_4s" Name="sim_inspiral:h_end_time_ns"/>
<Column Type="real_4" Name="sim_inspiral:eff_dist_t"/>
<Column Type="int_4s" Name="sim_inspiral:l_end_time_ns"/>
<Column Type="real_4" Name="sim_inspiral:alpha2"/>
<Column Type="real_4" Name="sim_inspiral:alpha3"/>
<Column Type="real_4" Name="sim_inspiral:alpha1"/>
<Column Type="real_4" Name="sim_inspiral:alpha6"/>
<Column Type="real_4" Name="sim_inspiral:alpha4"/>
<Column Type="real_4" Name="sim_inspiral:alpha5"/>
<Column Type="int_4s" Name="sim_inspiral:l_end_time"/>
<Column Type="real_4" Name="sim_inspiral:polarization"/>
<Column Type="lstring" Name="sim_inspiral:waveform"/>
<Column Type="real_4" Name="sim_inspiral:phi0"/>
<Column Type="real_4" Name="sim_inspiral:inclination"/>
<Column Type="ilwd:char" Name="sim_inspiral:simulation_id"/>
<Column Type="real_4" Name="sim_inspiral:f_lower"/>
<Column Type="int_4s" Name="sim_inspiral:g_end_time_ns"/>
<Column Type="real_4" Name="sim_inspiral:eff_dist_v"/>
<Column Type="real_4" Name="sim_inspiral:beta"/>
<Column Type="int_4s" Name="sim_inspiral:g_end_time"/>
<Column Type="real_4" Name="sim_inspiral:alpha"/>
<Column Type="real_4" Name="sim_inspiral:f_final"/>
<Column Type="real_4" Name="sim_inspiral:mass1"/>
<Column Type="real_4" Name="sim_inspiral:mass2"/>