Commit b68da6ee authored by Alexander Pace's avatar Alexander Pace

merging in changes for 2.7.0

parent 92ebf04c
Pipeline #145857 passed with stages
in 15 minutes and 12 seconds
ligo-gracedb (2.7.0-1) unstable; urgency=low
* Backend rewrite using "requests" package
-- Alexander E. Pace <alexander.pace@ligo.org> Thu, 06 Aug 2020 11:03:15 -0700
ligo-gracedb (2.6.1-1) unstable; urgency=low
* SSL_PROTOCOL fix for python < 2.7.13
......
......@@ -9,7 +9,7 @@ X-Python3-Version: >=3.5
Package: python-ligo-gracedb
Architecture: all
Depends: ${misc:Depends}, ${python:Depends}, python-ligo-common, python-future, python-setuptools, python-cryptography
Depends: ${misc:Depends}, ${python:Depends}, python-ligo-common, python-future, python-setuptools, python-cryptography, python-requests
Provides: ${python:Provides}
Description: Gravitational-wave Candidate Event Database - Python
The gravitational-wave candidate event database (GraceDB) is a prototype
......@@ -21,7 +21,7 @@ Description: Gravitational-wave Candidate Event Database - Python
Package: python3-ligo-gracedb
Architecture: all
Depends: ${misc:Depends}, ${python3:Depends}, python3-ligo-common, python3-future, python3-setuptools, python3-cryptography
Depends: ${misc:Depends}, ${python3:Depends}, python3-ligo-common, python3-future, python3-setuptools, python3-cryptography, python3-requests
Provides: ${python3:Provides}
Description: Gravitational-wave Candidate Event Database - Python 3
The gravitational-wave candidate event database (GraceDB) is a prototype
......
%define name ligo-gracedb
%define version 2.6.1
%define unmangled_version 2.6.1
%define version 2.7.0
%define unmangled_version 2.7.0
%define release 1
Summary: Gravity Wave Candidate Event Database
......@@ -44,6 +44,7 @@ Requires: python-six
Requires: python2-ligo-common
Requires: python-future
Requires: python2-cryptography
Requires: python-requests
%{?python_provide:%python_provide python2-%{name}}
......
# -*- coding: utf-8 -*-
# Copyright (C) Alexander Pace, Tanner Prestegard,
# Branson Stephens, Brian Moe (2020)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>
# Sources:
# 1) https://stackoverflow.com/questions/45539422/can-we-reload-a-page-url-
# in-python-using-urllib-or-urllib2-or-requests-or-mechan
# 2) https://2.python-requests.org/en/master/user/advanced/#example-
# specific-ssl-version
# 3) https://urllib3.readthedocs.io/en/1.2.1/pools.html
from functools import partial
from requests.adapters import HTTPAdapter
from urllib3.connection import HTTPSConnection
from urllib3.connectionpool import HTTPSConnectionPool, HTTPConnectionPool
from .cert import check_certificate_expiration
class GraceDbCertAdapter(HTTPAdapter):
def __init__(self, cert=None, reload_buffer=0, **kwargs):
super(GraceDbCertAdapter, self).__init__(**kwargs)
https_pool_cls = partial(
GraceDbCertHTTPSConnectionPool,
cert=cert,
reload_buffer=reload_buffer)
self.poolmanager.pool_classes_by_scheme = {
'http': HTTPConnectionPool,
'https': https_pool_cls
}
class GraceDbCertHTTPSConnection(HTTPSConnection):
def __init__(self, host, cert=None, reload_buffer=0, **kwargs):
# At this point, te HTTPSConnection is initialized
# but unconnected. Set this property to 'True'
self.unestablished_connection = True
super(GraceDbCertHTTPSConnection, self).__init__(host, **kwargs)
@property
def unestablished_connection(self):
return self._unestablished_connection
@unestablished_connection.setter
def unestablished_connection(self, value):
self._unestablished_connection = value
def connect(self):
# Connected. After this step, the unestablished
# property is false.
self.unestablished_connection = False
super(GraceDbCertHTTPSConnection, self).connect()
class GraceDbCertHTTPSConnectionPool(HTTPSConnectionPool):
# ConnectionPool object gets used in the HTTPAdapter.
# "ConnectionCls" is a HTTP(S)COnnection object to use
# As the underlying connection.
# Source: https://urllib3.readthedocs.io/en/latest/
# reference/#module-urllib3.connectionpool
ConnectionCls = GraceDbCertHTTPSConnection
def __init__(self, host, port=None, cert=None,
reload_buffer=0, **kwargs):
super(GraceDbCertHTTPSConnectionPool, self).__init__(
host, port=port, **kwargs)
self._cert = cert
self._reload_buffer = reload_buffer
def _expired_cert(self):
return check_certificate_expiration(
self._cert,
self._reload_buffer)
def _get_conn(self, timeout=None):
while True:
# Start the connection object. At this step, the connection
# unestablished variable is true
conn = super(GraceDbCertHTTPSConnectionPool, self)._get_conn(
timeout)
# 'returning' the connection object then triggers the
# connection to be established. Establish a new connection
# if it's unestablished, or if the cert expiration is within the
# reload buffer. Establishing the new connection will (hopefully
# load the new cert.
if conn.unestablished_connection or not self._expired_cert():
return conn
# otherwise, kill the connection which will reset unestablished_..
# to true and then exit the loop.
conn.close()
# -*- coding: utf-8 -*-
# Copyright (C) Alexander Pace, Tanner Prestegard,
# Branson Stephens, Brian Moe (2020)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>
# This file contains x509 certificate loading and checking tools.
# Mostly duplicated effort from Tanner's work in support of certificate
# reloading, but it could be useful for giving the client the option
# of checking their certificates, or displaying certificate validity and
# expiration dates when they use the 'gracedb credentials client' command.
from cryptography import x509
from cryptography.hazmat.backends import default_backend
import datetime
far_future = datetime.timedelta(days=365)
# Takes in the path of a certificate and retrrns an x509.cryptography
# certificate object. This removes the prior check that the auth_type
# be 'x509' since this should only be called when reload_certificate
# is true, which checks for x509, or in cases in general when users want
# to check a cert without connecting a client.
def load_certificate(cert):
""" Loads in a path to a x509 certificate and returns a
x509.cryptography object """
with open(cert, 'rb') as cert_obj:
cert_obj = cert_obj.read()
# Try loading with PEM, then try loading with DER, then give up.
try:
return x509.load_pem_x509_certificate(
cert_obj, default_backend()
)
except ValueError:
try:
return x509.load_pem_x509_certificate(
cert_obj, default_backend()
)
except ValueError:
raise RuntimeError('Error importing certificate')
# Checks certificate expiration and returns a boolean. Optionally checks
# that the time left until expiration within the 'reload_buffer' parameter.
def check_certificate_expiration(cert, reload_buffer=0):
""" Checks to see if a cert is expiring within an optional
reload_buffer parameter. Default, reload_buffer=0, meaning
is the certificate currently expired.
cert is either a string path to an x509 cert, or a x509
cryptography certificate object """
if not hasattr(cert, 'subject'):
cert = load_certificate(cert)
expired = (cert.not_valid_after - datetime.datetime.utcnow()) <= \
datetime.timedelta(seconds=reload_buffer)
return expired
......@@ -4,6 +4,7 @@ import os
import six
import sys
import textwrap
from requests import Response
from ligo.gracedb.rest import GraceDb, DEFAULT_SERVICE_URL
from ligo.gracedb import __version__
......@@ -21,15 +22,15 @@ class CommandLineClient(GraceDb):
# Hamstring 'adjustResponse' from the example REST client.
# We don't want it messing with the response from the server.
def adjustResponse(self, response):
response.json = lambda: self.load_json_or_die(response)
response.json = lambda: self.load_json_from_response(response)
return response
# TP 2019: not sure if we still need to override this from the GraceDb
# class, but leaving it for now.
@classmethod
def output_and_die(cls, msg):
sys.stderr.write(msg)
sys.exit(1)
# @classmethod
# def output_and_die(cls, msg):
# sys.stderr.write(msg)
# sys.exit(1)
class CommandLineInterface(CommandBase):
......@@ -169,8 +170,6 @@ class CommandLineInterface(CommandBase):
# Define kwargs for initializing client
client_kwargs = {
'service_url': args.service,
'proxy_host': proxy,
'proxy_port': proxyport,
'force_noauth': args.force_noauth,
'username': args.username,
'password': args.password,
......@@ -193,21 +192,24 @@ def main(args=None):
try:
response = cli(args)
except HTTPError as e:
print('Error: {code} {reason}'.format(code=e.status, reason=e.reason))
print('Error: {code} {reason}. {text}.'.format(
code=e.status_code,
reason=e.reason,
text=e.text))
sys.exit(1)
except Exception as e:
print('Error: {msg}'.format(msg=str(e)))
sys.exit(1)
if isinstance(response, six.moves.http_client.HTTPResponse):
if isinstance(response, Response):
if (cli.args.output_type == 'json'):
# Handle errors
if response.status >= 400:
output = '{code} {reason}'.format(code=response.status,
if response.status_code >= 400:
output = '{code} {reason}'.format(code=response.status_code,
reason=response.reason)
# Only add message if it's not really long (i.e., it's not
# an HTML error page)
msg = response.read()
msg = response.text
if isinstance(msg, bytes):
msg = msg.decode()
if (len(msg) < 1000):
......@@ -215,16 +217,21 @@ def main(args=None):
print(output)
sys.exit(1)
# Handle errors raised in load_json_or_die()
try:
output = response.json()
except Exception as e:
print(str(e))
if response.status_code == 204:
output = {}
print(output)
sys.exit(1)
else:
try:
output = response.json()
except Exception as e:
print(str(e))
sys.exit(1)
print(json.dumps(output, indent=4))
elif (cli.args.output_type == 'status'):
print('Server returned {status}: {reason}'.format(
status=response.status, reason=response.reason))
status=response.status_code, reason=response.reason))
elif isinstance(response, dict):
print(json.dumps(response, indent=4))
elif isinstance(response, six.string_types):
......
......@@ -94,7 +94,7 @@ class GetFileCommand(GetChildBase):
response = client.files(args.object_id, args.filename)
# Handle response
if response.status == 200:
if response.status_code == 200:
if args.destination == '-':
# For stdout, return string contents of file
file_contents = response.read()
......
......@@ -94,8 +94,8 @@ class PingCommand(RegisteredSubCommandBase):
def run(self, client, args):
response = client.ping()
output = 'Response from {server}: {status}'.format(
server=client._service_url, status=response.status)
if (response.status == 200):
server=client._service_url, status=response.status_code)
if (response.status_code == 200):
output += ' OK'
return output
......
......@@ -144,7 +144,7 @@ def test_ping_subcommand(CLI):
"""Test ping subcommand"""
func = 'ligo.gracedb.rest.GraceDb.ping'
with mock.patch(func) as mock_cli_func:
mock_cli_func.return_value = mock.MagicMock(status=123)
mock_cli_func.return_value = mock.MagicMock(status_code=123)
output = CLI(['ping'])
# Check call count
......@@ -159,7 +159,7 @@ def test_ping_subcommand(CLI):
# Check output
assert output == 'Response from {server}: {status}'.format(
server=CLI.client._service_url, status=mock_cli_func().status)
server=CLI.client._service_url, status=mock_cli_func().status_code)
@pytest.mark.parametrize(
......
......@@ -88,8 +88,6 @@ def test_cli_client_setup(CLI):
cli_args, cli_kwargs = mock_client.call_args
assert cli_args == ()
assert cli_kwargs['service_url'] == arg_dict['service-url']
assert cli_kwargs['proxy_host'] == arg_dict['proxy'].split(':')[0]
assert cli_kwargs['proxy_port'] == int(arg_dict['proxy'].split(':')[1])
assert cli_kwargs['username'] == arg_dict['username']
assert cli_kwargs['password'] == arg_dict['password']
assert cli_kwargs['cred'] == arg_dict['creds'].split(',')
import json
import pytest
import six
# import six
try:
from unittest import mock
except ImportError:
import mock
from ligo.gracedb.exceptions import HTTPError
from requests import Response
# Apply module-level mark
pytestmark = pytest.mark.cli
......@@ -23,8 +24,8 @@ def test_main_with_ok_response(main_tester, capsys, output_type):
status = 200
reason = 'because'
response_mock = mock.MagicMock(
spec=six.moves.http_client.HTTPResponse,
status=status, reason=reason)
spec=Response,
status_code=status, reason=reason)
mock_CLI().return_value = response_mock
# Set up response.json()
json_mock = mock.MagicMock()
......@@ -62,14 +63,10 @@ def test_main_with_bad_response(main_tester, capsys, response_msg):
status = 400
reason = 'Bad Request'
response_mock = mock.MagicMock(
spec=six.moves.http_client.HTTPResponse,
status=status, reason=reason)
spec=Response,
status_code=status, reason=reason,
text=response_msg)
mock_CLI().return_value = response_mock
# Set up response.read()
read_mock = mock.MagicMock()
read_mock.return_value = response_msg
# Use setattr since the MagicMock is autospecced
setattr(response_mock, 'read', read_mock)
# Set up CLI.args.output_type
mock_CLI().args.output_type = 'json'
......@@ -93,7 +90,9 @@ def test_main_with_bad_response(main_tester, capsys, response_msg):
code=status, reason=reason)
EXCEPTION_DATA = [Exception('test'), HTTPError(400, 'Bad Request', 'test')]
EXCEPTION_DATA = [Exception('test'),
HTTPError(400, 'Bad Request', 'test')]
@pytest.mark.parametrize("exc", EXCEPTION_DATA) # noqa: E302
def test_main_with_exception_in_CLI_call(main_tester, capsys, exc):
"""Test CLI wrapper when CLI call raises exception"""
......@@ -115,8 +114,8 @@ def test_main_with_exception_in_CLI_call(main_tester, capsys, exc):
# Test output
if isinstance(exc, HTTPError):
assert stdout.rstrip() == 'Error: {code} {reason}'.format(
code=exc.status, reason=exc.reason)
assert stdout.rstrip() == 'Error: {code} {reason}. {text}.'.format(
code=exc.status_code, reason=exc.reason, text=exc.text)
elif isinstance(exc, Exception):
assert stdout.rstrip() == 'Error: {msg}'.format(msg=str(exc))
......@@ -130,8 +129,8 @@ def test_main_with_exception_in_response_json(main_tester, capsys):
status = 200
reason = 'OK'
response_mock = mock.MagicMock(
spec=six.moves.http_client.HTTPResponse,
status=status, reason=reason)
spec=Response,
status_code=status, reason=reason)
mock_CLI().return_value = response_mock
# Set up response.read()
exc_msg = 'uh oh'
......
This diff is collapsed.
# Custom exceptions
class HTTPError(Exception):
def __init__(self, status, reason, message):
self.status = status
def __init__(self,
status_code,
reason,
text):
self.status = status_code
self.status_code = status_code
self.reason = reason
self.message = message
Exception.__init__(self, status, reason, message)
self.message = text
self.text = text
Exception.__init__(self,
status_code,
reason,
text)
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -6,7 +6,7 @@ pytestmark = pytest.mark.integration
def test_ping(client):
response = client.ping()
assert response.status == 200
assert response.status_code == 200
@pytest.mark.parametrize(
......
This diff is collapsed.
......@@ -30,7 +30,7 @@ def test_emobservations(client, create_obj, obj_type):
obj_id, emgroup, ra_list, ra_width_list, dec_list, dec_width_list,
start_time_list, duration_list, comment=comment
)
assert response.status == 201
assert response.status_code == 201
data = response.json()
assert data['comment'] == comment
assert data['group'] == emgroup
......@@ -46,14 +46,14 @@ def test_emobservations(client, create_obj, obj_type):
# Get list of emobservations and check results
response = client.emobservations(obj_id)
assert response.status == 200
assert response.status_code == 200
data = response.json()
assert len(data['observations']) == 1
assert data['observations'][0]['N'] == emo_N
# Retrieve the individual emobservation directly
response = client.emobservations(obj_id, emo_N)
assert response.status == 200
assert response.status_code == 200
data = response.json()
assert data['comment'] == comment
assert data['group'] == emgroup
......
......@@ -11,7 +11,7 @@ def test_create_gstlal(client, test_data_dir):
# Create gstlal event
filename = os.path.join(test_data_dir, 'cbc-lm.xml')
response = client.createEvent('Test', 'gstlal', filename)
assert response.status == 201
assert response.status_code == 201
data = response.json()
# Test results
......@@ -21,6 +21,47 @@ def test_create_gstlal(client, test_data_dir):
assert data['extra_attributes']['CoincInspiral']['snr'] == 9.31793628458239
def test_create_gstlal_with_missing_table_entry(client, test_data_dir):
# Create gstlal event, with a missing snglinspiral table
# entry. The event should get made and the table populated with
# a None entry.
filename = os.path.join(test_data_dir, 'cbc-lm-missing-entry.xml')
response = client.createEvent('Test', 'gstlal', filename)
assert response.status_code == 201
data = response.json()
# Make sure warnings are empty:
assert data['warnings'] == []
# Test results
assert data['group'] == 'Test'
assert data['pipeline'] == 'gstlal'
assert data['gpstime'] == 971609248.151741
assert data['extra_attributes']['CoincInspiral']['snr'] == 9.31793628458239
# Extra results. Assert that the snr snglinspiral table entry is empty
assert data['extra_attributes']['SingleInspiral'][0].get('snr') is None
def test_create_gstlal_no_ilwdchar(client, test_data_dir):
# Create gstlal event, where the ligol ilwd:char entries
# have been replaced with int8's.
filename = os.path.join(test_data_dir, 'cbc-lm-no-ilwdchar.xml')
response = client.createEvent('Test', 'gstlal', filename)
assert response.status_code == 201
data = response.json()
# Make sure warnings are empty:
assert data['warnings'] == []
# Test results
assert data['group'] == 'Test'
assert data['pipeline'] == 'gstlal'
assert data['gpstime'] == 971609248.151741
# Also, this assert will fail if the event is not read correctly:
assert data['extra_attributes']['CoincInspiral']['snr'] == 9.31793628458239
@pytest.mark.xfail
def test_create_pycbc(client, test_data_dir):
# Create PyCBC event - need to get a PyCBC data file and clean it up,
......@@ -32,7 +73,7 @@ def test_create_mbta(client, test_data_dir):
# Create MBTA event
filename = os.path.join(test_data_dir, 'cbc-mbta.xml')
response = client.createEvent('Test', 'MBTAOnline', filename)
assert response.status == 201
assert response.status_code == 201
data = response.json()
# Test results
......@@ -47,11 +88,29 @@ def test_create_spiir(client, test_data_dir):
# Create SPIIR event
filename = os.path.join(test_data_dir, 'spiir-test.xml')
response = client.createEvent('Test', 'spiir', filename)
assert response.status == 201
assert response.status_code == 201
data = response.json()
# Test results
assert response.status == 201
assert response.status_code == 201
assert data['group'] == 'Test'
assert data['pipeline'] == 'spiir'
assert data['extra_attributes']['CoincInspiral']['mass'] == 3.98
assert data['far'] == 3.27e-07
def test_create_spiir_no_ilwdchar(client, test_data_dir):
# Create SPIIR event, with ilwd:char entires converted to int8s
filename = os.path.join(test_data_dir, 'spiir-test-no-ilwdchar.xml')
response = client.createEvent('Test', 'spiir', filename)
assert response.status_code == 201
data = response.json()
# Make sure warnings are empty:
assert data['warnings'] == []
# Test results
assert response.status_code == 201
assert data['group'] == 'Test'
assert data['pipeline'] == 'spiir'
assert data['extra_attributes']['CoincInspiral']['mass'] == 3.98
......@@ -62,7 +121,7 @@ def test_create_cwb(client, test_data_dir):
# Create CWB event
filename = os.path.join(test_data_dir, 'burst-cwb.txt')
response<