Commit b68da6ee authored by Alexander Pace's avatar Alexander Pace

merging in changes for 2.7.0

parent 92ebf04c
Pipeline #145857 passed with stages
in 15 minutes and 12 seconds
ligo-gracedb (2.7.0-1) unstable; urgency=low
* Backend rewrite using "requests" package
-- Alexander E. Pace <alexander.pace@ligo.org> Thu, 06 Aug 2020 11:03:15 -0700
ligo-gracedb (2.6.1-1) unstable; urgency=low
* SSL_PROTOCOL fix for python < 2.7.13
......
......@@ -9,7 +9,7 @@ X-Python3-Version: >=3.5
Package: python-ligo-gracedb
Architecture: all
Depends: ${misc:Depends}, ${python:Depends}, python-ligo-common, python-future, python-setuptools, python-cryptography
Depends: ${misc:Depends}, ${python:Depends}, python-ligo-common, python-future, python-setuptools, python-cryptography, python-requests
Provides: ${python:Provides}
Description: Gravitational-wave Candidate Event Database - Python
The gravitational-wave candidate event database (GraceDB) is a prototype
......@@ -21,7 +21,7 @@ Description: Gravitational-wave Candidate Event Database - Python
Package: python3-ligo-gracedb
Architecture: all
Depends: ${misc:Depends}, ${python3:Depends}, python3-ligo-common, python3-future, python3-setuptools, python3-cryptography
Depends: ${misc:Depends}, ${python3:Depends}, python3-ligo-common, python3-future, python3-setuptools, python3-cryptography, python3-requests
Provides: ${python3:Provides}
Description: Gravitational-wave Candidate Event Database - Python 3
The gravitational-wave candidate event database (GraceDB) is a prototype
......
%define name ligo-gracedb
%define version 2.6.1
%define unmangled_version 2.6.1
%define version 2.7.0
%define unmangled_version 2.7.0
%define release 1
Summary: Gravity Wave Candidate Event Database
......@@ -44,6 +44,7 @@ Requires: python-six
Requires: python2-ligo-common
Requires: python-future
Requires: python2-cryptography
Requires: python-requests
%{?python_provide:%python_provide python2-%{name}}
......
# -*- coding: utf-8 -*-
# Copyright (C) Alexander Pace, Tanner Prestegard,
# Branson Stephens, Brian Moe (2020)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>
# Sources:
# 1) https://stackoverflow.com/questions/45539422/can-we-reload-a-page-url-
# in-python-using-urllib-or-urllib2-or-requests-or-mechan
# 2) https://2.python-requests.org/en/master/user/advanced/#example-
# specific-ssl-version
# 3) https://urllib3.readthedocs.io/en/1.2.1/pools.html
from functools import partial
from requests.adapters import HTTPAdapter
from urllib3.connection import HTTPSConnection
from urllib3.connectionpool import HTTPSConnectionPool, HTTPConnectionPool
from .cert import check_certificate_expiration
class GraceDbCertAdapter(HTTPAdapter):
def __init__(self, cert=None, reload_buffer=0, **kwargs):
super(GraceDbCertAdapter, self).__init__(**kwargs)
https_pool_cls = partial(
GraceDbCertHTTPSConnectionPool,
cert=cert,
reload_buffer=reload_buffer)
self.poolmanager.pool_classes_by_scheme = {
'http': HTTPConnectionPool,
'https': https_pool_cls
}
class GraceDbCertHTTPSConnection(HTTPSConnection):
def __init__(self, host, cert=None, reload_buffer=0, **kwargs):
# At this point, te HTTPSConnection is initialized
# but unconnected. Set this property to 'True'
self.unestablished_connection = True
super(GraceDbCertHTTPSConnection, self).__init__(host, **kwargs)
@property
def unestablished_connection(self):
return self._unestablished_connection
@unestablished_connection.setter
def unestablished_connection(self, value):
self._unestablished_connection = value
def connect(self):
# Connected. After this step, the unestablished
# property is false.
self.unestablished_connection = False
super(GraceDbCertHTTPSConnection, self).connect()
class GraceDbCertHTTPSConnectionPool(HTTPSConnectionPool):
# ConnectionPool object gets used in the HTTPAdapter.
# "ConnectionCls" is a HTTP(S)COnnection object to use
# As the underlying connection.
# Source: https://urllib3.readthedocs.io/en/latest/
# reference/#module-urllib3.connectionpool
ConnectionCls = GraceDbCertHTTPSConnection
def __init__(self, host, port=None, cert=None,
reload_buffer=0, **kwargs):
super(GraceDbCertHTTPSConnectionPool, self).__init__(
host, port=port, **kwargs)
self._cert = cert
self._reload_buffer = reload_buffer
def _expired_cert(self):
return check_certificate_expiration(
self._cert,
self._reload_buffer)
def _get_conn(self, timeout=None):
while True:
# Start the connection object. At this step, the connection
# unestablished variable is true
conn = super(GraceDbCertHTTPSConnectionPool, self)._get_conn(
timeout)
# 'returning' the connection object then triggers the
# connection to be established. Establish a new connection
# if it's unestablished, or if the cert expiration is within the
# reload buffer. Establishing the new connection will (hopefully
# load the new cert.
if conn.unestablished_connection or not self._expired_cert():
return conn
# otherwise, kill the connection which will reset unestablished_..
# to true and then exit the loop.
conn.close()
# -*- coding: utf-8 -*-
# Copyright (C) Alexander Pace, Tanner Prestegard,
# Branson Stephens, Brian Moe (2020)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>
# This file contains x509 certificate loading and checking tools.
# Mostly duplicated effort from Tanner's work in support of certificate
# reloading, but it could be useful for giving the client the option
# of checking their certificates, or displaying certificate validity and
# expiration dates when they use the 'gracedb credentials client' command.
from cryptography import x509
from cryptography.hazmat.backends import default_backend
import datetime
far_future = datetime.timedelta(days=365)
# Takes in the path of a certificate and retrrns an x509.cryptography
# certificate object. This removes the prior check that the auth_type
# be 'x509' since this should only be called when reload_certificate
# is true, which checks for x509, or in cases in general when users want
# to check a cert without connecting a client.
def load_certificate(cert):
""" Loads in a path to a x509 certificate and returns a
x509.cryptography object """
with open(cert, 'rb') as cert_obj:
cert_obj = cert_obj.read()
# Try loading with PEM, then try loading with DER, then give up.
try:
return x509.load_pem_x509_certificate(
cert_obj, default_backend()
)
except ValueError:
try:
return x509.load_pem_x509_certificate(
cert_obj, default_backend()
)
except ValueError:
raise RuntimeError('Error importing certificate')
# Checks certificate expiration and returns a boolean. Optionally checks
# that the time left until expiration within the 'reload_buffer' parameter.
def check_certificate_expiration(cert, reload_buffer=0):
""" Checks to see if a cert is expiring within an optional
reload_buffer parameter. Default, reload_buffer=0, meaning
is the certificate currently expired.
cert is either a string path to an x509 cert, or a x509
cryptography certificate object """
if not hasattr(cert, 'subject'):
cert = load_certificate(cert)
expired = (cert.not_valid_after - datetime.datetime.utcnow()) <= \
datetime.timedelta(seconds=reload_buffer)
return expired
......@@ -4,6 +4,7 @@ import os
import six
import sys
import textwrap
from requests import Response
from ligo.gracedb.rest import GraceDb, DEFAULT_SERVICE_URL
from ligo.gracedb import __version__
......@@ -21,15 +22,15 @@ class CommandLineClient(GraceDb):
# Hamstring 'adjustResponse' from the example REST client.
# We don't want it messing with the response from the server.
def adjustResponse(self, response):
response.json = lambda: self.load_json_or_die(response)
response.json = lambda: self.load_json_from_response(response)
return response
# TP 2019: not sure if we still need to override this from the GraceDb
# class, but leaving it for now.
@classmethod
def output_and_die(cls, msg):
sys.stderr.write(msg)
sys.exit(1)
# @classmethod
# def output_and_die(cls, msg):
# sys.stderr.write(msg)
# sys.exit(1)
class CommandLineInterface(CommandBase):
......@@ -169,8 +170,6 @@ class CommandLineInterface(CommandBase):
# Define kwargs for initializing client
client_kwargs = {
'service_url': args.service,
'proxy_host': proxy,
'proxy_port': proxyport,
'force_noauth': args.force_noauth,
'username': args.username,
'password': args.password,
......@@ -193,21 +192,24 @@ def main(args=None):
try:
response = cli(args)
except HTTPError as e:
print('Error: {code} {reason}'.format(code=e.status, reason=e.reason))
print('Error: {code} {reason}. {text}.'.format(
code=e.status_code,
reason=e.reason,
text=e.text))
sys.exit(1)
except Exception as e:
print('Error: {msg}'.format(msg=str(e)))
sys.exit(1)
if isinstance(response, six.moves.http_client.HTTPResponse):
if isinstance(response, Response):
if (cli.args.output_type == 'json'):
# Handle errors
if response.status >= 400:
output = '{code} {reason}'.format(code=response.status,
if response.status_code >= 400:
output = '{code} {reason}'.format(code=response.status_code,
reason=response.reason)
# Only add message if it's not really long (i.e., it's not
# an HTML error page)
msg = response.read()
msg = response.text
if isinstance(msg, bytes):
msg = msg.decode()
if (len(msg) < 1000):
......@@ -215,16 +217,21 @@ def main(args=None):
print(output)
sys.exit(1)
# Handle errors raised in load_json_or_die()
try:
output = response.json()
except Exception as e:
print(str(e))
if response.status_code == 204:
output = {}
print(output)
sys.exit(1)
else:
try:
output = response.json()
except Exception as e:
print(str(e))
sys.exit(1)
print(json.dumps(output, indent=4))
elif (cli.args.output_type == 'status'):
print('Server returned {status}: {reason}'.format(
status=response.status, reason=response.reason))
status=response.status_code, reason=response.reason))
elif isinstance(response, dict):
print(json.dumps(response, indent=4))
elif isinstance(response, six.string_types):
......
......@@ -94,7 +94,7 @@ class GetFileCommand(GetChildBase):
response = client.files(args.object_id, args.filename)
# Handle response
if response.status == 200:
if response.status_code == 200:
if args.destination == '-':
# For stdout, return string contents of file
file_contents = response.read()
......
......@@ -94,8 +94,8 @@ class PingCommand(RegisteredSubCommandBase):
def run(self, client, args):
response = client.ping()
output = 'Response from {server}: {status}'.format(
server=client._service_url, status=response.status)
if (response.status == 200):
server=client._service_url, status=response.status_code)
if (response.status_code == 200):
output += ' OK'
return output
......
......@@ -144,7 +144,7 @@ def test_ping_subcommand(CLI):
"""Test ping subcommand"""
func = 'ligo.gracedb.rest.GraceDb.ping'
with mock.patch(func) as mock_cli_func:
mock_cli_func.return_value = mock.MagicMock(status=123)
mock_cli_func.return_value = mock.MagicMock(status_code=123)
output = CLI(['ping'])
# Check call count
......@@ -159,7 +159,7 @@ def test_ping_subcommand(CLI):
# Check output
assert output == 'Response from {server}: {status}'.format(
server=CLI.client._service_url, status=mock_cli_func().status)
server=CLI.client._service_url, status=mock_cli_func().status_code)
@pytest.mark.parametrize(
......
......@@ -88,8 +88,6 @@ def test_cli_client_setup(CLI):
cli_args, cli_kwargs = mock_client.call_args
assert cli_args == ()
assert cli_kwargs['service_url'] == arg_dict['service-url']
assert cli_kwargs['proxy_host'] == arg_dict['proxy'].split(':')[0]
assert cli_kwargs['proxy_port'] == int(arg_dict['proxy'].split(':')[1])
assert cli_kwargs['username'] == arg_dict['username']
assert cli_kwargs['password'] == arg_dict['password']
assert cli_kwargs['cred'] == arg_dict['creds'].split(',')
import json
import pytest
import six
# import six
try:
from unittest import mock
except ImportError:
import mock
from ligo.gracedb.exceptions import HTTPError
from requests import Response
# Apply module-level mark
pytestmark = pytest.mark.cli
......@@ -23,8 +24,8 @@ def test_main_with_ok_response(main_tester, capsys, output_type):
status = 200
reason = 'because'
response_mock = mock.MagicMock(
spec=six.moves.http_client.HTTPResponse,
status=status, reason=reason)
spec=Response,
status_code=status, reason=reason)
mock_CLI().return_value = response_mock
# Set up response.json()
json_mock = mock.MagicMock()
......@@ -62,14 +63,10 @@ def test_main_with_bad_response(main_tester, capsys, response_msg):
status = 400
reason = 'Bad Request'
response_mock = mock.MagicMock(
spec=six.moves.http_client.HTTPResponse,
status=status, reason=reason)
spec=Response,
status_code=status, reason=reason,
text=response_msg)
mock_CLI().return_value = response_mock
# Set up response.read()
read_mock = mock.MagicMock()
read_mock.return_value = response_msg
# Use setattr since the MagicMock is autospecced
setattr(response_mock, 'read', read_mock)
# Set up CLI.args.output_type
mock_CLI().args.output_type = 'json'
......@@ -93,7 +90,9 @@ def test_main_with_bad_response(main_tester, capsys, response_msg):
code=status, reason=reason)
EXCEPTION_DATA = [Exception('test'), HTTPError(400, 'Bad Request', 'test')]
EXCEPTION_DATA = [Exception('test'),
HTTPError(400, 'Bad Request', 'test')]
@pytest.mark.parametrize("exc", EXCEPTION_DATA) # noqa: E302
def test_main_with_exception_in_CLI_call(main_tester, capsys, exc):
"""Test CLI wrapper when CLI call raises exception"""
......@@ -115,8 +114,8 @@ def test_main_with_exception_in_CLI_call(main_tester, capsys, exc):
# Test output
if isinstance(exc, HTTPError):
assert stdout.rstrip() == 'Error: {code} {reason}'.format(
code=exc.status, reason=exc.reason)
assert stdout.rstrip() == 'Error: {code} {reason}. {text}.'.format(
code=exc.status_code, reason=exc.reason, text=exc.text)
elif isinstance(exc, Exception):
assert stdout.rstrip() == 'Error: {msg}'.format(msg=str(exc))
......@@ -130,8 +129,8 @@ def test_main_with_exception_in_response_json(main_tester, capsys):
status = 200
reason = 'OK'
response_mock = mock.MagicMock(
spec=six.moves.http_client.HTTPResponse,
status=status, reason=reason)
spec=Response,
status_code=status, reason=reason)
mock_CLI().return_value = response_mock
# Set up response.read()
exc_msg = 'uh oh'
......
# -*- coding: utf-8 -*-
# Copyright (C) Alexander Pace, Tanner Prestegard,
# Branson Stephens, Brian Moe (2020)
#
# This file is part of gracedb
#
# gracedb is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# It is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import json as json_lib
from warnings import warn
from requests import Session
from .extern.safe_netrc import netrc
from os import getuid
from .version import __version__
from .adapter import GraceDbCertAdapter
from .utils import hook_response
# To remove later: python2 compatibility fix:
if sys.version_info[0] > 2:
from urllib.parse import urlparse
else:
from urlparse import urlparse
DEFAULT_SERVICE_URL = "https://gracedb.ligo.org/api/"
class GraceDBClient(Session):
"""
url (:obj:`str`, optional): URL of server API
cred (:obj:`tuple` or :obj:`str, optional): a tuple or list of
(``/path/to/cert/file``, ``/path/to/key/file) or a single path to
a combined proxy file (if using an X.509 certificate for
authentication)
username (:obj:`str`, optional): username for basic auth
password (:obj:`str`, optional): password for basic auth
force_noauth (:obj:`bool`, optional): set to True if you want to skip
credential lookup and use this client as an unauthenticated user
fail_if_noauth (:obj:`bool`, optional): set to True if you want the
constructor to fail if no authentication credentials are provided
or found
reload_certificate (:obj:`bool`, optional): if ``True``, your
certificate will be checked before each request whether it is
within ``reload_buffer`` seconds of expiration, and if so, it will
be reloaded. Useful for processes which may live longer than the
certificate lifetime and have an automated method for certificate
renewal. The path to the new/renewed certificate **must** be the
same as for the old certificate.
reload_buffer (:obj:`int`, optional): buffer (in seconds) for reloading
a certificate in advance of its expiration. Only used if
``reload_certificate`` is ``True``.
Authentication details:
You can:
1. Provide a path to an X.509 certificate and key or a single
combined proxy file
2. Provide a username and password
Or:
The code will look for a certificate in a default location
(``/tmp/x509up_u%d``, where ``%d`` is your user ID)
The code will look for a username and password for the specified
server in ``$HOME/.netrc``
"""
def __init__(self, url=DEFAULT_SERVICE_URL, cred=None, username=None,
password=None, force_noauth=False, fail_if_noauth=False,
reload_certificate=False, reload_buffer=300,
*args, **kwargs):
super(GraceDBClient, self).__init__(*args, **kwargs)
# Initialize variables:
self.cert = None
self.auth = None
self.host = urlparse(url).hostname
self.auth_type = None
# Set up credentials. First, only attempt if the user did not
# force noauth:
if not force_noauth:
# Attempt to assign x509 credentials.
if cred or not (username or password):
self.cert = self._get_x509_credentials(cred)
# if we weren't able to determine a x509 certificate,
# then look for basic auth.
if not self.cert:
self.auth = self._process_basic_credentials(username,
password)
# If no basic auth credentials were provided, and
# fail_if_noauthis set, then fail.
if not self.auth:
if fail_if_noauth:
raise RuntimeError("No authentication credentials "
"could be found, and fail_if_"
"noauth is set.")
else:
warn("Authentication credentials not found, "
"proceeding with unauthorized session")
elif fail_if_noauth:
raise ValueError('You have provided conflicting parameters '
'to the client constructor: '
'fail_if_noauth=True and force_noauth=True.')
# Update session headers:
self.headers.update(self._update_headers())
# Adjust the response via a session hook:
self.hooks = {'response': hook_response}
if reload_certificate and self.auth_type == 'x509':
self.mount('https://', GraceDbCertAdapter(
cert=self.cert,
reload_buffer=reload_buffer))
def _process_basic_credentials(self, username, password):
""" Gathers basic auth credentials. First it looks for
the input username/password, then checks the netrc file
"""
# Checks username/password:
if username and password:
self.auth_type = 'basic'
return username, password
# If one or the other is provided, but not both, raise an error.
elif bool(username) != bool(password):
raise RuntimeError('You must provide both a username and a '
'password for basic authentication.')
# Finally, try the netrc fle:
else:
try:
netrc_auth = netrc().authenticators(self.host)
except IOError:
pass