merging in changes for 2.7.0

debian/changelog

+ligo-gracedb (2.7.0-1) unstable; urgency=low
+  * Backend rewrite using "requests" package
+
+ -- Alexander E. Pace <alexander.pace@ligo.org>  Thu, 06 Aug 2020 11:03:15 -0700
+
 ligo-gracedb (2.6.1-1) unstable; urgency=low
   * SSL_PROTOCOL fix for python < 2.7.13
 

debian/control

@@ -9,7 +9,7 @@ X-Python3-Version: >=3.5
 
 Package: python-ligo-gracedb
 Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}, python-ligo-common, python-future, python-setuptools, python-cryptography
+Depends: ${misc:Depends}, ${python:Depends}, python-ligo-common, python-future, python-setuptools, python-cryptography, python-requests
 Provides: ${python:Provides}
 Description: Gravitational-wave Candidate Event Database - Python
  The gravitational-wave candidate event database (GraceDB) is a prototype
@@ -21,7 +21,7 @@ Description: Gravitational-wave Candidate Event Database - Python
 
 Package: python3-ligo-gracedb
 Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}, python3-ligo-common, python3-future, python3-setuptools, python3-cryptography
+Depends: ${misc:Depends}, ${python3:Depends}, python3-ligo-common, python3-future, python3-setuptools, python3-cryptography, python3-requests
 Provides: ${python3:Provides}
 Description: Gravitational-wave Candidate Event Database - Python 3
  The gravitational-wave candidate event database (GraceDB) is a prototype

ligo-gracedb.spec

 %define name              ligo-gracedb
-%define version           2.6.1
-%define unmangled_version 2.6.1
+%define version           2.7.0
+%define unmangled_version 2.7.0
 %define release           1
 
 Summary:   Gravity Wave Candidate Event Database
@@ -44,6 +44,7 @@ Requires: python-six
 Requires: python2-ligo-common
 Requires: python-future
 Requires: python2-cryptography
+Requires: python-requests
 
 %{?python_provide:%python_provide python2-%{name}}
 

ligo/gracedb/adapter.py

+# -*- coding: utf-8 -*-
+# Copyright (C) Alexander Pace, Tanner Prestegard,
+#               Branson Stephens, Brian Moe (2020)
+#
+# This file is part of gracedb
+#
+# gracedb is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# It is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with gracedb.  If not, see <http://www.gnu.org/licenses/>
+
+# Sources:
+#  1) https://stackoverflow.com/questions/45539422/can-we-reload-a-page-url-
+#     in-python-using-urllib-or-urllib2-or-requests-or-mechan
+
+#  2) https://2.python-requests.org/en/master/user/advanced/#example-
+#     specific-ssl-version
+
+#  3) https://urllib3.readthedocs.io/en/1.2.1/pools.html
+
+
+from functools import partial
+from requests.adapters import HTTPAdapter
+from urllib3.connection import HTTPSConnection
+from urllib3.connectionpool import HTTPSConnectionPool, HTTPConnectionPool
+from .cert import check_certificate_expiration
+
+
+class GraceDbCertAdapter(HTTPAdapter):
+    def __init__(self, cert=None, reload_buffer=0, **kwargs):
+        super(GraceDbCertAdapter, self).__init__(**kwargs)
+        https_pool_cls = partial(
+            GraceDbCertHTTPSConnectionPool,
+            cert=cert,
+            reload_buffer=reload_buffer)
+
+        self.poolmanager.pool_classes_by_scheme = {
+            'http': HTTPConnectionPool,
+            'https': https_pool_cls
+        }
+
+
+class GraceDbCertHTTPSConnection(HTTPSConnection):
+    def __init__(self, host, cert=None, reload_buffer=0, **kwargs):
+        # At this point, te HTTPSConnection is initialized
+        # but unconnected. Set this property to 'True'
+        self.unestablished_connection = True
+        super(GraceDbCertHTTPSConnection, self).__init__(host, **kwargs)
+
+    @property
+    def unestablished_connection(self):
+        return self._unestablished_connection
+
+    @unestablished_connection.setter
+    def unestablished_connection(self, value):
+        self._unestablished_connection = value
+
+    def connect(self):
+        # Connected. After this step, the unestablished
+        # property is false.
+        self.unestablished_connection = False
+        super(GraceDbCertHTTPSConnection, self).connect()
+
+
+class GraceDbCertHTTPSConnectionPool(HTTPSConnectionPool):
+    # ConnectionPool object gets used in the HTTPAdapter.
+    # "ConnectionCls" is a HTTP(S)COnnection object to use
+    # As the underlying connection.
+
+    # Source: https://urllib3.readthedocs.io/en/latest/
+    #         reference/#module-urllib3.connectionpool
+
+    ConnectionCls = GraceDbCertHTTPSConnection
+
+    def __init__(self, host, port=None, cert=None,
+                 reload_buffer=0, **kwargs):
+
+        super(GraceDbCertHTTPSConnectionPool, self).__init__(
+            host, port=port, **kwargs)
+
+        self._cert = cert
+        self._reload_buffer = reload_buffer
+
+    def _expired_cert(self):
+        return check_certificate_expiration(
+            self._cert,
+            self._reload_buffer)
+
+    def _get_conn(self, timeout=None):
+        while True:
+            # Start the connection object. At this step, the connection
+            # unestablished variable is true
+            conn = super(GraceDbCertHTTPSConnectionPool, self)._get_conn(
+                timeout)
+
+            # 'returning' the connection object then triggers the
+            # connection to be established. Establish a new connection
+            # if it's unestablished, or if the cert expiration is within the
+            # reload buffer. Establishing the new connection will (hopefully
+            # load the new cert.
+            if conn.unestablished_connection or not self._expired_cert():
+                return conn
+
+            # otherwise, kill the connection which will reset unestablished_..
+            # to true and then exit the loop.
+            conn.close()

ligo/gracedb/cert.py

+# -*- coding: utf-8 -*-
+# Copyright (C) Alexander Pace, Tanner Prestegard,
+#               Branson Stephens, Brian Moe (2020)
+#
+# This file is part of gracedb
+#
+# gracedb is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# It is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with gracedb.  If not, see <http://www.gnu.org/licenses/>
+
+# This file contains x509 certificate loading and checking tools.
+# Mostly duplicated effort from Tanner's work in support of certificate
+# reloading, but it could be useful for giving the client the option
+# of checking their certificates, or displaying certificate validity and
+# expiration dates when they use the 'gracedb credentials client' command.
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+import datetime
+
+far_future = datetime.timedelta(days=365)
+
+
+# Takes in the path of a certificate and retrrns an x509.cryptography
+# certificate object. This removes the prior check that the auth_type
+# be 'x509' since this should only be called when reload_certificate
+# is true, which checks for x509, or in cases in general when users want
+# to check a cert without connecting a client.
+
+def load_certificate(cert):
+    """ Loads in a path to a x509 certificate and returns a
+        x509.cryptography object """
+    with open(cert, 'rb') as cert_obj:
+        cert_obj = cert_obj.read()
+
+    # Try loading with PEM, then try loading with DER, then give up.
+
+    try:
+        return x509.load_pem_x509_certificate(
+            cert_obj, default_backend()
+        )
+    except ValueError:
+        try:
+            return x509.load_pem_x509_certificate(
+                cert_obj, default_backend()
+            )
+        except ValueError:
+            raise RuntimeError('Error importing certificate')
+
+
+# Checks certificate expiration and returns a boolean. Optionally checks
+# that the time left until expiration within the 'reload_buffer' parameter.
+
+def check_certificate_expiration(cert, reload_buffer=0):
+    """ Checks to see if a cert is expiring within an optional
+        reload_buffer parameter. Default, reload_buffer=0, meaning
+        is the certificate currently expired.
+
+        cert is either a string path to an x509 cert, or a x509
+        cryptography certificate object """
+
+    if not hasattr(cert, 'subject'):
+        cert = load_certificate(cert)
+
+    expired = (cert.not_valid_after - datetime.datetime.utcnow()) <= \
+        datetime.timedelta(seconds=reload_buffer)
+
+    return expired

ligo/gracedb/cli/client.py

@@ -4,6 +4,7 @@ import os
 import six
 import sys
 import textwrap
+from requests import Response
 
 from ligo.gracedb.rest import GraceDb, DEFAULT_SERVICE_URL
 from ligo.gracedb import __version__
@@ -21,15 +22,15 @@ class CommandLineClient(GraceDb):
     # Hamstring 'adjustResponse' from the example REST client.
     # We don't want it messing with the response from the server.
     def adjustResponse(self, response):
-        response.json = lambda: self.load_json_or_die(response)
+        response.json = lambda: self.load_json_from_response(response)
         return response
 
     # TP 2019: not sure if we still need to override this from the GraceDb
     # class, but leaving it for now.
-    @classmethod
-    def output_and_die(cls, msg):
-        sys.stderr.write(msg)
-        sys.exit(1)
+    # @classmethod
+    # def output_and_die(cls, msg):
+    #    sys.stderr.write(msg)
+    #    sys.exit(1)
 
 
 class CommandLineInterface(CommandBase):
@@ -169,8 +170,6 @@ class CommandLineInterface(CommandBase):
         # Define kwargs for initializing client
         client_kwargs = {
             'service_url': args.service,
-            'proxy_host': proxy,
-            'proxy_port': proxyport,
             'force_noauth': args.force_noauth,
             'username': args.username,
             'password': args.password,
@@ -193,21 +192,24 @@ def main(args=None):
     try:
         response = cli(args)
     except HTTPError as e:
-        print('Error: {code} {reason}'.format(code=e.status, reason=e.reason))
+        print('Error: {code} {reason}. {text}.'.format(
+            code=e.status_code,
+            reason=e.reason,
+            text=e.text))
         sys.exit(1)
     except Exception as e:
         print('Error: {msg}'.format(msg=str(e)))
         sys.exit(1)
 
-    if isinstance(response, six.moves.http_client.HTTPResponse):
+    if isinstance(response, Response):
         if (cli.args.output_type == 'json'):
             # Handle errors
-            if response.status >= 400:
-                output = '{code} {reason}'.format(code=response.status,
+            if response.status_code >= 400:
+                output = '{code} {reason}'.format(code=response.status_code,
                                                   reason=response.reason)
                 # Only add message if it's not really long (i.e., it's not
                 # an HTML error page)
-                msg = response.read()
+                msg = response.text
                 if isinstance(msg, bytes):
                     msg = msg.decode()
                 if (len(msg) < 1000):
@@ -215,16 +217,21 @@ def main(args=None):
                 print(output)
                 sys.exit(1)
             # Handle errors raised in load_json_or_die()
-            try:
-                output = response.json()
-            except Exception as e:
-                print(str(e))
+            if response.status_code == 204:
+                output = {}
+                print(output)
                 sys.exit(1)
+            else:
+                try:
+                    output = response.json()
+                except Exception as e:
+                    print(str(e))
+                    sys.exit(1)
 
             print(json.dumps(output, indent=4))
         elif (cli.args.output_type == 'status'):
             print('Server returned {status}: {reason}'.format(
-                status=response.status, reason=response.reason))
+                status=response.status_code, reason=response.reason))
     elif isinstance(response, dict):
         print(json.dumps(response, indent=4))
     elif isinstance(response, six.string_types):

ligo/gracedb/cli/commands/get.py

@@ -94,7 +94,7 @@ class GetFileCommand(GetChildBase):
         response = client.files(args.object_id, args.filename)
 
         # Handle response
-        if response.status == 200:
+        if response.status_code == 200:
             if args.destination == '-':
                 # For stdout, return string contents of file
                 file_contents = response.read()

ligo/gracedb/cli/commands/subcommands.py

@@ -94,8 +94,8 @@ class PingCommand(RegisteredSubCommandBase):
     def run(self, client, args):
         response = client.ping()
         output = 'Response from {server}: {status}'.format(
-            server=client._service_url, status=response.status)
-        if (response.status == 200):
+            server=client._service_url, status=response.status_code)
+        if (response.status_code == 200):
             output += ' OK'
         return output
 

ligo/gracedb/cli/commands/tests/test_level1_subcommands.py

@@ -144,7 +144,7 @@ def test_ping_subcommand(CLI):
     """Test ping subcommand"""
     func = 'ligo.gracedb.rest.GraceDb.ping'
     with mock.patch(func) as mock_cli_func:
-        mock_cli_func.return_value = mock.MagicMock(status=123)
+        mock_cli_func.return_value = mock.MagicMock(status_code=123)
         output = CLI(['ping'])
 
     # Check call count
@@ -159,7 +159,7 @@ def test_ping_subcommand(CLI):
 
     # Check output
     assert output == 'Response from {server}: {status}'.format(
-        server=CLI.client._service_url, status=mock_cli_func().status)
+        server=CLI.client._service_url, status=mock_cli_func().status_code)
 
 
 @pytest.mark.parametrize(

ligo/gracedb/cli/tests/test_cli.py

@@ -88,8 +88,6 @@ def test_cli_client_setup(CLI):
     cli_args, cli_kwargs = mock_client.call_args
     assert cli_args == ()
     assert cli_kwargs['service_url'] == arg_dict['service-url']
-    assert cli_kwargs['proxy_host'] == arg_dict['proxy'].split(':')[0]
-    assert cli_kwargs['proxy_port'] == int(arg_dict['proxy'].split(':')[1])
     assert cli_kwargs['username'] == arg_dict['username']
     assert cli_kwargs['password'] == arg_dict['password']
     assert cli_kwargs['cred'] == arg_dict['creds'].split(',')

ligo/gracedb/cli/tests/test_main.py

 import json
 import pytest
-import six
+# import six
 try:
     from unittest import mock
 except ImportError:
     import mock
 
 from ligo.gracedb.exceptions import HTTPError
+from requests import Response
 
 # Apply module-level mark
 pytestmark = pytest.mark.cli
@@ -23,8 +24,8 @@ def test_main_with_ok_response(main_tester, capsys, output_type):
         status = 200
         reason = 'because'
         response_mock = mock.MagicMock(
-            spec=six.moves.http_client.HTTPResponse,
-            status=status, reason=reason)
+            spec=Response,
+            status_code=status, reason=reason)
         mock_CLI().return_value = response_mock
         # Set up response.json()
         json_mock = mock.MagicMock()
@@ -62,14 +63,10 @@ def test_main_with_bad_response(main_tester, capsys, response_msg):
         status = 400
         reason = 'Bad Request'
         response_mock = mock.MagicMock(
-            spec=six.moves.http_client.HTTPResponse,
-            status=status, reason=reason)
+            spec=Response,
+            status_code=status, reason=reason,
+            text=response_msg)
         mock_CLI().return_value = response_mock
-        # Set up response.read()
-        read_mock = mock.MagicMock()
-        read_mock.return_value = response_msg
-        # Use setattr since the MagicMock is autospecced
-        setattr(response_mock, 'read', read_mock)
 
         # Set up CLI.args.output_type
         mock_CLI().args.output_type = 'json'
@@ -93,7 +90,9 @@ def test_main_with_bad_response(main_tester, capsys, response_msg):
             code=status, reason=reason)
 
 
-EXCEPTION_DATA = [Exception('test'), HTTPError(400, 'Bad Request', 'test')]
+EXCEPTION_DATA = [Exception('test'),
+                  HTTPError(400, 'Bad Request', 'test')]
+
 @pytest.mark.parametrize("exc", EXCEPTION_DATA)  # noqa: E302
 def test_main_with_exception_in_CLI_call(main_tester, capsys, exc):
     """Test CLI wrapper when CLI call raises exception"""
@@ -115,8 +114,8 @@ def test_main_with_exception_in_CLI_call(main_tester, capsys, exc):
 
     # Test output
     if isinstance(exc, HTTPError):
-        assert stdout.rstrip() == 'Error: {code} {reason}'.format(
-            code=exc.status, reason=exc.reason)
+        assert stdout.rstrip() == 'Error: {code} {reason}. {text}.'.format(
+            code=exc.status_code, reason=exc.reason, text=exc.text)
     elif isinstance(exc, Exception):
         assert stdout.rstrip() == 'Error: {msg}'.format(msg=str(exc))
 
@@ -130,8 +129,8 @@ def test_main_with_exception_in_response_json(main_tester, capsys):
         status = 200
         reason = 'OK'
         response_mock = mock.MagicMock(
-            spec=six.moves.http_client.HTTPResponse,
-            status=status, reason=reason)
+            spec=Response,
+            status_code=status, reason=reason)
         mock_CLI().return_value = response_mock
         # Set up response.read()
         exc_msg = 'uh oh'

ligo/gracedb/client.py

+# -*- coding: utf-8 -*-
+# Copyright (C) Alexander Pace, Tanner Prestegard,
+#               Branson Stephens, Brian Moe (2020)
+#
+# This file is part of gracedb
+#
+# gracedb is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# It is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with gracedb.  If not, see <http://www.gnu.org/licenses/>.
+import os
+import sys
+import json as json_lib
+
+from warnings import warn
+from requests import Session
+from .extern.safe_netrc import netrc
+from os import getuid
+from .version import __version__
+from .adapter import GraceDbCertAdapter
+from .utils import hook_response
+
+# To remove later: python2 compatibility fix:
+if sys.version_info[0] > 2:
+    from urllib.parse import urlparse
+else:
+    from urlparse import urlparse
+
+DEFAULT_SERVICE_URL = "https://gracedb.ligo.org/api/"
+
+
+class GraceDBClient(Session):
+    """
+    url (:obj:`str`, optional): URL of server API
+    cred (:obj:`tuple` or :obj:`str, optional): a tuple or list of
+        (``/path/to/cert/file``, ``/path/to/key/file) or a single path to
+        a combined proxy file (if using an X.509 certificate for
+        authentication)
+    username (:obj:`str`, optional): username for basic auth
+    password (:obj:`str`, optional): password for basic auth
+    force_noauth (:obj:`bool`, optional): set to True if you want to skip
+        credential lookup and use this client as an unauthenticated user
+    fail_if_noauth (:obj:`bool`, optional): set to True if you want the
+        constructor to fail if no authentication credentials are provided
+        or found
+    reload_certificate (:obj:`bool`, optional): if ``True``, your
+        certificate will be checked before each request whether it is
+        within ``reload_buffer`` seconds of expiration, and if so, it will
+        be reloaded. Useful for processes which may live longer than the
+        certificate lifetime and have an automated method for certificate
+        renewal. The path to the new/renewed certificate **must** be the
+        same as for the old certificate.
+    reload_buffer (:obj:`int`, optional): buffer (in seconds) for reloading
+        a certificate in advance of its expiration. Only used if
+        ``reload_certificate`` is ``True``.
+
+    Authentication details:
+    You can:
+        1. Provide a path to an X.509 certificate and key or a single
+           combined proxy file
+        2. Provide a username and password
+    Or:
+        The code will look for a certificate in a default location
+            (``/tmp/x509up_u%d``, where ``%d`` is your user ID)
+        The code will look for a username and password for the specified
+            server in ``$HOME/.netrc``
+    """
+
+    def __init__(self, url=DEFAULT_SERVICE_URL, cred=None, username=None,
+                 password=None, force_noauth=False, fail_if_noauth=False,
+                 reload_certificate=False, reload_buffer=300,
+                 *args, **kwargs):
+        super(GraceDBClient, self).__init__(*args, **kwargs)
+
+        # Initialize variables:
+        self.cert = None
+        self.auth = None
+        self.host = urlparse(url).hostname
+        self.auth_type = None
+
+        # Set up credentials. First, only attempt if the user did not
+        # force noauth:
+        if not force_noauth:
+            # Attempt to assign x509 credentials.
+            if cred or not (username or password):
+                self.cert = self._get_x509_credentials(cred)
+
+            # if we weren't able to determine a x509 certificate,
+            # then look for basic auth.
+            if not self.cert:
+                self.auth = self._process_basic_credentials(username,
+                                                            password)
+
+                # If no basic auth credentials were provided, and
+                # fail_if_noauthis set, then fail.
+                if not self.auth:
+                    if fail_if_noauth:
+                        raise RuntimeError("No authentication credentials "
+                                           "could be found, and fail_if_"
+                                           "noauth is set.")
+                    else:
+                        warn("Authentication credentials not found, "
+                             "proceeding with unauthorized session")
+
+        elif fail_if_noauth:
+            raise ValueError('You have provided conflicting parameters '
+                             'to the client constructor: '
+                             'fail_if_noauth=True and force_noauth=True.')
+
+        # Update session headers:
+        self.headers.update(self._update_headers())
+
+        # Adjust the response via a session hook:
+        self.hooks = {'response': hook_response}
+
+        if reload_certificate and self.auth_type == 'x509':
+            self.mount('https://', GraceDbCertAdapter(
+                       cert=self.cert,
+                       reload_buffer=reload_buffer))
+
+    def _process_basic_credentials(self, username, password):
+        """ Gathers basic auth credentials. First it looks for
+            the input username/password, then checks the netrc file
+        """
+
+        # Checks username/password:
+        if username and password:
+            self.auth_type = 'basic'
+            return username, password
+        # If one or the other is provided, but not both, raise an error.
+        elif bool(username) != bool(password):
+            raise RuntimeError('You must provide both a username and a '
+                               'password for basic authentication.')
+        # Finally, try the netrc fle:
+        else:
+            try:
+                netrc_auth = netrc().authenticators(self.host)
+            except IOError: