merging in changes for 2.7.0

debian/changelog

+ligo-gracedb (2.7.0-1) unstable; urgency=low
+  * Backend rewrite using "requests" package
+
+ -- Alexander E. Pace <alexander.pace@ligo.org>  Thu, 06 Aug 2020 11:03:15 -0700
+
 ligo-gracedb (2.6.1-1) unstable; urgency=low
   * SSL_PROTOCOL fix for python < 2.7.13
 

debian/control

@@ -9,7 +9,7 @@ X-Python3-Version: >=3.5
 
 Package: python-ligo-gracedb
 Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}, python-ligo-common, python-future, python-setuptools, python-cryptography
+Depends: ${misc:Depends}, ${python:Depends}, python-ligo-common, python-future, python-setuptools, python-cryptography, python-requests
 Provides: ${python:Provides}
 Description: Gravitational-wave Candidate Event Database - Python
  The gravitational-wave candidate event database (GraceDB) is a prototype
@@ -21,7 +21,7 @@ Description: Gravitational-wave Candidate Event Database - Python
 
 Package: python3-ligo-gracedb
 Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}, python3-ligo-common, python3-future, python3-setuptools, python3-cryptography
+Depends: ${misc:Depends}, ${python3:Depends}, python3-ligo-common, python3-future, python3-setuptools, python3-cryptography, python3-requests
 Provides: ${python3:Provides}
 Description: Gravitational-wave Candidate Event Database - Python 3
  The gravitational-wave candidate event database (GraceDB) is a prototype

ligo-gracedb.spec

 %define name              ligo-gracedb
-%define version           2.6.1
-%define unmangled_version 2.6.1
+%define version           2.7.0
+%define unmangled_version 2.7.0
 %define release           1
 
 Summary:   Gravity Wave Candidate Event Database
@@ -44,6 +44,7 @@ Requires: python-six
 Requires: python2-ligo-common
 Requires: python-future
 Requires: python2-cryptography
+Requires: python-requests
 
 %{?python_provide:%python_provide python2-%{name}}
 

ligo/gracedb/adapter.py

+# -*- coding: utf-8 -*-
+# Copyright (C) Alexander Pace, Tanner Prestegard,
+#               Branson Stephens, Brian Moe (2020)
+#
+# This file is part of gracedb
+#
+# gracedb is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# It is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with gracedb.  If not, see <http://www.gnu.org/licenses/>
+
+# Sources:
+#  1) https://stackoverflow.com/questions/45539422/can-we-reload-a-page-url-
+#     in-python-using-urllib-or-urllib2-or-requests-or-mechan
+
+#  2) https://2.python-requests.org/en/master/user/advanced/#example-
+#     specific-ssl-version
+
+#  3) https://urllib3.readthedocs.io/en/1.2.1/pools.html
+
+
+from functools import partial
+from requests.adapters import HTTPAdapter
+from urllib3.connection import HTTPSConnection
+from urllib3.connectionpool import HTTPSConnectionPool, HTTPConnectionPool
+from .cert import check_certificate_expiration
+
+
+class GraceDbCertAdapter(HTTPAdapter):
+    def __init__(self, cert=None, reload_buffer=0, **kwargs):
+        super(GraceDbCertAdapter, self).__init__(**kwargs)
+        https_pool_cls = partial(
+            GraceDbCertHTTPSConnectionPool,
+            cert=cert,
+            reload_buffer=reload_buffer)
+
+        self.poolmanager.pool_classes_by_scheme = {
+            'http': HTTPConnectionPool,
+            'https': https_pool_cls
+        }
+
+
+class GraceDbCertHTTPSConnection(HTTPSConnection):
+    def __init__(self, host, cert=None, reload_buffer=0, **kwargs):
+        # At this point, te HTTPSConnection is initialized
+        # but unconnected. Set this property to 'True'
+        self.unestablished_connection = True
+        super(GraceDbCertHTTPSConnection, self).__init__(host, **kwargs)
+
+    @property
+    def unestablished_connection(self):
+        return self._unestablished_connection
+
+    @unestablished_connection.setter
+    def unestablished_connection(self, value):
+        self._unestablished_connection = value
+
+    def connect(self):
+        # Connected. After this step, the unestablished
+        # property is false.
+        self.unestablished_connection = False
+        super(GraceDbCertHTTPSConnection, self).connect()
+
+
+class GraceDbCertHTTPSConnectionPool(HTTPSConnectionPool):
+    # ConnectionPool object gets used in the HTTPAdapter.
+    # "ConnectionCls" is a HTTP(S)COnnection object to use
+    # As the underlying connection.
+
+    # Source: https://urllib3.readthedocs.io/en/latest/
+    #         reference/#module-urllib3.connectionpool
+
+    ConnectionCls = GraceDbCertHTTPSConnection
+
+    def __init__(self, host, port=None, cert=None,
+                 reload_buffer=0, **kwargs):
+
+        super(GraceDbCertHTTPSConnectionPool, self).__init__(
+            host, port=port, **kwargs)
+
+        self._cert = cert
+        self._reload_buffer = reload_buffer
+
+    def _expired_cert(self):
+        return check_certificate_expiration(
+            self._cert,
+            self._reload_buffer)
+
+    def _get_conn(self, timeout=None):
+        while True:
+            # Start the connection object. At this step, the connection
+            # unestablished variable is true
+            conn = super(GraceDbCertHTTPSConnectionPool, self)._get_conn(
+                timeout)
+
+            # 'returning' the connection object then triggers the
+            # connection to be established. Establish a new connection
+            # if it's unestablished, or if the cert expiration is within the
+            # reload buffer. Establishing the new connection will (hopefully
+            # load the new cert.
+            if conn.unestablished_connection or not self._expired_cert():
+                return conn
+
+            # otherwise, kill the connection which will reset unestablished_..
+            # to true and then exit the loop.
+            conn.close()

ligo/gracedb/cert.py

+# -*- coding: utf-8 -*-
+# Copyright (C) Alexander Pace, Tanner Prestegard,
+#               Branson Stephens, Brian Moe (2020)
+#
+# This file is part of gracedb
+#
+# gracedb is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# It is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with gracedb.  If not, see <http://www.gnu.org/licenses/>
+
+# This file contains x509 certificate loading and checking tools.
+# Mostly duplicated effort from Tanner's work in support of certificate
+# reloading, but it could be useful for giving the client the option
+# of checking their certificates, or displaying certificate validity and
+# expiration dates when they use the 'gracedb credentials client' command.
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+import datetime
+
+far_future = datetime.timedelta(days=365)
+
+
+# Takes in the path of a certificate and retrrns an x509.cryptography
+# certificate object. This removes the prior check that the auth_type
+# be 'x509' since this should only be called when reload_certificate
+# is true, which checks for x509, or in cases in general when users want
+# to check a cert without connecting a client.
+
+def load_certificate(cert):
+    """ Loads in a path to a x509 certificate and returns a
+        x509.cryptography object """
+    with open(cert, 'rb') as cert_obj:
+        cert_obj = cert_obj.read()
+
+    # Try loading with PEM, then try loading with DER, then give up.
+
+    try:
+        return x509.load_pem_x509_certificate(
+            cert_obj, default_backend()
+        )
+    except ValueError:
+        try:
+            return x509.load_pem_x509_certificate(
+                cert_obj, default_backend()
+            )
+        except ValueError:
+            raise RuntimeError('Error importing certificate')
+
+
+# Checks certificate expiration and returns a boolean. Optionally checks
+# that the time left until expiration within the 'reload_buffer' parameter.
+
+def check_certificate_expiration(cert, reload_buffer=0):
+    """ Checks to see if a cert is expiring within an optional
+        reload_buffer parameter. Default, reload_buffer=0, meaning
+        is the certificate currently expired.
+
+        cert is either a string path to an x509 cert, or a x509
+        cryptography certificate object """
+
+    if not hasattr(cert, 'subject'):
+        cert = load_certificate(cert)
+
+    expired = (cert.not_valid_after - datetime.datetime.utcnow()) <= \
+        datetime.timedelta(seconds=reload_buffer)
+
+    return expired

ligo/gracedb/cli/client.py

@@ -4,6 +4,7 @@ import os
 import six
 import sys
 import textwrap
+from requests import Response
 
 from ligo.gracedb.rest import GraceDb, DEFAULT_SERVICE_URL
 from ligo.gracedb import __version__
@@ -21,15 +22,15 @@ class CommandLineClient(GraceDb):
     # Hamstring 'adjustResponse' from the example REST client.
     # We don't want it messing with the response from the server.
     def adjustResponse(self, response):
-        response.json = lambda: self.load_json_or_die(response)
+        response.json = lambda: self.load_json_from_response(response)
         return response
 
     # TP 2019: not sure if we still need to override this from the GraceDb
     # class, but leaving it for now.
-    @classmethod
-    def output_and_die(cls, msg):
-        sys.stderr.write(msg)
-        sys.exit(1)
+    # @classmethod
+    # def output_and_die(cls, msg):
+    #    sys.stderr.write(msg)
+    #    sys.exit(1)
 
 
 class CommandLineInterface(CommandBase):
@@ -169,8 +170,6 @@ class CommandLineInterface(CommandBase):
         # Define kwargs for initializing client
         client_kwargs = {
             'service_url': args.service,
-            'proxy_host': proxy,
-            'proxy_port': proxyport,
             'force_noauth': args.force_noauth,
             'username': args.username,
             'password': args.password,
@@ -193,21 +192,24 @@ def main(args=None):
     try:
         response = cli(args)
     except HTTPError as e:
-        print('Error: {code} {reason}'.format(code=e.status, reason=e.reason))
+        print('Error: {code} {reason}. {text}.'.format(
+            code=e.status_code,
+            reason=e.reason,
+            text=e.text))
         sys.exit(1)
     except Exception as e:
         print('Error: {msg}'.format(msg=str(e)))
         sys.exit(1)
 
-    if isinstance(response, six.moves.http_client.HTTPResponse):
+    if isinstance(response, Response):
         if (cli.args.output_type == 'json'):
             # Handle errors
-            if response.status >= 400:
-                output = '{code} {reason}'.format(code=response.status,
+            if response.status_code >= 400:
+                output = '{code} {reason}'.format(code=response.status_code,
                                                   reason=response.reason)
                 # Only add message if it's not really long (i.e., it's not
                 # an HTML error page)
-                msg = response.read()
+                msg = response.text
                 if isinstance(msg, bytes):
                     msg = msg.decode()
                 if (len(msg) < 1000):
@@ -215,16 +217,21 @@ def main(args=None):
                 print(output)
                 sys.exit(1)
             # Handle errors raised in load_json_or_die()
-            try:
-                output = response.json()
-            except Exception as e:
-                print(str(e))
+            if response.status_code == 204:
+                output = {}
+                print(output)
                 sys.exit(1)
+            else:
+                try:
+                    output = response.json()
+                except Exception as e:
+                    print(str(e))
+                    sys.exit(1)
 
             print(json.dumps(output, indent=4))
         elif (cli.args.output_type == 'status'):
             print('Server returned {status}: {reason}'.format(
-                status=response.status, reason=response.reason))
+                status=response.status_code, reason=response.reason))
     elif isinstance(response, dict):
         print(json.dumps(response, indent=4))
     elif isinstance(response, six.string_types):

ligo/gracedb/cli/commands/get.py

@@ -94,7 +94,7 @@ class GetFileCommand(GetChildBase):
         response = client.files(args.object_id, args.filename)
 
         # Handle response
-        if response.status == 200:
+        if response.status_code == 200:
             if args.destination == '-':
                 # For stdout, return string contents of file
                 file_contents = response.read()

ligo/gracedb/cli/commands/subcommands.py

@@ -94,8 +94,8 @@ class PingCommand(RegisteredSubCommandBase):
     def run(self, client, args):
         response = client.ping()
         output = 'Response from {server}: {status}'.format(
-            server=client._service_url, status=response.status)
-        if (response.status == 200):
+            server=client._service_url, status=response.status_code)
+        if (response.status_code == 200):
             output += ' OK'
         return output
 

ligo/gracedb/cli/commands/tests/test_level1_subcommands.py

@@ -144,7 +144,7 @@ def test_ping_subcommand(CLI):
     """Test ping subcommand"""
     func = 'ligo.gracedb.rest.GraceDb.ping'
     with mock.patch(func) as mock_cli_func:
-        mock_cli_func.return_value = mock.MagicMock(status=123)
+        mock_cli_func.return_value = mock.MagicMock(status_code=123)
         output = CLI(['ping'])
 
     # Check call count
@@ -159,7 +159,7 @@ def test_ping_subcommand(CLI):
 
     # Check output
     assert output == 'Response from {server}: {status}'.format(
-        server=CLI.client._service_url, status=mock_cli_func().status)
+        server=CLI.client._service_url, status=mock_cli_func().status_code)
 
 
 @pytest.mark.parametrize(

ligo/gracedb/cli/tests/test_cli.py

@@ -88,8 +88,6 @@ def test_cli_client_setup(CLI):
     cli_args, cli_kwargs = mock_client.call_args
     assert cli_args == ()
     assert cli_kwargs['service_url'] == arg_dict['service-url']
-    assert cli_kwargs['proxy_host'] == arg_dict['proxy'].split(':')[0]
-    assert cli_kwargs['proxy_port'] == int(arg_dict['proxy'].split(':')[1])
     assert cli_kwargs['username'] == arg_dict['username']
     assert cli_kwargs['password'] == arg_dict['password']
     assert cli_kwargs['cred'] == arg_dict['creds'].split(',')

ligo/gracedb/cli/tests/test_main.py

 import json
 import pytest
-import six
+# import six
 try:
     from unittest import mock
 except ImportError:
     import mock
 
 from ligo.gracedb.exceptions import HTTPError
+from requests import Response
 
 # Apply module-level mark
 pytestmark = pytest.mark.cli
@@ -23,8 +24,8 @@ def test_main_with_ok_response(main_tester, capsys, output_type):
         status = 200
         reason = 'because'
         response_mock = mock.MagicMock(
-            spec=six.moves.http_client.HTTPResponse,
-            status=status, reason=reason)
+            spec=Response,
+            status_code=status, reason=reason)
         mock_CLI().return_value = response_mock
         # Set up response.json()
         json_mock = mock.MagicMock()
@@ -62,14 +63,10 @@ def test_main_with_bad_response(main_tester, capsys, response_msg):
         status = 400
         reason = 'Bad Request'
         response_mock = mock.MagicMock(
-            spec=six.moves.http_client.HTTPResponse,
-            status=status, reason=reason)
+            spec=Response,
+            status_code=status, reason=reason,
+            text=response_msg)
         mock_CLI().return_value = response_mock
-        # Set up response.read()
-        read_mock = mock.MagicMock()
-        read_mock.return_value = response_msg
-        # Use setattr since the MagicMock is autospecced
-        setattr(response_mock, 'read', read_mock)
 
         # Set up CLI.args.output_type
         mock_CLI().args.output_type = 'json'
@@ -93,7 +90,9 @@ def test_main_with_bad_response(main_tester, capsys, response_msg):
             code=status, reason=reason)
 
 
-EXCEPTION_DATA = [Exception('test'), HTTPError(400, 'Bad Request', 'test')]
+EXCEPTION_DATA = [Exception('test'),
+                  HTTPError(400, 'Bad Request', 'test')]
+
 @pytest.mark.parametrize("exc", EXCEPTION_DATA)  # noqa: E302
 def test_main_with_exception_in_CLI_call(main_tester, capsys, exc):
     """Test CLI wrapper when CLI call raises exception"""
@@ -115,8 +114,8 @@ def test_main_with_exception_in_CLI_call(main_tester, capsys, exc):
 
     # Test output
     if isinstance(exc, HTTPError):
-        assert stdout.rstrip() == 'Error: {code} {reason}'.format(
-            code=exc.status, reason=exc.reason)
+        assert stdout.rstrip() == 'Error: {code} {reason}. {text}.'.format(
+            code=exc.status_code, reason=exc.reason, text=exc.text)
     elif isinstance(exc, Exception):
         assert stdout.rstrip() == 'Error: {msg}'.format(msg=str(exc))
 
@@ -130,8 +129,8 @@ def test_main_with_exception_in_response_json(main_tester, capsys):
         status = 200
         reason = 'OK'
         response_mock = mock.MagicMock(
-            spec=six.moves.http_client.HTTPResponse,
-            status=status, reason=reason)
+            spec=Response,
+            status_code=status, reason=reason)
         mock_CLI().return_value = response_mock
         # Set up response.read()
         exc_msg = 'uh oh'

ligo/gracedb/exceptions.py

 # Custom exceptions
-
-
 class HTTPError(Exception):
-    def __init__(self, status, reason, message):
-        self.status = status
+    def __init__(self,
+                 status_code,
+                 reason,
+                 text):
+        self.status = status_code
+        self.status_code = status_code
         self.reason = reason
-        self.message = message
-        Exception.__init__(self, status, reason, message)
+        self.message = text
+        self.text = text
+        Exception.__init__(self,
+                           status_code,
+                           reason,
+                           text)

ligo/gracedb/test/integration/test_api_root.py

@@ -6,7 +6,7 @@ pytestmark = pytest.mark.integration
 
 def test_ping(client):
     response = client.ping()
-    assert response.status == 200
+    assert response.status_code == 200
 
 
 @pytest.mark.parametrize(

ligo/gracedb/test/integration/test_emobservations.py

@@ -30,7 +30,7 @@ def test_emobservations(client, create_obj, obj_type):
         obj_id, emgroup, ra_list, ra_width_list, dec_list, dec_width_list,
         start_time_list, duration_list, comment=comment
     )
-    assert response.status == 201
+    assert response.status_code == 201
     data = response.json()
     assert data['comment'] == comment
     assert data['group'] == emgroup
@@ -46,14 +46,14 @@ def test_emobservations(client, create_obj, obj_type):
 
     # Get list of emobservations and check results
     response = client.emobservations(obj_id)
-    assert response.status == 200
+    assert response.status_code == 200
     data = response.json()
     assert len(data['observations']) == 1
     assert data['observations'][0]['N'] == emo_N
 
     # Retrieve the individual emobservation directly
     response = client.emobservations(obj_id, emo_N)
-    assert response.status == 200
+    assert response.status_code == 200
     data = response.json()
     assert data['comment'] == comment
     assert data['group'] == emgroup

ligo/gracedb/test/integration/test_events.py

@@ -11,7 +11,7 @@ def test_create_gstlal(client, test_data_dir):
     # Create gstlal event
     filename = os.path.join(test_data_dir, 'cbc-lm.xml')
     response = client.createEvent('Test', 'gstlal', filename)
-    assert response.status == 201
+    assert response.status_code == 201
     data = response.json()
 
     # Test results
@@ -21,6 +21,47 @@ def test_create_gstlal(client, test_data_dir):
     assert data['extra_attributes']['CoincInspiral']['snr'] == 9.31793628458239
 
 
+def test_create_gstlal_with_missing_table_entry(client, test_data_dir):
+    # Create gstlal event, with a missing snglinspiral table
+    # entry. The event should get made and the table populated with
+    # a None entry.
+    filename = os.path.join(test_data_dir, 'cbc-lm-missing-entry.xml')
+    response = client.createEvent('Test', 'gstlal', filename)
+    assert response.status_code == 201
+    data = response.json()
+
+    # Make sure warnings are empty:
+    assert data['warnings'] == []
+
+    # Test results
+    assert data['group'] == 'Test'
+    assert data['pipeline'] == 'gstlal'
+    assert data['gpstime'] == 971609248.151741
+    assert data['extra_attributes']['CoincInspiral']['snr'] == 9.31793628458239
+
+    # Extra results. Assert that the snr snglinspiral table entry is empty
+    assert data['extra_attributes']['SingleInspiral'][0].get('snr') is None
+
+
+def test_create_gstlal_no_ilwdchar(client, test_data_dir):
+    # Create gstlal event, where the ligol ilwd:char entries
+    # have been replaced with int8's.
+    filename = os.path.join(test_data_dir, 'cbc-lm-no-ilwdchar.xml')
+    response = client.createEvent('Test', 'gstlal', filename)
+    assert response.status_code == 201
+    data = response.json()
+
+    # Make sure warnings are empty:
+    assert data['warnings'] == []
+
+    # Test results
+    assert data['group'] == 'Test'
+    assert data['pipeline'] == 'gstlal'
+    assert data['gpstime'] == 971609248.151741
+    # Also, this assert will fail if the event is not read correctly:
+    assert data['extra_attributes']['CoincInspiral']['snr'] == 9.31793628458239
+
+
 @pytest.mark.xfail
 def test_create_pycbc(client, test_data_dir):
     # Create PyCBC event - need to get a PyCBC data file and clean it up,
@@ -32,7 +73,7 @@ def test_create_mbta(client, test_data_dir):
     # Create MBTA event
     filename = os.path.join(test_data_dir, 'cbc-mbta.xml')
     response = client.createEvent('Test', 'MBTAOnline', filename)
-    assert response.status == 201
+    assert response.status_code == 201
     data = response.json()
 
     # Test results
@@ -47,11 +88,29 @@ def test_create_spiir(client, test_data_dir):
     # Create SPIIR event
     filename = os.path.join(test_data_dir, 'spiir-test.xml')
     response = client.createEvent('Test', 'spiir', filename)
-    assert response.status == 201
+    assert response.status_code == 201
     data = response.json()
 
     # Test results
-    assert response.status == 201
+    assert response.status_code == 201
+    assert data['group'] == 'Test'
+    assert data['pipeline'] == 'spiir'
+    assert data['extra_attributes']['CoincInspiral']['mass'] == 3.98
+    assert data['far'] == 3.27e-07
+
+
+def test_create_spiir_no_ilwdchar(client, test_data_dir):
+    # Create SPIIR event, with ilwd:char entires converted to int8s
+    filename = os.path.join(test_data_dir, 'spiir-test-no-ilwdchar.xml')
+    response = client.createEvent('Test', 'spiir', filename)
+    assert response.status_code == 201
+    data = response.json()
+
+    # Make sure warnings are empty:
+    assert data['warnings'] == []
+
+    # Test results
+    assert response.status_code == 201
     assert data['group'] == 'Test'
     assert data['pipeline'] == 'spiir'
     assert data['extra_attributes']['CoincInspiral']['mass'] == 3.98
@@ -62,7 +121,7 @@ def test_create_cwb(client, test_data_dir):
     # Create CWB event
     filename = os.path.join(test_data_dir, 'burst-cwb.txt')
     response<