Skip tests that error with URLError instead of hard fail
Occassionally the test suite for pesummary fails because of network issues that prevent downloading data needed for the tests, including here in the TestPublicPrincetonO1O2
test class:
def setup(self):
"""Setup the TestCoreBilbyFile class
"""
from pesummary.core.fetch import download_and_read_file
if not os.path.isdir(".outdir"):
os.mkdir(".outdir")
> self.file = download_and_read_file(
"https://github.com/jroulet/O2_samples/raw/master/GW150914.npy",
read_file=False, outdir=".outdir"
)
It would be great to wrap this sort of setup inside a try/except
block so that network issues (urllib.error.URLError
) are caught and the tests skipped, instead of the whole suite failing.
Full pytest traceback
_________ ERROR at setup of TestPublicPrincetonO1O2.test_samples_dict __________
self = <urllib.request.HTTPSHandler object at 0x7f334e071910>
http_class = <class 'http.client.HTTPSConnection'>
req = <urllib.request.Request object at 0x7f334e07ca60>
http_conn_args = {'check_hostname': None, 'context': <ssl.SSLContext object at 0x7f334dabcf40>}
host = 'github.com', h = <http.client.HTTPSConnection object at 0x7f334b00be50>
def do_open(self, http_class, req, **http_conn_args):
"""Return an HTTPResponse object for the request, using http_class.
http_class must implement the HTTPConnection API from http.client.
"""
host = req.host
if not host:
raise URLError('no host given')
# will parse host:port
h = http_class(host, timeout=req.timeout, **http_conn_args)
h.set_debuglevel(self._debuglevel)
headers = dict(req.unredirected_hdrs)
headers.update({k: v for k, v in req.headers.items()
if k not in headers})
# TODO(jhylton): Should this be redesigned to handle
# persistent connections?
# We want to make an HTTP/1.1 request, but the addinfourl
# class isn't prepared to deal with a persistent connection.
# It will try to read all remaining data from the socket,
# which will block while the server waits for the next request.
# So make sure the connection gets closed after the (only)
# request.
headers["Connection"] = "close"
headers = {name.title(): val for name, val in headers.items()}
if req._tunnel_host:
tunnel_headers = {}
proxy_auth_hdr = "Proxy-Authorization"
if proxy_auth_hdr in headers:
tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
# Proxy-Authorization should not be sent to origin
# server.
del headers[proxy_auth_hdr]
h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
try:
try:
> h.request(req.get_method(), req.selector, req.data, headers,
encode_chunked=req.has_header('Transfer-encoding'))
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/urllib/request.py:1346:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <http.client.HTTPSConnection object at 0x7f334b00be50>, method = 'GET'
url = '/jroulet/O2_samples/raw/master/GW150914.npy', body = None
headers = {'Accept': '*/*', 'Connection': 'close', 'Host': 'github.com', 'User-Agent': 'astropy'}
def request(self, method, url, body=None, headers={}, *,
encode_chunked=False):
"""Send a complete request to the server."""
> self._send_request(method, url, body, headers, encode_chunked)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/http/client.py:1285:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <http.client.HTTPSConnection object at 0x7f334b00be50>, method = 'GET'
url = '/jroulet/O2_samples/raw/master/GW150914.npy', body = None
headers = {'Accept': '*/*', 'Connection': 'close', 'Host': 'github.com', 'User-Agent': 'astropy'}
encode_chunked = False
def _send_request(self, method, url, body, headers, encode_chunked):
# Honor explicitly requested Host: and Accept-Encoding: headers.
header_names = frozenset(k.lower() for k in headers)
skips = {}
if 'host' in header_names:
skips['skip_host'] = 1
if 'accept-encoding' in header_names:
skips['skip_accept_encoding'] = 1
self.putrequest(method, url, **skips)
# chunked encoding will happen if HTTP/1.1 is used and either
# the caller passes encode_chunked=True or the following
# conditions hold:
# 1. content-length has not been explicitly set
# 2. the body is a file or iterable, but not a str or bytes-like
# 3. Transfer-Encoding has NOT been explicitly set by the caller
if 'content-length' not in header_names:
# only chunk body if not explicitly set for backwards
# compatibility, assuming the client code is already handling the
# chunking
if 'transfer-encoding' not in header_names:
# if content-length cannot be automatically determined, fall
# back to chunked encoding
encode_chunked = False
content_length = self._get_content_length(body, method)
if content_length is None:
if body is not None:
if self.debuglevel > 0:
print('Unable to determine size of %r' % body)
encode_chunked = True
self.putheader('Transfer-Encoding', 'chunked')
else:
self.putheader('Content-Length', str(content_length))
else:
encode_chunked = False
for hdr, value in headers.items():
self.putheader(hdr, value)
if isinstance(body, str):
# RFC 2616 Section 3.7.1 says that text default has a
# default charset of iso-8859-1.
body = _encode(body, 'body')
> self.endheaders(body, encode_chunked=encode_chunked)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/http/client.py:1331:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <http.client.HTTPSConnection object at 0x7f334b00be50>
message_body = None
def endheaders(self, message_body=None, *, encode_chunked=False):
"""Indicate that the last header line has been sent to the server.
This method sends the request to the server. The optional message_body
argument can be used to pass a message body associated with the
request.
"""
if self.__state == _CS_REQ_STARTED:
self.__state = _CS_REQ_SENT
else:
raise CannotSendHeader()
> self._send_output(message_body, encode_chunked=encode_chunked)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/http/client.py:1280:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <http.client.HTTPSConnection object at 0x7f334b00be50>
message_body = None, encode_chunked = False
def _send_output(self, message_body=None, encode_chunked=False):
"""Send the currently buffered request and clear the buffer.
Appends an extra \\r\\n to the buffer.
A message_body may be specified, to be appended to the request.
"""
self._buffer.extend((b"", b""))
msg = b"\r\n".join(self._buffer)
del self._buffer[:]
> self.send(msg)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/http/client.py:1040:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <http.client.HTTPSConnection object at 0x7f334b00be50>
data = b'GET /jroulet/O2_samples/raw/master/GW150914.npy HTTP/1.1\r\nAccept-Encoding: identity\r\nHost: github.com\r\nUser-Agent: astropy\r\nAccept: */*\r\nConnection: close\r\n\r\n'
def send(self, data):
"""Send `data' to the server.
``data`` can be a string object, a bytes object, an array object, a
file-like object that supports a .read() method, or an iterable object.
"""
if self.sock is None:
if self.auto_open:
> self.connect()
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/http/client.py:980:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <http.client.HTTPSConnection object at 0x7f334b00be50>
def connect(self):
"Connect to a host on a given (SSL) port."
> super().connect()
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/http/client.py:1447:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <http.client.HTTPSConnection object at 0x7f334b00be50>
def connect(self):
"""Connect to the host and port specified in __init__."""
> self.sock = self._create_connection(
(self.host,self.port), self.timeout, self.source_address)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/http/client.py:946:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
address = ('github.com', 443), timeout = 10.0, source_address = None
def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
source_address=None):
"""Connect to *address* and return the socket object.
Convenience function. Connect to *address* (a 2-tuple ``(host,
port)``) and return the socket object. Passing the optional
*timeout* parameter will set the timeout on the socket instance
before attempting to connect. If no *timeout* is supplied, the
global default timeout setting returned by :func:`getdefaulttimeout`
is used. If *source_address* is set it must be a tuple of (host, port)
for the socket to bind as a source address before making the connection.
A host of '' or port 0 tells the OS to use the default.
"""
host, port = address
err = None
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
af, socktype, proto, canonname, sa = res
sock = None
try:
sock = socket(af, socktype, proto)
if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
sock.settimeout(timeout)
if source_address:
sock.bind(source_address)
sock.connect(sa)
# Break explicitly a reference cycle
err = None
return sock
except error as _:
err = _
if sock is not None:
sock.close()
if err is not None:
try:
> raise err
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/socket.py:844:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
address = ('github.com', 443), timeout = 10.0, source_address = None
def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
source_address=None):
"""Connect to *address* and return the socket object.
Convenience function. Connect to *address* (a 2-tuple ``(host,
port)``) and return the socket object. Passing the optional
*timeout* parameter will set the timeout on the socket instance
before attempting to connect. If no *timeout* is supplied, the
global default timeout setting returned by :func:`getdefaulttimeout`
is used. If *source_address* is set it must be a tuple of (host, port)
for the socket to bind as a source address before making the connection.
A host of '' or port 0 tells the OS to use the default.
"""
host, port = address
err = None
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
af, socktype, proto, canonname, sa = res
sock = None
try:
sock = socket(af, socktype, proto)
if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
sock.settimeout(timeout)
if source_address:
sock.bind(source_address)
> sock.connect(sa)
E socket.timeout: timed out
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/socket.py:832: timeout
During handling of the above exception, another exception occurred:
self = <pesummary.tests.read_test.TestPublicPrincetonO1O2 object at 0x7f3355dd9e80>
def setup(self):
"""Setup the TestCoreBilbyFile class
"""
from pesummary.core.fetch import download_and_read_file
if not os.path.isdir(".outdir"):
os.mkdir(".outdir")
> self.file = download_and_read_file(
"https://github.com/jroulet/O2_samples/raw/master/GW150914.npy",
read_file=False, outdir=".outdir"
)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/site-packages/pesummary/tests/read_test.py:1728:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/site-packages/pesummary/core/fetch.py:142: in download_and_read_file
local = _function(url, **download_kwargs)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/site-packages/pesummary/core/fetch.py:114: in _download_file
return download_file(url, **kwargs)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/site-packages/astropy/utils/data.py:1394: in download_file
raise errors[sources[0]]
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/site-packages/astropy/utils/data.py:1357: in download_file
f_name = _download_file_from_source(
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/site-packages/astropy/utils/data.py:1161: in _download_file_from_source
with _try_url_open(source_url, timeout=timeout, http_headers=http_headers,
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/site-packages/astropy/utils/data.py:1091: in _try_url_open
return urlopener.open(req, timeout=timeout)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/urllib/request.py:517: in open
response = self._open(req, data)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/urllib/request.py:534: in _open
result = self._call_chain(self.handle_open, protocol, protocol +
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/urllib/request.py:494: in _call_chain
result = func(*args)
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/urllib/request.py:1389: in https_open
return self.do_open(http.client.HTTPSConnection, req,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPSHandler object at 0x7f334e071910>
http_class = <class 'http.client.HTTPSConnection'>
req = <urllib.request.Request object at 0x7f334e07ca60>
http_conn_args = {'check_hostname': None, 'context': <ssl.SSLContext object at 0x7f334dabcf40>}
host = 'github.com', h = <http.client.HTTPSConnection object at 0x7f334b00be50>
def do_open(self, http_class, req, **http_conn_args):
"""Return an HTTPResponse object for the request, using http_class.
http_class must implement the HTTPConnection API from http.client.
"""
host = req.host
if not host:
raise URLError('no host given')
# will parse host:port
h = http_class(host, timeout=req.timeout, **http_conn_args)
h.set_debuglevel(self._debuglevel)
headers = dict(req.unredirected_hdrs)
headers.update({k: v for k, v in req.headers.items()
if k not in headers})
# TODO(jhylton): Should this be redesigned to handle
# persistent connections?
# We want to make an HTTP/1.1 request, but the addinfourl
# class isn't prepared to deal with a persistent connection.
# It will try to read all remaining data from the socket,
# which will block while the server waits for the next request.
# So make sure the connection gets closed after the (only)
# request.
headers["Connection"] = "close"
headers = {name.title(): val for name, val in headers.items()}
if req._tunnel_host:
tunnel_headers = {}
proxy_auth_hdr = "Proxy-Authorization"
if proxy_auth_hdr in headers:
tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
# Proxy-Authorization should not be sent to origin
# server.
del headers[proxy_auth_hdr]
h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
try:
try:
h.request(req.get_method(), req.selector, req.data, headers,
encode_chunked=req.has_header('Transfer-encoding'))
except OSError as err: # timeout error
> raise URLError(err)
E urllib.error.URLError: <urlopen error timed out>
/builds/computing/conda/envs/igwn-py39-proposed/lib/python3.9/urllib/request.py:1349: URLError