From a02fa70d2ae2eedf560fe551669e1477d9fc0102 Mon Sep 17 00:00:00 2001
From: "duncan.macleod" <duncan.macleod@ligo.org>
Date: Mon, 6 Nov 2023 20:56:13 -0800
Subject: [PATCH] ui: support dynamic API selection
new `api` keyword argument to all UI functions, new `-A/--api` command-line option
---
gwdatafind/__main__.py | 35 +++++++---
gwdatafind/api/__init__.py | 19 +++++-
gwdatafind/tests/test_main.py | 14 ++++
gwdatafind/tests/test_ui.py | 71 +++++++++++++-------
gwdatafind/ui.py | 119 +++++++++++++++++++++++++++++-----
5 files changed, 205 insertions(+), 53 deletions(-)
diff --git a/gwdatafind/__main__.py b/gwdatafind/__main__.py
index 486a455..0c6d47e 100644
--- a/gwdatafind/__main__.py
+++ b/gwdatafind/__main__.py
@@ -27,6 +27,7 @@ from . import (
__version__,
ui,
)
+from .api import DEFAULT_API
from .io import (
format_cache,
lal_cache,
@@ -206,7 +207,7 @@ def command_line():
"-x",
"--extension",
metavar="EXT",
- default="gwf",
+ default=ui.DEFAULT_EXT,
help="file extension for which to search",
)
@@ -228,6 +229,12 @@ def command_line():
action="store_true",
help="attempt to authenticate without a grid proxy",
)
+ sargs.add_argument(
+ "-A",
+ "--api",
+ default=DEFAULT_API,
+ help="API version to use",
+ )
oargs = parser.add_argument_group(
"Output options",
@@ -319,7 +326,11 @@ def ping(args, out):
exitcode : `int` or `None`
the return value of the action or `None` to indicate success.
"""
- ui.ping(host=args.server, ext=args.extension)
+ ui.ping(
+ ext=args.extension,
+ host=args.server,
+ api=args.api,
+ )
print(f"LDRDataFindServer at {args.server} is alive", file=out)
@@ -340,9 +351,10 @@ def show_observatories(args, out):
the return value of the action or `None` to indicate success.
"""
sitelist = ui.find_observatories(
- host=args.server,
- match=args.match,
ext=args.extension,
+ match=args.match,
+ host=args.server,
+ api=args.api,
)
print("\n".join(sitelist), file=out)
@@ -365,9 +377,10 @@ def show_types(args, out):
"""
typelist = ui.find_types(
site=args.observatory,
+ ext=args.extension,
match=args.match,
host=args.server,
- ext=args.extension,
+ api=args.api,
)
print("\n".join(typelist), file=out)
@@ -393,8 +406,9 @@ def show_times(args, out):
frametype=args.type,
gpsstart=args.gpsstart,
gpsend=args.gpsend,
- host=args.server,
ext=args.extension,
+ host=args.server,
+ api=args.api,
)
print("# seg\tstart \tstop \tduration", file=out)
for i, seg in enumerate(seglist):
@@ -423,10 +437,11 @@ def latest(args, out):
cache = ui.find_latest(
args.observatory,
args.type,
+ ext=args.extension,
urltype=args.url_type,
on_missing="warn",
host=args.server,
- ext=args.extension,
+ api=args.api,
)
return postprocess_cache(cache, args, out)
@@ -452,6 +467,7 @@ def filename(args, out):
urltype=args.url_type,
on_missing="warn",
host=args.server,
+ api=args.api,
)
return postprocess_cache(cache, args, out)
@@ -477,11 +493,12 @@ def show_urls(args, out):
args.type,
args.gpsstart,
args.gpsend,
+ ext=args.extension,
match=args.match,
urltype=args.url_type,
- host=args.server,
on_gaps="ignore",
- ext=args.extension,
+ host=args.server,
+ api=args.api,
)
return postprocess_cache(cache, args, out)
diff --git a/gwdatafind/api/__init__.py b/gwdatafind/api/__init__.py
index 0dbab9e..cf624d3 100644
--- a/gwdatafind/api/__init__.py
+++ b/gwdatafind/api/__init__.py
@@ -15,10 +15,27 @@
# You should have received a copy of the GNU General Public License
# along with GWDataFind. If not, see <http://www.gnu.org/licenses/>.
-"""API definitions for the GWDataFind Client."""
+"""API definitions for the GWDataFind Client.
+
+Each API module must define the same set of functions that
+return paths to be queried on the host to return the various
+endpoints.
+
+The required functions and signatures are:
+
+- ``ping_path()``
+- ``find_observatories_path()``
+- ``find_types_path(site=None)``
+- ``find_times_path(site, frametype, start, end)``
+- ``find_url_path(framefile)``
+- ``find_latest_path(site, frametype, urltype)``
+- ``find_urls_path(site, frametype, start, end, urltype=None, match=None)``
+"""
# list of APIs
APIS = (
"ldr",
"v1",
)
+
+DEFAULT_API = "ldr"
diff --git a/gwdatafind/tests/test_main.py b/gwdatafind/tests/test_main.py
index 7d2caa1..60e7b0b 100644
--- a/gwdatafind/tests/test_main.py
+++ b/gwdatafind/tests/test_main.py
@@ -119,12 +119,14 @@ def test_sanity_check_fail(clargs):
def test_ping(mping):
args = argparse.Namespace(
server="test.datafind.com:443",
+ api="v1",
extension="gwf",
)
out = StringIO()
main.ping(args, out)
mping.assert_called_with(
host=args.server,
+ api=args.api,
ext=args.extension,
)
out.seek(0)
@@ -137,6 +139,7 @@ def test_show_observatories(mfindobs):
mfindobs.return_value = ["A", "B", "C"]
args = argparse.Namespace(
server="test.datafind.com:443",
+ api="v1",
extension="gwf",
match="test",
)
@@ -145,6 +148,7 @@ def test_show_observatories(mfindobs):
out.seek(0)
mfindobs.assert_called_with(
host=args.server,
+ api=args.api,
match=args.match,
ext=args.extension,
)
@@ -156,6 +160,7 @@ def test_show_types(mfindtypes):
mfindtypes.return_value = ["A", "B", "C"]
args = argparse.Namespace(
server="test.datafind.com:443",
+ api="v1",
extension="gwf",
observatory="X",
match="test",
@@ -165,6 +170,7 @@ def test_show_types(mfindtypes):
out.seek(0)
mfindtypes.assert_called_with(
host=args.server,
+ api=args.api,
match=args.match,
site=args.observatory,
ext=args.extension,
@@ -177,6 +183,7 @@ def test_show_times(mfindtimes):
mfindtimes.return_value = [segment(0, 1), segment(1, 2), segment(3, 4)]
args = argparse.Namespace(
server="test.datafind.com:443",
+ api="v1",
extension="gwf",
observatory="X",
type="test",
@@ -187,6 +194,7 @@ def test_show_times(mfindtimes):
main.show_times(args, out)
mfindtimes.assert_called_with(
host=args.server,
+ api=args.api,
site=args.observatory,
frametype=args.type,
gpsstart=args.gpsstart,
@@ -204,6 +212,7 @@ def test_latest(mlatest):
mlatest.return_value = ["file:///test/X-test-0-10.gwf"]
args = argparse.Namespace(
server="test.datafind.com:443",
+ api="v1",
extension="gwf",
observatory="X",
type="test",
@@ -219,6 +228,7 @@ def test_latest(mlatest):
urltype=args.url_type,
on_missing="warn",
host=args.server,
+ api=args.api,
ext=args.extension,
)
out.seek(0)
@@ -230,6 +240,7 @@ def test_filename(mfindurl):
mfindurl.return_value = ["file:///test/X-test-0-10.gwf"]
args = argparse.Namespace(
server="test.datafind.com:443",
+ api="v1",
filename="X-test-0-10.gwf",
url_type="file",
type=None,
@@ -243,6 +254,7 @@ def test_filename(mfindurl):
urltype=args.url_type,
on_missing="warn",
host=args.server,
+ api=args.api,
)
out.seek(0)
assert out.read().rstrip() == mfindurl.return_value[0]
@@ -258,6 +270,7 @@ def test_show_urls(mfindurls, ext):
mfindurls.return_value = urls
args = argparse.Namespace(
server="test.datafind.com:443",
+ api="v1",
extension=ext,
observatory="X",
type="test",
@@ -280,6 +293,7 @@ def test_show_urls(mfindurls, ext):
on_gaps="ignore",
ext=ext,
host=args.server,
+ api=args.api,
)
out.seek(0)
assert list(map(str.rstrip, out.readlines())) == urls
diff --git a/gwdatafind/tests/test_ui.py b/gwdatafind/tests/test_ui.py
index eb096ca..1b7e86d 100644
--- a/gwdatafind/tests/test_ui.py
+++ b/gwdatafind/tests/test_ui.py
@@ -23,12 +23,13 @@ import igwn_segments as segments
import pytest
from .. import ui
-from ..api import ldr as api_ldr
+from ..api import v1 as api
from . import yield_fixture
__author__ = "Duncan Macleod <duncan.macleod@ligo.org>"
TEST_SERVER = "test.datafind.org"
+TEST_API = "v1"
TEST_URL_BASE = f"https://{TEST_SERVER}"
TEST_DATA = {
"A": {
@@ -44,6 +45,10 @@ TEST_DATA = {
},
}
+# partials with the TEST_API set by default:
+find_url = partial(ui.find_url, api=TEST_API)
+find_urls = partial(ui.find_urls, api=TEST_API)
+
def _url(suffix):
return f"{TEST_URL_BASE}/{suffix}"
@@ -83,13 +88,15 @@ def noauth():
# default host
(None, TEST_URL_BASE),
))
+# mock out the API function lookup (we are testing URL formatting, not APIs)
+@mock.patch("gwdatafind.ui._api_func", lambda x, y: lambda: "test")
def test_url_scheme_handling(in_, url):
- assert ui._url(in_, lambda: "test") == f"{url}/test"
+ assert ui._url(in_, "api", "funcname") == f"{url}/test"
def test_ping(requests_mock):
- requests_mock.get(_url(api_ldr.ping_path()), status_code=200)
- ui.ping()
+ requests_mock.get(_url(api.ping_path()), status_code=200)
+ ui.ping(api=TEST_API)
@pytest.mark.parametrize(("match", "result"), (
@@ -98,10 +105,13 @@ def test_ping(requests_mock):
))
def test_find_observatories(match, result, requests_mock):
requests_mock.get(
- _url(api_ldr.find_observatories_path()),
+ _url(api.find_observatories_path()),
json=list(TEST_DATA),
)
- assert ui.find_observatories(match=match) == list(set(result))
+ assert ui.find_observatories(
+ api=TEST_API,
+ match=match,
+ ) == list(set(result))
@pytest.mark.parametrize(("site", "match", "result"), (
@@ -115,10 +125,11 @@ def test_find_types(site, match, result, requests_mock):
else:
respdata = [ft for site in TEST_DATA for ft in TEST_DATA[site]]
requests_mock.get(
- _url(api_ldr.find_types_path(site=site)),
+ _url(api.find_types_path(site=site)),
json=respdata,
)
assert ui.find_types(
+ api=TEST_API,
site=site,
match=match,
) == list(set(result))
@@ -128,10 +139,16 @@ def test_find_times(requests_mock):
site = "A"
frametype = "A1_TEST"
requests_mock.get(
- _url(api_ldr.find_times_path(site, frametype, 1, 100)),
+ _url(api.find_times_path(site, frametype, 1, 100)),
json=TEST_DATA[site][frametype],
)
- assert ui.find_times(site, frametype, 1, 100) == segments.segmentlist([
+ assert ui.find_times(
+ site,
+ frametype,
+ 1,
+ 100,
+ api=TEST_API,
+ ) == segments.segmentlist([
segments.segment(0, 10),
segments.segment(10, 20),
segments.segment(30, 50),
@@ -144,12 +161,12 @@ def test_find_url(requests_mock):
"gsiftp://localhost:2811/data/A/A1_TEST/A-A1_TEST-0-1.gwf",
]
requests_mock.get(
- _url(api_ldr.find_url_path("A-A1_TEST-0-1.gwf")),
+ _url(api.find_url_path("A-A1_TEST-0-1.gwf")),
json=urls,
)
- assert ui.find_url("/my/data/A-A1_TEST-0-1.gwf") == urls[:1]
- assert ui.find_url("/my/data/A-A1_TEST-0-1.gwf", urltype=None) == urls
- assert ui.find_url(
+ assert find_url("/my/data/A-A1_TEST-0-1.gwf") == urls[:1]
+ assert find_url("/my/data/A-A1_TEST-0-1.gwf", urltype=None) == urls
+ assert find_url(
"/my/data/A-A1_TEST-0-1.gwf",
urltype="gsiftp",
) == urls[1:]
@@ -157,22 +174,22 @@ def test_find_url(requests_mock):
def test_find_url_on_missing(requests_mock):
requests_mock.get(
- _url(api_ldr.find_url_path("A-A1_TEST-0-1.gwf")),
+ _url(api.find_url_path("A-A1_TEST-0-1.gwf")),
json=[],
)
# on_missing="ignore"
with warnings.catch_warnings():
warnings.simplefilter("error")
- assert ui.find_url("A-A1_TEST-0-1.gwf", on_missing="ignore") == []
+ assert find_url("A-A1_TEST-0-1.gwf", on_missing="ignore") == []
# on_missing="warn"
with pytest.warns(UserWarning):
- assert ui.find_url("A-A1_TEST-0-1.gwf", on_missing="warn") == []
+ assert find_url("A-A1_TEST-0-1.gwf", on_missing="warn") == []
# on_missing="error"
with pytest.raises(RuntimeError):
- ui.find_url("A-A1_TEST-0-1.gwf", on_missing="error")
+ find_url("A-A1_TEST-0-1.gwf", on_missing="error")
def test_find_latest(requests_mock):
@@ -183,10 +200,14 @@ def test_find_latest(requests_mock):
"gsiftp://localhost:2811/data/A/A1_TEST/A-A1_TEST-0-1.gwf",
]
requests_mock.get(
- _url(api_ldr.find_latest_path("A", "A1_TEST", "file")),
+ _url(api.find_latest_path("A", "A1_TEST", "file")),
json=urls[:1],
)
- assert ui.find_latest("A", "A1_TEST") == urls[:1]
+ assert ui.find_latest(
+ "A",
+ "A1_TEST",
+ api=TEST_API,
+ ) == urls[:1]
def _file_url(seg):
@@ -196,28 +217,28 @@ def _file_url(seg):
def test_find_urls(requests_mock):
urls = list(map(_file_url, TEST_DATA["A"]["A1_TEST"][:2]))
requests_mock.get(
- _url(api_ldr.find_urls_path("A", "A1_TEST", 0, 20, "file")),
+ _url(api.find_urls_path("A", "A1_TEST", 0, 20, "file")),
json=urls,
)
- assert ui.find_urls("A", "A1_TEST", 0, 20, on_gaps="error") == urls
+ assert find_urls("A", "A1_TEST", 0, 20, on_gaps="error") == urls
def test_find_urls_on_gaps(requests_mock):
urls = list(map(_file_url, TEST_DATA["A"]["A1_TEST"]))
requests_mock.get(
- _url(api_ldr.find_urls_path("A", "A1_TEST", 0, 100, "file")),
+ _url(api.find_urls_path("A", "A1_TEST", 0, 100, "file")),
json=urls,
)
# on_gaps="ignore"
with warnings.catch_warnings():
warnings.simplefilter("error")
- assert ui.find_urls("A", "A1_TEST", 0, 100, on_gaps="ignore") == urls
+ assert find_urls("A", "A1_TEST", 0, 100, on_gaps="ignore") == urls
# on_missing="warn"
with pytest.warns(UserWarning):
- assert ui.find_urls("A", "A1_TEST", 0, 100, on_gaps="warn") == urls
+ assert find_urls("A", "A1_TEST", 0, 100, on_gaps="warn") == urls
# on_missing="error"
with pytest.raises(RuntimeError):
- ui.find_urls("A", "A1_TEST", 0, 100, on_gaps="error")
+ find_urls("A", "A1_TEST", 0, 100, on_gaps="error")
diff --git a/gwdatafind/ui.py b/gwdatafind/ui.py
index 4eec86e..6b1cc5d 100644
--- a/gwdatafind/ui.py
+++ b/gwdatafind/ui.py
@@ -23,6 +23,7 @@ referred to in usage as ``gwdatafind.<function>`` and not
"""
from functools import wraps
+from importlib import import_module
from re import compile as compile_regex
from urllib.parse import urlparse
from warnings import warn
@@ -31,12 +32,25 @@ import igwn_segments as segments
import requests
from igwn_auth_utils.requests import get as _get
-from .api import ldr as api_ldr
+from .api import DEFAULT_API
from .utils import (
file_segment,
get_default_host,
)
+try:
+ from functools import cache
+except ImportError: # python < 3.9
+ from functools import (
+ lru_cache,
+ partial,
+ )
+
+ @wraps(lru_cache)
+ def cache(func):
+ """Wrapper around `functools.lru_cache().`"""
+ return lru_cache(maxsize=None)(func)
+
__author__ = "Duncan Macleod <duncan.macleod@ligo.org>"
__all__ = [
@@ -49,6 +63,29 @@ __all__ = [
"find_urls",
]
+DEFAULT_EXT = "gwf"
+
+
+# -- api handling --------------------
+
+def _api_mod(api):
+ """Return the API implementation module with the module name ``api``.
+ """
+ api = (api or DEFAULT_API).lower()
+ try:
+ return import_module(f".api.{api}", package=__package__)
+ except ImportError:
+ raise ValueError(f"unsupported api '{api}'")
+
+
+@cache
+def _api_func(api, name):
+ """Return the function with ``name`` for the matching ``api``.
+ """
+ return getattr(_api_mod(api), name)
+
+
+# -- user interface ------------------
@wraps(_get)
def get(url, *args, **kwargs):
@@ -90,17 +127,26 @@ def get_json(*args, **kwargs):
return response.json()
-def _url(host, api_func, *args, **kwargs):
+def _url(
+ host,
+ api,
+ funcname,
+ *args,
+ **kwargs,
+):
"""Construct the full URL for a query to ``host`` using an API function.
Parameters
----------
host : `str`, `None`
- the host to query, if `None` `~gwdatafind.utils.get_default_host()`
+ The host to query, if `None` `~gwdatafind.utils.get_default_host()`
will be used to discover the default host.
- api_func : `callable`
- the function from the :mod:`gwdatafind.api` module to use in
+ api : `str`, optional
+ The API version to use.
+
+ funcname: `str`
+ The name of the function from the API implementation to use in
constructing the URL path.
*args, **kwargs
@@ -112,6 +158,7 @@ def _url(host, api_func, *args, **kwargs):
url : `str`
a full URL including scheme, host, and path
"""
+ api_func = _api_func(api, funcname)
path = api_func(*args, **kwargs)
if host is None:
host = get_default_host()
@@ -127,7 +174,13 @@ def _url(host, api_func, *args, **kwargs):
return f"{host.rstrip('/')}/{path}"
-def ping(host=None, ext=api_ldr.DEFAULT_EXT, session=None, **request_kw):
+def ping(
+ host=None,
+ api=DEFAULT_API,
+ ext=DEFAULT_EXT,
+ session=None,
+ **request_kw,
+):
"""Ping the GWDataFind host to test for life.
Parameters
@@ -137,6 +190,9 @@ def ping(host=None, ext=api_ldr.DEFAULT_EXT, session=None, **request_kw):
:func:`~gwdatafind.utils.get_default_host` will be used to
discover the default host.
+ api : `str`, optional
+ The API version to use.
+
ext : `str`, optional
the file extension for which to search.
@@ -161,7 +217,7 @@ def ping(host=None, ext=api_ldr.DEFAULT_EXT, session=None, **request_kw):
requests.RequestException
if the request fails for any reason
"""
- qurl = _url(host, api_ldr.ping_path, ext=ext)
+ qurl = _url(host, api, "ping_path")
response = get(qurl, session=session, **request_kw)
response.raise_for_status()
@@ -169,7 +225,8 @@ def ping(host=None, ext=api_ldr.DEFAULT_EXT, session=None, **request_kw):
def find_observatories(
match=None,
host=None,
- ext=api_ldr.DEFAULT_EXT,
+ api=DEFAULT_API,
+ ext=DEFAULT_EXT,
session=None,
**request_kw,
):
@@ -186,6 +243,9 @@ def find_observatories(
:func:`~gwdatafind.utils.get_default_host` will be used to
discover the default host.
+ api : `str`, optional
+ The API version to use.
+
ext : `str`, optional
the file extension for which to search.
@@ -222,7 +282,7 @@ def find_observatories(
>>> find_observatories(match="H", host="datafind.gwosc.org")
['H']
"""
- qurl = _url(host, api_ldr.find_observatories_path, ext=ext)
+ qurl = _url(host, api, "find_observatories_path", ext=ext)
sites = set(get_json(qurl, session=session, **request_kw))
if match:
match = compile_regex(match).search
@@ -234,7 +294,8 @@ def find_types(
site=None,
match=None,
host=None,
- ext=api_ldr.DEFAULT_EXT,
+ api=DEFAULT_API,
+ ext=DEFAULT_EXT,
session=None,
**request_kw,
):
@@ -254,6 +315,9 @@ def find_types(
:func:`~gwdatafind.utils.get_default_host` will be used to
discover the default host.
+ api : `str`, optional
+ The API version to use.
+
ext : `str`, optional
the file extension for which to search.
@@ -292,7 +356,7 @@ def find_types(
(accurate as of Nov 18 2021)
""" # noqa: E501
- qurl = _url(host, api_ldr.find_types_path, site=site, ext=ext)
+ qurl = _url(host, api, "find_types_path", site=site, ext=ext)
types = set(get_json(qurl, session=session, **request_kw))
if match:
match = compile_regex(match).search
@@ -306,7 +370,8 @@ def find_times(
gpsstart=None,
gpsend=None,
host=None,
- ext=api_ldr.DEFAULT_EXT,
+ api=DEFAULT_API,
+ ext=DEFAULT_EXT,
session=None,
**request_kw,
):
@@ -334,6 +399,9 @@ def find_times(
:func:`~gwdatafind.utils.get_default_host` will be used to
discover the default host.
+ api : `str`, optional
+ The API version to use.
+
ext : `str`, optional
the file extension for which to search.
@@ -377,7 +445,8 @@ def find_times(
""" # noqa: E501
qurl = _url(
host,
- api_ldr.find_times_path,
+ api,
+ "find_times_path",
site,
frametype,
gpsstart,
@@ -410,6 +479,7 @@ def find_url(
urltype="file",
on_missing="error",
host=None,
+ api=DEFAULT_API,
session=None,
**request_kw,
):
@@ -436,6 +506,9 @@ def find_url(
:func:`~gwdatafind.utils.get_default_host` will be used to
discover the default host.
+ api : `str`, optional
+ The API version to use.
+
session : `requests.Session`, optional
the connection session to use; if not given, a
:class:`igwn_auth_utils.requests.Session` will be
@@ -465,7 +538,7 @@ def find_url(
RuntimeError
if no matching URLs are found and ``on_missing="error"`` was given
"""
- qurl = _url(host, api_ldr.find_url_path, framefile)
+ qurl = _url(host, api, "find_url_path", framefile)
return _get_urls(
qurl,
scheme=urltype,
@@ -481,7 +554,8 @@ def find_latest(
urltype="file",
on_missing="error",
host=None,
- ext=api_ldr.DEFAULT_EXT,
+ api=DEFAULT_API,
+ ext=DEFAULT_EXT,
session=None,
**request_kw,
):
@@ -510,6 +584,9 @@ def find_latest(
:func:`~gwdatafind.utils.get_default_host` will be used to
discover the default host.
+ api : `str`, optional
+ The API version to use.
+
ext : `str`, optional
the file extension for which to search.
@@ -549,7 +626,8 @@ def find_latest(
""" # noqa: E501
qurl = _url(
host,
- api_ldr.find_latest_path,
+ api,
+ "find_latest_path",
site,
frametype,
ext=ext,
@@ -567,7 +645,8 @@ def find_urls(
urltype="file",
on_gaps="warn",
host=None,
- ext=api_ldr.DEFAULT_EXT,
+ api=DEFAULT_API,
+ ext=DEFAULT_EXT,
session=None,
**request_kw,
):
@@ -606,6 +685,9 @@ def find_urls(
:func:`~gwdatafind.utils.get_default_host` will be used to
discover the default host.
+ api : `str`, optional
+ The API version to use.
+
ext : `str`, optional
the file extension for which to search.
@@ -635,7 +717,8 @@ def find_urls(
"""
qurl = _url(
host,
- api_ldr.find_urls_path,
+ api,
+ "find_urls_path",
site,
frametype,
gpsstart,
--
GitLab