...
 
Commits (3)
......@@ -101,10 +101,6 @@ build:debian:buster:
reports:
junit: junit.xml
test:python3.5:
<<: *test
image: python:3.5
test:python3.6:
<<: *test
image: python:3.6
......
......@@ -43,11 +43,11 @@ class DataFlask(Flask):
def get_cache_data(self, *keys):
while not self.manager.ready:
# cache file is not ready
self.logger.debug("Waiting for cache...")
time.sleep(.5)
self.manager.lock.acquire()
try:
print(self.manager.cache)
return self._get_cache_data(keys)
finally:
self.manager.lock.release()
......@@ -59,10 +59,7 @@ class DataFlask(Flask):
return self.manager.cache.get(last, {})
ent = self.manager.cache.get(keys.pop(0), {})
for key in keys:
try:
ent = ent[key]
except KeyError:
return {}
ent = ent[key]
return ent.get(last, {})
def shutdown(self):
......
......@@ -14,7 +14,7 @@ from functools import (reduce, wraps)
from math import inf as INF
from urllib.parse import urlparse
from flask import (Blueprint, current_app, jsonify)
from flask import (Blueprint, current_app, jsonify, request)
from ligo.segments import (segmentlist, segment)
......@@ -71,7 +71,7 @@ def find_types(ext, site):
"""
ecache = current_app.manager.cache.get(ext, {})
if site == 'all':
sites = ecache.keys()
sites = list(ecache.keys())
else:
sites = [site]
return [tag for site in sites for
......@@ -114,7 +114,14 @@ def find_url(ext, site, tag, filename):
# parse GPS information from filename
_, _, start, dur = filename.split('-')
dur = dur[:-len(ext)].rstrip('.')
return list(_find_urls(ext, site, tag, int(start), int(start + dur)))
# find urls
return list(_find_urls(
ext,
site,
tag,
int(start),
int(start + dur),
))
@blueprint.route('<ext>/<site>/<tag>/<int:start>,<int:end>.json')
......@@ -123,7 +130,15 @@ def find_url(ext, site, tag, filename):
def find_urls(ext, site, tag, start, end, urltype=None):
"""Find all URLs in a given GPS time interval
"""
return list(_find_urls(ext, site, tag, start, end, urltype=urltype))
return list(_find_urls(
ext,
site,
tag,
start,
end,
urltype=urltype,
**request.args,
))
@blueprint.route('<ext>/<site>/<tag>/latest.json')
......@@ -132,8 +147,15 @@ def find_urls(ext, site, tag, start, end, urltype=None):
def find_latest_url(ext, site, tag, urltype=None):
"""Find the latest URL(s) for a given tag
"""
return list(_find_urls(ext, site, tag, 0, INF,
urltype=urltype, latest=True))
return list(_find_urls(
ext,
site,
tag,
0,
INF,
urltype=urltype,
latest=True,
))
# -- URL matcher --------------------------------------------------------------
......@@ -162,11 +184,15 @@ def _find_urls(ext, site, tag, start, end, urltype=None, match=None,
# if running a 'latest' URL search, restrict the search to
# the most recent available segment for this frametype
if latest and seglist: # 'if seglist' to prevent IndexError
# get latest segment for this path
latest = _get_latest_segment(seglist, cdur)
if latest[1] <= maxgps: # if this is not an improvement, move on
continue
# only keep the segment of the last file
maxgps = latest[1]
seglist = [latest]
# empty matches to keep only this one
lfns = defaultdict(list)
# loop over segments and construct file URLs
for seg in seglist:
......@@ -224,14 +250,11 @@ def _filter_preference(urls):
# parse filter preference as a dict of regex keys
# each with a list of regexs as value
filter_preference = {
re.compile(key):
list(map(re.compile, value)) for (key, value) in
json.loads(
current_app.config.get(
'filter_preference',
'{}',
).replace('\'', '"'),
)
re.compile(key): list(map(re.compile, value)) for (key, value) in
json.loads(current_app.config["LDRDataFindServer"].get(
'filter_preference',
'{}',
).replace('\'', '"')).items()
}
matches = defaultdict(list)
......@@ -248,8 +271,6 @@ def _filter_preference(urls):
keep = []
for pattern, murls in matches.items():
if not murls: # no match, don't care
continue
if len(murls) == 1: # one match, just keep
keep.extend(murls)
continue
......@@ -285,4 +306,4 @@ def _rank_select_urls(urls, patterns):
for url in urls:
if pattern.search(url):
return [url]
return urls
return urls # we should never get to here
......@@ -49,7 +49,6 @@ setup(
"Operating System :: Unix",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Topic :: Scientific/Engineering",
......
/test/path,H,H1_TEST_1,1,8,gwf 1556913881 1 {1000000000 1000000008}
/test/path,H,H1_TEST_1,1,8,h5 1556913881 1 {1000000000 1000000008}
/test/path,L,L1_TEST_1,1,4,gwf 1556913881 4 {1000000000 1000000008 1000000012 1000000020}
/test/preferred/path,H,H1_TEST_1,1,8,h5 1556913881 1 {1000000000 1000000008}
/test/other/path,H,H1_TEST_1,1,8,h5 1556913881 1 {1000000000 1000000008}
/test/path2,L,L1_TEST_1,1,4,gwf 1556913881 2 {1000000012 1000000020}
/test/path,L,L1_TEST_1,1,4,gwf 1556913881 2 {1000000000 1000000008}
/test/path,L,L1_TEST_1,1,4,h5 1556913881 2 {1000000000 1000000008}
/test/path,L,L1_TEST_2,1,8,gwf 1556913881 1 {1000000000 1000000008}
/test/path,L,L1_TEST_IGNORE,1,8,gwf 1556913881 1 {1000000000 1000000008}
/test/path,X,X1_TEST_2,1,8,gwf 1556913881 1 {1000000000 1000000008}
/test/path,Y,Y1_EXCLUDE_1,1,8,gwf 1556913881 1 {1000000000 1000000008}
[LDRDataFindServer]
framecachetimeout = 10
framecachefile = tests/cache.dat
site_exclude_pattern = ^X$
frametype_exclude_pattern = .+EXCLUDE.+
frametype_include_pattern = .+_TEST_\d+
filter_preference = """{'^file': ['preferred', 'hdfs']}"""
......@@ -2,40 +2,79 @@
# Copyright (2019) Duncan Macleod
# Licensed under GPLv3+ - see LICENSE
import os
from pathlib import Path
from unittest import mock
import pytest
from gwdatafind_server import views
from gwdatafind_server import (
config,
views,
)
# -- test the app ---------------------
# -- app ------------------------------
def test_shutdown(app):
"""Test that `app.shutdown()` kills the cache thread
"""
assert app.manager.is_alive()
app.shutdown()
assert not app.manager.is_alive()
# -- test the view --------------------
# -- config ---------------------------
@mock.patch.dict(
os.environ,
{"LDR_LOCATION": str(Path(__file__).parent)},
)
def test_get_config_path():
"""Test that get_config_path() can resolve a file
"""
assert config.get_config_path() == str(
Path(os.environ["LDR_LOCATION"]) / "gwdatafind-server.ini"
)
@mock.patch.dict(os.environ, {'LDR_LOCATION': "/missing/path"})
def test_get_config_path_error():
"""Test that get_config_path() errors when it can's resolve a file
"""
with pytest.raises(ValueError):
config.get_config_path()
# -- views ----------------------------
def test_find_observatories(client):
"""Test the `find_observatories` view
"""
resp = client.get("/LDR/services/data/v1/gwf.json")
assert resp.status_code == 200
assert sorted(resp.json) == ["H", "L"]
def test_find_types(client):
resp = client.get("/LDR/services/data/v1/gwf/L.json")
@pytest.mark.parametrize('type_, types', [
('L', ["L1_TEST_1", "L1_TEST_2"]),
('all', ["H1_TEST_1", "L1_TEST_1", "L1_TEST_2"]),
])
def test_find_types(client, type_, types):
"""Test the `find_types` view
"""
resp = client.get("/LDR/services/data/v1/gwf/{}.json".format(type_))
assert resp.status_code == 200
assert sorted(resp.json) == ["L1_TEST_1", "L1_TEST_2"]
assert sorted(resp.json) == sorted(types)
@pytest.mark.parametrize('ext, segs', [
('gwf', [[1000000000, 1000000008], [1000000012, 1000000020]]),
('h5', [[1000000000, 1000000008]]),
])
def test_find_all_times(client, ext, segs):
def test_find_times_all(client, ext, segs):
"""Test the `find_times` view without specifying limits
"""
resp = client.get(
"/LDR/services/data/v1/{}/L/L1_TEST_1/segments.json".format(ext),
)
......@@ -44,6 +83,8 @@ def test_find_all_times(client, ext, segs):
def test_find_times(client):
"""Test the `find_times` view with limits
"""
resp = client.get(
"/LDR/services/data/v1/gwf/L/L1_TEST_1/segments/"
"1000000007,1000000013.json",
......@@ -57,6 +98,8 @@ def test_find_times(client):
@mock.patch.object(views, "_DEFAULT_GSIFTP_HOST", new="testhost")
def test_find_url(client):
"""Test the `find_url` view
"""
resp = client.get(
"/LDR/services/data/v1"
"/h5/L/L1_TEST_1/L-L1_TEST_1-1000000004-4.h5.json",
......@@ -70,47 +113,67 @@ def test_find_url(client):
@mock.patch.object(views, "_DEFAULT_GSIFTP_HOST", new="testhost")
def test_find_urls(client):
"""Test the `find_urls` view with no special options
"""
resp = client.get(
"/LDR/services/data/v1/gwf/L/L1_TEST_1/1000000004,1000000016.json",
)
assert resp.status_code == 200
assert sorted(resp.json) == [
"file://localhost/test/path/L-L1_TEST_1-1000000004-4.gwf",
"file://localhost/test/path/L-L1_TEST_1-1000000012-4.gwf",
"file://localhost/test/path2/L-L1_TEST_1-1000000012-4.gwf",
"gsiftp://testhost:15000/test/path/L-L1_TEST_1-1000000004-4.gwf",
"gsiftp://testhost:15000/test/path/L-L1_TEST_1-1000000012-4.gwf",
"gsiftp://testhost:15000/test/path2/L-L1_TEST_1-1000000012-4.gwf",
]
def test_find_urls_urltype(client):
def test_find_urls_fancy(client):
"""Test the `find_urls` view with extra options
"""
resp = client.get(
"/LDR/services/data/v1"
"/gwf/L/L1_TEST_1/1000000004,1000000016/file.json",
"/gwf/L/L1_TEST_1/1000000000,1000000008/file.json?match=04",
)
assert resp.status_code == 200
assert sorted(resp.json) == [
"file://localhost/test/path/L-L1_TEST_1-1000000004-4.gwf",
"file://localhost/test/path/L-L1_TEST_1-1000000012-4.gwf",
]
def test_find_urls_filter_preference(client):
"""Test the `find_urls` view with `filter_preference`
"""
resp = client.get(
"/LDR/services/data/v1"
"/h5/H/H1_TEST_1/1000000000,1000000004/file.json",
)
assert resp.status_code == 200
assert sorted(resp.json) == [
"file://localhost/test/preferred/path/H-H1_TEST_1-1000000000-8.h5",
]
@mock.patch.object(views, "_DEFAULT_GSIFTP_HOST", new="testhost")
def test_find_latest(client):
"""Test the `find_latest` view
"""
resp = client.get(
"/LDR/services/data/v1/gwf/L/L1_TEST_1/latest.json",
)
assert resp.status_code == 200
assert resp.json == [
"file://localhost/test/path/L-L1_TEST_1-1000000016-4.gwf",
"gsiftp://testhost:15000/test/path/L-L1_TEST_1-1000000016-4.gwf",
"file://localhost/test/path2/L-L1_TEST_1-1000000016-4.gwf",
"gsiftp://testhost:15000/test/path2/L-L1_TEST_1-1000000016-4.gwf",
]
def test_find_latest_urltype(client):
"""Test the `find_latest` view with urltype
"""
resp = client.get(
"/LDR/services/data/v1/gwf/L/L1_TEST_1/latest/file.json",
)
assert resp.status_code == 200
assert resp.json == [
"file://localhost/test/path/L-L1_TEST_1-1000000016-4.gwf",
"file://localhost/test/path2/L-L1_TEST_1-1000000016-4.gwf",
]