Maintenance will be performed on git.ligo.org, chat.ligo.org, containers.ligo.org, and docs.ligo.org on Tuesday 26 May 2020 starting at approximately 10am CDT. It is expected to take around 30 minutes and will involve a short period of downtime, around 5 minutes, towards the end of the maintenance period. Please address any questions, comments, or concerns to uwm-help@cgca.uwm.edu.

Commit c2c07916 authored by Duncan Macleod's avatar Duncan Macleod

Merge branch 'more-tests' into 'master'

Improved tests and fixed a few bugs

See merge request duncanmmacleod/gwdatafind-server!1
parents 92323b9c cdd2b2ee
Pipeline #60828 passed with stages
in 2 minutes and 17 seconds
......@@ -101,10 +101,6 @@ build:debian:buster:
reports:
junit: junit.xml
test:python3.5:
<<: *test
image: python:3.5
test:python3.6:
<<: *test
image: python:3.6
......
......@@ -43,11 +43,11 @@ class DataFlask(Flask):
def get_cache_data(self, *keys):
while not self.manager.ready:
# cache file is not ready
self.logger.debug("Waiting for cache...")
time.sleep(.5)
self.manager.lock.acquire()
try:
print(self.manager.cache)
return self._get_cache_data(keys)
finally:
self.manager.lock.release()
......@@ -59,10 +59,7 @@ class DataFlask(Flask):
return self.manager.cache.get(last, {})
ent = self.manager.cache.get(keys.pop(0), {})
for key in keys:
try:
ent = ent[key]
except KeyError:
return {}
ent = ent[key]
return ent.get(last, {})
def shutdown(self):
......
......@@ -14,7 +14,7 @@ from functools import (reduce, wraps)
from math import inf as INF
from urllib.parse import urlparse
from flask import (Blueprint, current_app, jsonify)
from flask import (Blueprint, current_app, jsonify, request)
from ligo.segments import (segmentlist, segment)
......@@ -71,7 +71,7 @@ def find_types(ext, site):
"""
ecache = current_app.manager.cache.get(ext, {})
if site == 'all':
sites = ecache.keys()
sites = list(ecache.keys())
else:
sites = [site]
return [tag for site in sites for
......@@ -114,7 +114,14 @@ def find_url(ext, site, tag, filename):
# parse GPS information from filename
_, _, start, dur = filename.split('-')
dur = dur[:-len(ext)].rstrip('.')
return list(_find_urls(ext, site, tag, int(start), int(start + dur)))
# find urls
return list(_find_urls(
ext,
site,
tag,
int(start),
int(start + dur),
))
@blueprint.route('<ext>/<site>/<tag>/<int:start>,<int:end>.json')
......@@ -123,7 +130,15 @@ def find_url(ext, site, tag, filename):
def find_urls(ext, site, tag, start, end, urltype=None):
"""Find all URLs in a given GPS time interval
"""
return list(_find_urls(ext, site, tag, start, end, urltype=urltype))
return list(_find_urls(
ext,
site,
tag,
start,
end,
urltype=urltype,
**request.args,
))
@blueprint.route('<ext>/<site>/<tag>/latest.json')
......@@ -132,8 +147,15 @@ def find_urls(ext, site, tag, start, end, urltype=None):
def find_latest_url(ext, site, tag, urltype=None):
"""Find the latest URL(s) for a given tag
"""
return list(_find_urls(ext, site, tag, 0, INF,
urltype=urltype, latest=True))
return list(_find_urls(
ext,
site,
tag,
0,
INF,
urltype=urltype,
latest=True,
))
# -- URL matcher --------------------------------------------------------------
......@@ -162,11 +184,15 @@ def _find_urls(ext, site, tag, start, end, urltype=None, match=None,
# if running a 'latest' URL search, restrict the search to
# the most recent available segment for this frametype
if latest and seglist: # 'if seglist' to prevent IndexError
# get latest segment for this path
latest = _get_latest_segment(seglist, cdur)
if latest[1] <= maxgps: # if this is not an improvement, move on
continue
# only keep the segment of the last file
maxgps = latest[1]
seglist = [latest]
# empty matches to keep only this one
lfns = defaultdict(list)
# loop over segments and construct file URLs
for seg in seglist:
......@@ -224,14 +250,11 @@ def _filter_preference(urls):
# parse filter preference as a dict of regex keys
# each with a list of regexs as value
filter_preference = {
re.compile(key):
list(map(re.compile, value)) for (key, value) in
json.loads(
current_app.config.get(
'filter_preference',
'{}',
).replace('\'', '"'),
)
re.compile(key): list(map(re.compile, value)) for (key, value) in
json.loads(current_app.config["LDRDataFindServer"].get(
'filter_preference',
'{}',
).replace('\'', '"')).items()
}
matches = defaultdict(list)
......@@ -248,8 +271,6 @@ def _filter_preference(urls):
keep = []
for pattern, murls in matches.items():
if not murls: # no match, don't care
continue
if len(murls) == 1: # one match, just keep
keep.extend(murls)
continue
......@@ -285,4 +306,4 @@ def _rank_select_urls(urls, patterns):
for url in urls:
if pattern.search(url):
return [url]
return urls
return urls # we should never get to here
......@@ -49,7 +49,6 @@ setup(
"Operating System :: Unix",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Topic :: Scientific/Engineering",
......
/test/path,H,H1_TEST_1,1,8,gwf 1556913881 1 {1000000000 1000000008}
/test/path,H,H1_TEST_1,1,8,h5 1556913881 1 {1000000000 1000000008}
/test/path,L,L1_TEST_1,1,4,gwf 1556913881 4 {1000000000 1000000008 1000000012 1000000020}
/test/preferred/path,H,H1_TEST_1,1,8,h5 1556913881 1 {1000000000 1000000008}
/test/other/path,H,H1_TEST_1,1,8,h5 1556913881 1 {1000000000 1000000008}
/test/path2,L,L1_TEST_1,1,4,gwf 1556913881 2 {1000000012 1000000020}
/test/path,L,L1_TEST_1,1,4,gwf 1556913881 2 {1000000000 1000000008}
/test/path,L,L1_TEST_1,1,4,h5 1556913881 2 {1000000000 1000000008}
/test/path,L,L1_TEST_2,1,8,gwf 1556913881 1 {1000000000 1000000008}
/test/path,L,L1_TEST_IGNORE,1,8,gwf 1556913881 1 {1000000000 1000000008}
/test/path,X,X1_TEST_2,1,8,gwf 1556913881 1 {1000000000 1000000008}
/test/path,Y,Y1_EXCLUDE_1,1,8,gwf 1556913881 1 {1000000000 1000000008}
[LDRDataFindServer]
framecachetimeout = 10
framecachefile = tests/cache.dat
site_exclude_pattern = ^X$
frametype_exclude_pattern = .+EXCLUDE.+
frametype_include_pattern = .+_TEST_\d+
filter_preference = """{'^file': ['preferred', 'hdfs']}"""
......@@ -2,40 +2,79 @@
# Copyright (2019) Duncan Macleod
# Licensed under GPLv3+ - see LICENSE
import os
from pathlib import Path
from unittest import mock
import pytest
from gwdatafind_server import views
from gwdatafind_server import (
config,
views,
)
# -- test the app ---------------------
# -- app ------------------------------
def test_shutdown(app):
"""Test that `app.shutdown()` kills the cache thread
"""
assert app.manager.is_alive()
app.shutdown()
assert not app.manager.is_alive()
# -- test the view --------------------
# -- config ---------------------------
@mock.patch.dict(
os.environ,
{"LDR_LOCATION": str(Path(__file__).parent)},
)
def test_get_config_path():
"""Test that get_config_path() can resolve a file
"""
assert config.get_config_path() == str(
Path(os.environ["LDR_LOCATION"]) / "gwdatafind-server.ini"
)
@mock.patch.dict(os.environ, {'LDR_LOCATION': "/missing/path"})
def test_get_config_path_error():
"""Test that get_config_path() errors when it can's resolve a file
"""
with pytest.raises(ValueError):
config.get_config_path()
# -- views ----------------------------
def test_find_observatories(client):
"""Test the `find_observatories` view
"""
resp = client.get("/LDR/services/data/v1/gwf.json")
assert resp.status_code == 200
assert sorted(resp.json) == ["H", "L"]
def test_find_types(client):
resp = client.get("/LDR/services/data/v1/gwf/L.json")
@pytest.mark.parametrize('type_, types', [
('L', ["L1_TEST_1", "L1_TEST_2"]),
('all', ["H1_TEST_1", "L1_TEST_1", "L1_TEST_2"]),
])
def test_find_types(client, type_, types):
"""Test the `find_types` view
"""
resp = client.get("/LDR/services/data/v1/gwf/{}.json".format(type_))
assert resp.status_code == 200
assert sorted(resp.json) == ["L1_TEST_1", "L1_TEST_2"]
assert sorted(resp.json) == sorted(types)
@pytest.mark.parametrize('ext, segs', [
('gwf', [[1000000000, 1000000008], [1000000012, 1000000020]]),
('h5', [[1000000000, 1000000008]]),
])
def test_find_all_times(client, ext, segs):
def test_find_times_all(client, ext, segs):
"""Test the `find_times` view without specifying limits
"""
resp = client.get(
"/LDR/services/data/v1/{}/L/L1_TEST_1/segments.json".format(ext),
)
......@@ -44,6 +83,8 @@ def test_find_all_times(client, ext, segs):
def test_find_times(client):
"""Test the `find_times` view with limits
"""
resp = client.get(
"/LDR/services/data/v1/gwf/L/L1_TEST_1/segments/"
"1000000007,1000000013.json",
......@@ -57,6 +98,8 @@ def test_find_times(client):
@mock.patch.object(views, "_DEFAULT_GSIFTP_HOST", new="testhost")
def test_find_url(client):
"""Test the `find_url` view
"""
resp = client.get(
"/LDR/services/data/v1"
"/h5/L/L1_TEST_1/L-L1_TEST_1-1000000004-4.h5.json",
......@@ -70,47 +113,67 @@ def test_find_url(client):
@mock.patch.object(views, "_DEFAULT_GSIFTP_HOST", new="testhost")
def test_find_urls(client):
"""Test the `find_urls` view with no special options
"""
resp = client.get(
"/LDR/services/data/v1/gwf/L/L1_TEST_1/1000000004,1000000016.json",
)
assert resp.status_code == 200
assert sorted(resp.json) == [
"file://localhost/test/path/L-L1_TEST_1-1000000004-4.gwf",
"file://localhost/test/path/L-L1_TEST_1-1000000012-4.gwf",
"file://localhost/test/path2/L-L1_TEST_1-1000000012-4.gwf",
"gsiftp://testhost:15000/test/path/L-L1_TEST_1-1000000004-4.gwf",
"gsiftp://testhost:15000/test/path/L-L1_TEST_1-1000000012-4.gwf",
"gsiftp://testhost:15000/test/path2/L-L1_TEST_1-1000000012-4.gwf",
]
def test_find_urls_urltype(client):
def test_find_urls_fancy(client):
"""Test the `find_urls` view with extra options
"""
resp = client.get(
"/LDR/services/data/v1"
"/gwf/L/L1_TEST_1/1000000004,1000000016/file.json",
"/gwf/L/L1_TEST_1/1000000000,1000000008/file.json?match=04",
)
assert resp.status_code == 200
assert sorted(resp.json) == [
"file://localhost/test/path/L-L1_TEST_1-1000000004-4.gwf",
"file://localhost/test/path/L-L1_TEST_1-1000000012-4.gwf",
]
def test_find_urls_filter_preference(client):
"""Test the `find_urls` view with `filter_preference`
"""
resp = client.get(
"/LDR/services/data/v1"
"/h5/H/H1_TEST_1/1000000000,1000000004/file.json",
)
assert resp.status_code == 200
assert sorted(resp.json) == [
"file://localhost/test/preferred/path/H-H1_TEST_1-1000000000-8.h5",
]
@mock.patch.object(views, "_DEFAULT_GSIFTP_HOST", new="testhost")
def test_find_latest(client):
"""Test the `find_latest` view
"""
resp = client.get(
"/LDR/services/data/v1/gwf/L/L1_TEST_1/latest.json",
)
assert resp.status_code == 200
assert resp.json == [
"file://localhost/test/path/L-L1_TEST_1-1000000016-4.gwf",
"gsiftp://testhost:15000/test/path/L-L1_TEST_1-1000000016-4.gwf",
"file://localhost/test/path2/L-L1_TEST_1-1000000016-4.gwf",
"gsiftp://testhost:15000/test/path2/L-L1_TEST_1-1000000016-4.gwf",
]
def test_find_latest_urltype(client):
"""Test the `find_latest` view with urltype
"""
resp = client.get(
"/LDR/services/data/v1/gwf/L/L1_TEST_1/latest/file.json",
)
assert resp.status_code == 200
assert resp.json == [
"file://localhost/test/path/L-L1_TEST_1-1000000016-4.gwf",
"file://localhost/test/path2/L-L1_TEST_1-1000000016-4.gwf",
]
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment