Merge branch 'osdf-urls' into 'master'

Add support for OSDF URL transforms

Closes #18

See merge request !64

config/gwdatafind-server.ini

@@ -51,3 +51,25 @@ authorization = None
 
 [[PUBLIC_IP]] # Change to your public IP
 authorization = grid-mapfile
+
+# OSDF configuration
+[[osdf kagra]]
+posix_prefix = /cvmfs/kagra.storage.igwn.org/igwn
+osdf_base_path = /igwn
+
+[[osdf ligo]]
+posix_prefix = /cvmfs/ligo.storage.igwn.org/igwn
+osdf_base_path = /igwn
+
+[[osdf virgo]]
+posix_prefix = /cvmfs/virgo.storage.igwn.org/igwn
+osdf_base_path = /igwn
+
+# legacy paths
+[[osdf ligo-osgstorage]]
+posix_prefix = /cvmfs/ligo.osgstorage.org/frames
+osdf_base_path = /user/ligo/frames
+
+[[osdf ligo-oasis]]
+posix_prefix = /cvmfs/oasis.opensciencegrid.org/ligo/frames
+osdf_base_path = /user/ligo/frames

gwdatafind_server/api/utils.py

@@ -78,3 +78,25 @@ def _gsiftp_url(path):
     host = config.get('gsiftphost', _DEFAULT_GSIFTP_HOST)
     port = config.get('gsiftpport', _DEFAULT_GSIFTP_PORT)
     return f'gsiftp://{host}:{port}{path}'
+
+
+def _osdf_url(path):
+    """Format a POSIX ``path`` as an ``osdf://`` URL
+
+    Examples
+    --------
+    >>> _osdf_url('/cvmfs/x.storage.igwn.org/data/X-TEST-0-1.gwf')
+    'osdf:///data/X-TEST-0-1.gwf'
+    """
+    config = current_app.config
+    base = None
+    for section in filter(lambda x: x.startswith("osdf "), config):
+        prefix = config[section].get("posix_prefix", f"/cvmfs/{section}")
+        if path.startswith(prefix):
+            base = config[section].get("osdf_base_path", "")
+            break
+
+    if base is None:  # nothing found, just return the original path
+        return None
+
+    return f"osdf://{path.replace(prefix, base)}"

gwdatafind_server/api/v1.py

@@ -26,6 +26,7 @@ from .utils import (
     error_as_json,
     _file_url,
     _gsiftp_url,
+    _osdf_url,
 )
 
 blueprint = Blueprint(
@@ -251,6 +252,12 @@ def find_latest_url(ext, site, tag, urltype=None):
 
 # -- URL matcher --------------------------------------------------------------
 
+URL_TRANSFORM = {
+    "file": _file_url,
+    "gsiftp": _gsiftp_url,
+    "osdf": _osdf_url,
+}
+
 
 def _get_latest_segment(seglist, duration):
     """Get segment for latest file of the given duration in a segment list
@@ -259,9 +266,17 @@ def _get_latest_segment(seglist, duration):
     return segment(end-duration, end)
 
 
-def _find_urls(ext, site, tag, start, end, urltype=None, match=None,
-               latest=False):
-    """Find all URLs for the given GPS interval
+def _find_urls(
+    ext,
+    site,
+    tag,
+    start,
+    end,
+    urltype=None,
+    match=None,
+    latest=False,
+):
+    """Find all URLs for the given GPS interval.
     """
     cache = get_dataset_cache(ext, site, tag)
 
@@ -295,20 +310,24 @@ def _find_urls(ext, site, tag, start, end, urltype=None, match=None,
                 gps += cdur
 
     # convert paths to URLs for various schemes
+    urltypes = set(URL_TRANSFORM.keys())
+    if urltype:
+        urltypes &= {urltype}
     allurls = {}
     for lfn in lfns:
         allurls[lfn] = []
         for path in lfns[lfn]:
-            # build file:// and gsiftp:// URL for each LFN
-            allurls[lfn].extend((
-                _file_url(path),
-                _gsiftp_url(path),
+            # build URL for each urltype (ignoring None)
+            allurls[lfn].extend(filter(
+                None,
+                (URL_TRANSFORM[_urlt](path) for _urlt in sorted(urltypes))
             ))
 
     # filter URLs for each LFN and return
     urls = []
     for lfn in allurls:
         urls.extend(_filter_urls(allurls[lfn], urltype=urltype, regex=match))
+
     return urls
 
 

tests/cache.dat

@@ -8,3 +8,5 @@
 /test/path,L,L1_TEST_IGNORE,1,8,gwf 1556913881 1 {1000000000 1000000008}
 /test/path,X,X1_TEST_2,1,8,gwf 1556913881 1 {1000000000 1000000008}
 /test/path,Y,Y1_EXCLUDE_1,1,8,gwf 1556913881 1 {1000000000 1000000008}
+# test CVMFS OSDF transforms
+/cvmfs/h.storage.igwn.org/igwn/test,H,H1_TEST_1,1,4,gwf 1556913881 1 {1000000008 1000000016}

tests/gwdatafind-server.ini

@@ -17,3 +17,7 @@ scitokens_issuer = "test"
 scitokens_audience = "TEST"
 # SciTokens Scope
 scitokens_scope = "gwdatafind.read"
+
+[[osdf test]]
+posix_prefix = /cvmfs/h.storage.igwn.org
+osdf_base_path = "/export"

tests/test_api_v1.py

@@ -155,6 +155,29 @@ def test_find_urls_noninteger(client, start, end):
     ]
 
 
+def test_find_urls_osdf(client):
+    """Check that paths get transformed into OSDF URLs appropriately.
+    """
+    resp = client.get(
+        "/services/data/v1"
+        "/gwf/H/H1_TEST_1/1000000000,1000000012/osdf.json",
+    )
+    assert resp.status_code == 200
+    assert sorted(resp.json) == [
+        "osdf:///export/igwn/test/H-H1_TEST_1-1000000008-4.gwf",
+    ]
+
+
+def test_find_urls_osdf_empty(client):
+    """Check that URLS that don't support OSDF return empty.
+    """
+    resp = client.get(
+        "/services/data/v1/gwf/L/L1_TEST_1/1000000004,1000000016/osdf.json",
+    )
+    assert resp.status_code == 200
+    assert not resp.json  # no hits that support OSDF
+
+
 @mock.patch.object(api_utils, "_DEFAULT_GSIFTP_HOST", new="testhost")
 def test_find_latest(client):
     """Test the `find_latest` view