...
 
Commits (29)
stages:
- build
- test
# -- build ------------------
.build: &build
stage: build
after_script:
- ls -l dist/
artifacts:
expire_in: 3h
paths:
- dist
build:tarball:
<<: *build
image: python:3.6
script:
- python -m pip install six
- python setup.py sdist bdist_wheel
build:el7:
<<: *build
image: ligo/base:el7
before_script:
- yum install -yq
rpm-build
epel-rpm-macros
python-rpm-macros
python3-rpm-macros
python-setuptools
python34-setuptools
python2-six
python34-six
script:
- python setup.py sdist
- rpmbuild -tb dist/ligo-gracedb*.tar.gz
- mv -v ~/rpmbuild/RPMS/*/python*-ligo-gracedb-*.rpm dist/
- rm -rf dist/*.tar.gz
.build:debian: &build_debian
<<: *build
before_script:
- apt-get update -yqq
- apt-get install -yq
dpkg-dev
debhelper
dh-python
python-all-dev
python3-all-dev
python-setuptools
python3-setuptools
python-six
python3-six
help2man
script:
- pushd .
- python setup.py sdist
- version=$(python setup.py --version)
- cd dist
- tar -xf ligo-gracedb-${version}.tar.gz
- cd ligo-gracedb-${version}
- dpkg-buildpackage -us -uc -b
- popd
- rm -rf dist/ligo-gracedb*
build:debian:jessie:
<<: *build_debian
image: debian:jessie
build:debian:stretch:
<<: *build_debian
image: debian:stretch
# -- test -------------------
.test: &test
stage: test
image: python
before_script:
- python -m pip install six
- python -m pip install .
script:
- python -c "import ligo.gracedb"
- gracedb --help
test:python2.7:
<<: *test
image: python:2.7
test:python3.6:
<<: *test
image: python:3.6
../ligo/gracedb/cli.py
\ No newline at end of file
......@@ -3,13 +3,7 @@ Packaging procedure:
1. alter these files w/new version/release numbers:
ligo-gracedb.spec
debian/changelog (timestamp comes from relevant git hash)
setup.py
ligo/gracedb/__init__.py
ligo/gracedb/cli.py
ligo/gracedb/test/test.py
bin/gracedb
(in other words, whatever has GIT_TAG in it).
ligo/gracedb/version.py
2. check that the package is good
......@@ -18,7 +12,7 @@ Packaging procedure:
git push --tags
4. generate the source tarball to put into the repo
python setup.py sdist
python setup.py sdist bdist_wheel
*. to generate .deb and .rpm for testing.
python setup.py bdist_rpm
......
%define name ligo-gracedb
%define version 1.28
%define unmangled_version 1.28
%define release 1
%define release 2
Summary: Gravity Wave Candidate Event Database
Name: %{name}
......@@ -12,31 +12,79 @@ License: GPL
Group: Development/Libraries
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
Prefix: %{_prefix}
BuildArch: noarch
Vendor: Tanner Prestegard <tanner.prestegard@ligo.org>, Alexander Pace <alexander.pace@ligo.org>
Requires: ligo-common python-six
BuildRequires: python-setuptools
Url: http://www.lsc-group.phys.uwm.edu/daswg/gracedb.html
BuildArch: noarch
BuildRequires: rpm-build
BuildRequires: epel-rpm-macros
BuildRequires: python-rpm-macros
BuildRequires: python3-rpm-macros
BuildRequires: python-setuptools
BuildRequires: python%{python3_pkgversion}-setuptools
%description
The gravitational-wave candidate event database (GraceDB) is a prototype
system to organize candidate events from gravitational-wave searches and
to provide an environment to record information about follow-ups. A simple
client tool is provided to submit a candidate event to the database.
# -- python2-ligo-gracedb
%package -n python2-%{name}
Summary: %{summary}
Provides: %{name}
Obsoletes: %{name} < 1.27-2
Conflicts: %{name}
Requires: python-six
Requires: python2-ligo-common
%{?python_provide:%python_provide python2-%{name}}
%description -n python2-%{name}
The gravitational-wave candidate event database (GraceDB) is a prototype
system to organize candidate events from gravitational-wave searches and
to provide an environment to record information about follow-ups. A simple
client tool is provided to submit a candidate event to the database.
# -- python-3X-ligo-gracedb
%package -n python%{python3_pkgversion}-%{name}
Summary: %{summary}
Requires: python%{python3_pkgversion}-six
Requires: python%{python3_pkgversion}-ligo-common
%{?python_provide:%python_provide python%{python3_pkgversion}-%{name}}
%description -n python%{python3_pkgversion}-%{name}
The gravitational-wave candidate event database (GraceDB) is a prototype
system to organize candidate events from gravitational-wave searches and
to provide an environment to record information about follow-ups. A simple
client tool is provided to submit a candidate event to the database.
# -- build steps
%prep
%setup -n %{name}-%{unmangled_version}
%build
python setup.py build
# build python3 first
%py3_build
# so that the scripts come from python2
%py2_build
%install
python setup.py install --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
%py2_install
%py3_install
%clean
rm -rf $RPM_BUILD_ROOT
%files -f INSTALLED_FILES
%defattr(-,root,root)
%files -n python2-%{name}
%{_bindir}/gracedb
%{python2_sitelib}/*
%exclude %{python_sitelib}/ligo/gracedb/*pyo
%exclude %{python_sitelib}/ligo/gracedb/test/*pyo
%files -n python%{python3_pkgversion}-%{name}
%{python3_sitelib}/*
%exclude %{python3_sitelib}/ligo/gracedb/test/*pyo
......@@ -15,13 +15,6 @@
#
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
from .version import __version__
__all__ = ["cli", "rest"]
GIT_TAG = 'gracedb-1.28-1'
# issue 717. Required for backward compatibility -- make sure "from ligo import gracedb"
# works as it used to.
from .cli import *
from .rest import ProxyHTTPConnection, ProxyHTTPSConnection
__all__ = ["cli", "exceptions", "rest"]
......@@ -19,14 +19,9 @@
from __future__ import print_function
import os, sys, shutil
import json
from ligo.gracedb.rest import GraceDb, GraceDbBasic
import six
DEFAULT_SERVICE_URL = "https://gracedb.ligo.org/api"
DEFAULT_BASIC_URL = "https://gracedb.ligo.org/apibasic/"
GIT_TAG = 'gracedb-1.28-1'
from ligo.gracedb.rest import GraceDb, GraceDbBasic
from ligo.gracedb.rest import DEFAULT_SERVICE_URL, DEFAULT_BASIC_SERVICE_URL
DEFAULT_COLUMNS = "graceid,labels,group,pipeline,search,far,gpstime,created,dataurl"
......@@ -132,7 +127,10 @@ Client = derive_client()
# Main
def main():
def main(args=None):
if args is None:
args = sys.argv[1:]
usage ="""%%prog [options] GROUP PIPELINE SEARCH EVENTFILE
where GROUP is one of %(groups)s
PIPELINE is one of %(pipelines)s
......@@ -261,7 +259,7 @@ Longer strings will be truncated.""" % {
default=[], type=str
)
options, args = op.parse_args()
options, args = op.parse_args(args)
if isinstance(options.labels, str):
options.labels = options.labels.split(',')
......@@ -294,7 +292,7 @@ Longer strings will be truncated.""" % {
error("To use the basic auth client, specify a basic auth service URL or use the default.")
exit(1)
else:
service = DEFAULT_BASIC_URL
service = DEFAULT_BASIC_SERVICE_URL
# Client subclass according to preferred auth method.
global Client
......
# Custom exceptions
class HTTPError(Exception):
def __init__(self, status, reason, message):
self.status = status
self.reason = reason
self.message = message
Exception.__init__(self, (status, '{} / {}'.format(reason, message)))
......@@ -24,7 +24,7 @@ import sys
if os.name == 'posix':
import pwd
import json
from six.moves.urllib.parse import urlparse
from six.moves.urllib.parse import urlparse, urlencode
from base64 import b64encode
from netrc import netrc, NetrcParseError
from subprocess import Popen, PIPE
......@@ -34,6 +34,9 @@ from datetime import datetime
import six
from six.moves import map
from .exceptions import HTTPError
from .version import __version__
DEFAULT_SERVICE_URL = "https://gracedb.ligo.org/api/"
DEFAULT_BASIC_SERVICE_URL = "https://gracedb.ligo.org/apibasic/"
KNOWN_TEST_HOSTS = ['moe.phys.uwm.edu', 'embb-dev.ligo.caltech.edu', 'simdb.phys.uwm.edu',]
......@@ -146,16 +149,6 @@ def is_expired(cert_file):
return expired, err
#-----------------------------------------------------------------
# Exception(s)
class HTTPError(Exception):
def __init__(self, status, reason, message):
self.status = status
self.reason = reason
self.message = message
Exception.__init__(self, (status, reason+" / "+message))
#-----------------------------------------------------------------
# HTTP/S Proxy classes
# Taken from: http://code.activestate.com/recipes/456195/
......@@ -337,9 +330,19 @@ class GsiRest(object):
url = url and str(url)
priming_url = priming_url and str(priming_url)
# Add version string to user-agent header
version_header = {'User-Agent': 'gracedb-client/{version}'.format(
version=__version__)}
if headers is None:
headers = version_header
else:
headers.update(version_header)
conn = self.getConnection()
if priming_url:
self.make_request(conn, "GET", priming_url, headers={'connection' : 'keep-alive'})
priming_header = {'connection': 'keep-alive'}
priming_header.update(version_header)
self.make_request(conn, "GET", priming_url, headers=priming_header)
response = self.get_response(conn)
if response.status != 200:
response = self.adjustResponse(response)
......@@ -388,7 +391,7 @@ class GsiRest(object):
if not files:
# Simple urlencoded body
if isinstance(body, dict):
# XXX What about the headers in the params?
# XXX What about the headers in the params?
if 'content-type' not in headers:
headers['content-type'] = "application/json"
body = json.dumps(body)
......@@ -397,11 +400,11 @@ class GsiRest(object):
if isinstance(body, dict):
body = list(body.items())
content_type, body = encode_multipart_formdata(body, files)
# XXX What about the headers in the params?
# XXX What about the headers in the params?
headers = {
'content-type': content_type,
'content-length': str(len(body)),
# 'connection': 'keep-alive',
#'connection': 'keep-alive',
}
return self.request(method, url, body, headers)
......@@ -675,10 +678,10 @@ class GraceDb(GsiRest):
qdict = {}
if query: qdict['query'] = query
if count: qdict['count'] = count
if orderby: qdict['orderby'] = orderby
if orderby: qdict['sort'] = orderby
if columns: qdict['columns'] = columns
if qdict:
uri += "?" + urllib.urlencode(qdict)
uri += "?" + urlencode(qdict)
while uri:
response = self.get(uri).json()
events = response.get('events',[])
......@@ -698,7 +701,7 @@ class GraceDb(GsiRest):
"""
uri = self.links['events']
if query:
uri += "?" + urllib.urlencode({'query': query})
uri += "?" + urlencode({'query': query})
return self.get(uri).json()['numRows']
def files(self, graceid, filename=None, raw=False):
......@@ -729,7 +732,7 @@ class GraceDb(GsiRest):
uri = template.format(graceid=graceid, filename=filename or "")
return self.get(uri)
def writeFile(self, graceid, filename, filecontents=None):
def writeFile(self, object_id, filename, filecontents=None):
"""Upload a file
Required args: graceid, filename
......@@ -746,19 +749,9 @@ class GraceDb(GsiRest):
>>> print r.status
"""
template = self.templates['files-template']
uri = template.format(graceid=graceid, filename=os.path.basename(filename))
if filecontents is None:
if filename == '-':
filename = 'stdin'
filecontents = sys.stdin.read()
else:
filecontents = open(filename, "rb").read()
elif hasattr(filecontents, 'read'):
# XXX Does not scale well.
filecontents = filecontents.read()
files = [('upload', os.path.basename(filename), filecontents)]
return self.put(uri, files=files)
print("WARNING: the writeFile() method is deprecated in favor "
"of writeLog() and will be removed in a future release.")
return self.writeLog(object_id, "FILE UPLOAD", filename, filecontents)
def logs(self, graceid):
"""Get all log messages associated with an event
......@@ -1033,6 +1026,11 @@ class GraceDb(GsiRest):
return self.get(uri)
def createTag(self, graceid, n, tagname, displayName=None):
print('WARNING: the createTag() method is deprecated in favor of '
'addTag() and will be removed in a future release.')
return self.createTag(graceid, n, tagname, displayName)
def addTag(self, graceid, n, tagname, displayName=None):
"""Add a new tag to a log message
Required arguments: graceid, n (the number of the log message)
......@@ -1053,6 +1051,11 @@ class GraceDb(GsiRest):
return self.put(uri, body={'displayName': displayName})
def deleteTag(self, graceid, n, tagname):
print('WARNING: the deleteTag() method is deprecated in favor of '
'removeTag() and will be removed in a future release.')
return self.removeTag(graceid, n, tagname)
def removeTag(self, graceid, n, tagname):
"""Remove a tag from a given log message
Required arguments: graceid, n (the number of the log message)
......@@ -1237,8 +1240,16 @@ class GraceDbBasic(GraceDb):
conn = self.getConnection()
headers = headers or {}
headers.update(self.authn_header)
# Add version string to user-agent header
version_header = {'User-Agent': 'gracedb-client/{version}'.format(
version=__version__)}
headers.update(version_header)
# Make request and get response
self.make_request(conn, method, url, body, headers)
response = self.get_response(conn)
# Catch the 401 unauthorized response before sending to adjust
# response. Effectively, the 401 response will have special status.
if response.status == 401:
......
......@@ -170,10 +170,10 @@ class TestMain(TestGraceDb):
"""
uploadFile = os.path.join(self.TEST_DATA_DIR, "big.data")
r = self._gracedb.writeFile(self._eventId, uploadFile)
r = self._gracedb.writeLog(self._eventId, "FILE UPLOAD", uploadFile)
self.assertEqual(r.status, 201) # CREATED
r_content = r.json()
link = r_content['permalink']
link = r_content['file']
self.assertEqual(open(uploadFile, 'rb').read(),
self._gracedb.get(self._gracedb.files(self._eventId).json()
['big.data']).read())
......@@ -184,10 +184,10 @@ class TestMain(TestGraceDb):
"""Upload and re-upload a file"""
uploadFile = os.path.join(self.TEST_DATA_DIR, "upload.data")
r = self._gracedb.writeFile(self._eventId, uploadFile)
r = self._gracedb.writeLog(self._eventId, "FILE UPLOAD", uploadFile)
self.assertEqual(r.status, 201) # CREATED
r_content = r.json()
link = r_content['permalink']
link = r_content['file']
self.assertEqual(open(uploadFile, 'rb').read(),
self._gracedb.get(self._gracedb.files(self._eventId).json()
......@@ -198,11 +198,11 @@ class TestMain(TestGraceDb):
# Re-upload slightly different file.
uploadFile2 = os.path.join(self.TEST_DATA_DIR, "upload2.data")
r = self._gracedb.writeFile(self._eventId, filename="upload.data",
filecontents=open(uploadFile2, 'r'))
r = self._gracedb.writeLog(self._eventId, "FILE UPLOAD",
filename="upload.data", filecontents=open(uploadFile2, 'r'))
self.assertEqual(r.status, 201) # CREATED
r_content = r.json()
link2 = r_content['permalink']
link2 = r_content['file']
self.assertEqual(open(uploadFile2, 'rb').read(),
self._gracedb.get(self._gracedb.files(self._eventId).json()
......@@ -249,7 +249,8 @@ class TestMain(TestGraceDb):
self.assertEqual(mbta_event['group'], "Test")
self.assertEqual(mbta_event['pipeline'], "MBTAOnline")
self.assertEqual(float(mbta_event['gpstime']), 1078903329.421037)
self.assertEqual(mbta_event['far'], 4.006953918826065e-7)
original_far = 4.006953918826065e-7
self.assertTrue((mbta_event['far'] - original_far) < original_far*1e-14)
def test_create_hardwareinjection(self):
"""Create a HardwareInjection event"""
......@@ -290,7 +291,7 @@ class TestMain(TestGraceDb):
Raises exception if workaround fails.
"""
uploadFile = os.path.join(self.TEST_DATA_DIR, "upload.data.gz")
r = self._gracedb.writeFile(self._eventId, uploadFile)
r = self._gracedb.writeLog(self._eventId, "FILE UPLOAD", uploadFile)
self.assertEqual(r.status, 201) # CREATED
def test_unicode_param(self):
......@@ -300,7 +301,7 @@ class TestMain(TestGraceDb):
Raises exception if workaround fails.
"""
uploadFile = os.path.join(self.TEST_DATA_DIR, "upload.data.gz")
r = self._gracedb.writeFile(self._eventId, uploadFile)
r = self._gracedb.writeLog(self._eventId, "FILE UPLOAD", uploadFile)
self.assertEqual(r.status, 201) # CREATED
def test_label_event(self):
......@@ -358,48 +359,6 @@ class TestMain(TestGraceDb):
event_logs = self._gracedb.logs(graceid).read()
self.assertTrue(message.encode() in event_logs)
def test_issue717(self):
# non-backwards-compatibility of cli.Client import.
# import and use should not cause an import error.
from ligo import gracedb
gracedb.Client
gracedb.ProxyHTTPConnection
gracedb.error
gracedb.ProxyHTTPSConnection
def test_gittag(self):
# try to make sure GIT_TAG is set properly.
import errno
version = "1.28"
try:
# If we are in the source dir (setup.py is available)
# make sure the version above agrees.
if os.path.exists("setup.py"):
setup_file = open("setup.py", 'r')
else:
setup_file = open("../../../setup.py", 'r')
v = ""
for line in setup_file:
if line.startswith("version"):
v = line.split('"')[1]
break
self.assertEqual(v, version)
except IOError as e:
if e.errno != errno.ENOENT:
raise
# GIT_TAG should look like "gracedb-VERSION-PKG"
# and VERSION should == version from above.
from ligo.gracedb import GIT_TAG as package_tag
package_tag = package_tag.split('-')[1]
self.assertTrue(package_tag.startswith(v))
from ligo.gracedb.cli import GIT_TAG as cli_tag
cli_tag = cli_tag.split('-')[1]
self.assertTrue(cli_tag.startswith(v))
self.assertEqual(cli_tag, package_tag)
if __name__ == "__main__":
# Import other unit tests
......
......@@ -61,7 +61,7 @@ class TestVOEvents(TestGraceDb):
# Tests -------------------------------------------------------------------
def test_create_preliminary_voevent(self):
"""Create ia preliminary VOEvent"""
"""Create a preliminary VOEvent"""
r = self._gracedb.createVOEvent(self._graceid, "Preliminary")
rdict = r.json()
self.assertTrue('voevent_type' in list(rdict.keys()))
......
__version__ = '1.28'
......@@ -2,3 +2,5 @@
requires = ligo-common
release = 1
[bdist_wheel]
universal=1
......@@ -16,14 +16,12 @@
# You should have received a copy of the GNU General Public License
# along with gracedb. If not, see <http://www.gnu.org/licenses/>.
import os
from setuptools import setup, find_packages
version = "1.28"
from ligo.gracedb import __version__
setup(
name = "ligo-gracedb",
version = version,
version = __version__,
maintainer = "Tanner Prestegard, Alexander Pace",
maintainer_email = "tanner.prestegard@ligo.org, alexander.pace@ligo.org",
description = "Gravitational Wave Candidate Event Database",
......@@ -38,9 +36,10 @@ setup(
install_requires = ['six'],
package_data = { 'ligo.gracedb.test' : ['data/*', 'test.sh', 'README'] },
scripts = [
os.path.join('bin','gracedb'),
],
entry_points={
'console_scripts': [
'gracedb=ligo.gracedb.cli:main',
],
}
)