Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • duncanmmacleod/dqsegdb-client
  • computing/dqsegdb/client
2 results
Show changes
Commits on Source (17)
; vim: set ft=dosini :
[flake8]
max-line-length = 120
select =
# pycodestyle errors
E,
# flake8-executable - https://pypi.org/project/flake8-executable/
EXE,
# pyflakes
F,
# bandit security
# bandit security - https://pypi.org/project/flake8-bandit/
S,
# pycodestyle warnings
W,
ignore =
# line break before binary operator, use W504
# W503 = warn about "line break before binary operator"
# W504 = warn about "line break after binary operator"
# ignore W503 = allow W503, not W504
# see https://peps.python.org/pep-0008/#should-a-line-break-before-or-after-a-binary-operator
W503,
exclude =
__pycache__,
......
stages:
- Source distribution
- Source packages
- Binary packages
- Testing
- Documentation
- Code coverage
- Code quality
include:
# tarball
- local: '/.gitlab/ci/dist.yml'
# RHEL packaging
- local: '/.gitlab/ci/rhel.yml'
# Debian packaging
- local: '/.gitlab/ci/debian.yml'
# Python tests
- local: '/.gitlab/ci/python.yml'
# Code coverage
- local: '/.gitlab/ci/coverage.yml'
# Code quality
- local: '/.gitlab/ci/analysis.yml'
# -- Python ------------------------
# Build tarball (sdist) and wheel, and run tests with PyTest for various
# Python versions, with coverage and test reports
# https://computing.docs.ligo.org/guide/gitlab/components/python/all/
- component: git.ligo.org/computing/gitlab/components/python/all@2
inputs:
# quality assurance
code_quality_analyzer: flake8
run_advanced_sast: true
# build-and-test
python_versions:
- "3.9"
- "3.10"
- "3.11"
pytest_options: >-
--cov=dqsegdb
-ra
--verbose
extra_test_commands: &extra_test_commands
# sanity check the scripts
- PREFIX=$(python3 -c "import sys; print(sys.prefix)")
- for entrypoint in $(ls ${PREFIX}/bin/*dqsegdb);
do
echo -e "\x1B[92m$ ${entrypoint} --help...\x1B[0m";
python3 -m coverage run
--append
--source=dqsegdb
${entrypoint} --help
; done
# -- Debian packaging --------------
# Build Debian source and binary packages from the tarball (separately)
# and runs test jobs
# https://computing.docs.ligo.org/guide/gitlab/components/debian/all/
- component: git.ligo.org/computing/gitlab/components/debian/all@2
inputs:
needs: [sdist]
debian_versions:
- bullseye
- bookworm
test_script: &system_test_script
# run pytest using the installed test suite (dqsegdb.tests)
- python3 -m pytest
--cov=dqsegdb
--junit-xml=junit.xml
--pyargs
-ra
--verbose
dqsegdb.tests
# run the same 'extra' commands to check the scripts
- *extra_test_commands
test_install: &system_test_install >-
dqsegdb
python3-pytest
python3-pytest-cov
# -- Red Hat packaging -------------
# Build RPMs from the tarball and runs test jobs
# https://computing.docs.ligo.org/guide/gitlab/components/redhat/all/
- component: git.ligo.org/computing/gitlab/components/redhat/all@2
inputs:
needs: [sdist]
redhat_versions:
- 8
test_script: *system_test_script
test_install: *system_test_install
# -- customisations ------------------
# allow Debian test jobs to fail until this project can be updated to
# support lscsoft-glue >= 4.0.0.
debian_test:
allow_failure: true
# ---------------------------
# Code Quality workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
file:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/
- python.yml
# -- code quality -----------
#
# These jobs check the code for
# quality issues
#
flake8:
stage: Code quality
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:flake8
- .python:flake8
needs: []
variables:
# don't fail the pipeline because of linting issues,
# these are presented in the code-quality box in the
# merge_request UI
FLAKE8_OPTIONS: "--exit-zero"
# ---------------------------
# Code coverage
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/python/
file: python.yml
# -- coverage ---------------
#
# This job combinces the individual
# coverage reports from each job
# to make a proper coverage result
#
coverage:
stage: Code coverage
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:coverage-combine
- .python:coverage-combine
# ---------------------------
# Debian packaging workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/
file: debian.yml
# local test template
- local: /.gitlab/ci/test.yml
# -- macros
.buster:
image: igwn/builder:buster
.bullseye:
image: igwn/builder:bullseye
# -- source packages --------
#
# These jobs make DSC packages
#
.dsc:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/#.debian:dsc
- .debian:dsc
stage: Source packages
needs:
- tarball
variables:
TARBALL: "dqsegdb-*.tar.*"
before_script:
- !reference [".debian:dsc", "before_script"]
# install dch
- apt-get -yqq install devscripts
# -- hack the tarball so that setuptools-scm version matches
# the debian changelog
- tar -zxf ${TARBALL}
# get version from the Python metadata
- PKG_VERSION=$(grep ^Version dqsegdb-*/PKG-INFO | cut -d\ -f2)
# add a changelog entry for this version
- (cd dqsegdb-*/; dch --force-bad-version --newversion ${PKG_VERSION}-9999 --controlmaint "ci build")
# repack the tarball
- tar -zcf ${TARBALL} */
dsc:buster:
extends:
- .dsc
- .buster
dsc:bullseye:
extends:
- .dsc
- .bullseye
# -- binary packages --------
#
# These jobs generate DEB
# binary packages from the
# DSC sources packages
#
.deb:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/#.debian:deb
- .debian:deb
stage: Binary packages
variables:
DSC: "dqsegdb_*.dsc"
deb:buster:
extends:
- .deb
- .buster
needs:
- dsc:buster
deb:bullseye:
extends:
- .deb
- .bullseye
needs:
- dsc:bullseye
# -- test -------------------
.test:debian:
extends:
# see /.gitlab/ci/test.yml
- .test
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/#.debian:base
- .debian:base
before_script:
# set up apt
- !reference [".debian:base", before_script]
# setup local apt repository
- apt-get -y -q -q install local-apt-repository
- mkdir -pv /srv/local-apt-repository
# fill our local apt repo and rebuild it
- mv -v *.deb /srv/local-apt-repository
- /usr/lib/local-apt-repository/rebuild
- apt-get -y -q update
# install our package(s)
- apt-get -y install python3-dqsegdb dqsegdb
# install testing dependencies
- apt-get -y -q install
findutils
python3-coverage
python3-pip
python3-pytest
python3-pytest-cov
python3-requests-mock
# upgrade coverage.py to >=5 so that .coverage files use new format
- python3 -m pip install "coverage>=5" --upgrade-strategy=only-if-needed
test:buster:
extends:
- .test:debian
- .buster
image: igwn/base:buster
needs:
- deb:buster
test:bullseye:
extends:
- .test:debian
- .bullseye
image: igwn/base:bullseye
needs:
- deb:bullseye
# -- lint -------------------
#
# These jobs check the code
# for quality issues
#
.lintian:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/#.debian:lint
- .debian:lint
stage: Code quality
variables:
LINTIAN_OPTIONS: "--color always --suppress-tags new-package-should-close-itp-bug,groff-message --allow-root --pedantic"
lintian:buster:
extends:
- .lintian
- .buster
needs:
- deb:buster
variables:
LINTIAN_OPTIONS: "--color always --suppress-tags new-package-should-close-itp-bug,latest-debian-changelog-entry-without-new-version --allow-root --pedantic"
lintian:bullseye:
extends:
- .lintian
- .bullseye
needs:
- deb:bullseye
# ---------------------------
# Distribution workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/python/
file: python.yml
# -- dist -------------------
#
# This job makes the dqsegdb-X.Y.Z.tar.gz
# distribution and uploads it as a job
# artifact
#
tarball:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:build
- .python:build
image: python:3
stage: Source distribution
variables:
SDIST: "true"
WHEEL: "false"
# ---------------------------
# Python workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/python/
file: python.yml
# local test template
- local: /.gitlab/ci/test.yml
# -- build ------------------
#
# this job builds the wheel
# distribution
#
wheel:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:build
- .python:build
image: python:3
stage: Binary packages
variables:
SDIST: "false"
WHEEL: "true"
# -- test -------------------
#
# These jobs run the tests
#
.test:pip:
extends:
# see /.gitlab/ci/test.yml
- .test
needs:
- wheel
variables:
INSTALL_TARGET: "dqsegdb-*.whl"
test:python3.6:
extends:
- .test:pip
image: python:3.6
test:python3.7:
extends:
- .test:pip
image: python:3.7
test:python3.8:
extends:
- .test:pip
image: python:3.8
test:python3.9:
extends:
- .test:pip
image: python:3.9
test:python3.10:
extends:
- .test:pip
image: python:3.10
# ---------------------------
# RHEL packaging workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/rhel/
file: rhel.yml
# local test template
- local: /.gitlab/ci/test.yml
# -- macros
.el7:
image: igwn/builder:el7-testing
variables:
EPEL: "true"
.el8:
image: igwn/builder:el8-testing
variables:
EPEL: "true"
# -- source packages --------
#
# These jobs make src RPMs
#
.srpm:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/rhel/#.rhel:srpm
- .rhel:srpm
stage: Source packages
needs:
- tarball
variables:
TARBALL: "dqsegdb-*.tar.*"
before_script:
- !reference [".rhel:srpm", "before_script"]
# hack the spec file to use the correct package version
- tar --file ${TARBALL} --wildcards --strip-components 1 --get dqsegdb*/etc/*.spec dqsegdb*/PKG-INFO
- PKG_VERSION=$(grep ^Version PKG-INFO | cut -d\ -f2)
- sed -i 's|define version\( *\)\(.*\)|define unmangled_version '${PKG_VERSION}'\n%define version '${PKG_VERSION/-/+}'|' etc/*.spec
- sed -i 's|pypi_source|pypi_source %{srcname} %{unmangled_version}|' etc/*.spec
script:
- rpmbuild -bs
--define "_srcrpmdir ${CI_PROJECT_DIR}"
--define "_sourcedir ${CI_PROJECT_DIR}"
etc/*.spec
srpm:el7:
extends:
- .srpm
- .el7
srpm:el8:
extends:
- .srpm
- .el8
# -- binary packages --------
#
# These jobs generate binary RPMs
# from the src RPMs
#
.rpm:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/rhel/#.rhel:rpm
- .rhel:rpm
stage: Binary packages
variables:
SRPM: "python-dqsegdb-*.src.rpm"
rpm:el7:
extends:
- .rpm
- .el7
needs:
- srpm:el7
rpm:el8:
extends:
- .rpm
- .el8
needs:
- srpm:el8
# -- test -------------------
.test:el:
extends:
# see /.gitlab/ci/test.yml
- .test
before_script:
# set up yum caching
- !reference [".rhel:base", before_script]
# configure EPEL
- yum -y -q install epel-release && yum -y -q install epel-rpm-macros
# install testing dependencies
- PY3=$(rpm --eval '%{?python3_pkgversion:%{python3_pkgversion}}%{!?python3_pkgversion:3}')
- yum -y -q install
findutils
python${PY3}-pip
python${PY3}-pytest
python${PY3}-pytest-cov
# install our package(s)
- yum -y -q install *.rpm
# upgrade coverage.py to >=5 so that .coverage files use new format
- python3 -m pip install "coverage>=5" --upgrade-strategy=only-if-needed
artifacts:
reports:
junit: null
test:el7:
extends:
- .test:el
- .el7
image: igwn/base:el7-testing
needs:
- rpm:el7
test:el8:
extends:
- .test:el
- .el8
image: igwn/base:el8-testing
needs:
- rpm:el8
# -- lint -------------------
#
# These jobs check the code
# for quality issues
#
.rpmlint:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/rhel/#.rhel:lint
- .rhel:lint
stage: Code quality
variables:
GIT_STRATEGY: fetch
RPMLINT_OPTIONS: '--info --file .rpmlintrc'
rpmlint:el7:
extends:
- .rpmlint
- .el7
needs:
- rpm:el7
rpmlint:el8:
extends:
- .rpmlint
- .el8
needs:
- rpm:el8
# ---------------------------
# Testing workflow
# ---------------------------
# test template used in other jobs
# see python.yml for example
.test:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:pytest
- .python:pytest
stage: Testing
variables:
# target for coverage
COVERAGE_TARGET: "dqsegdb"
# don't need the git repo
GIT_STRATEGY: none
# configure pytest
PYTEST_OPTIONS: "-ra -v --pyargs dqsegdb.tests"
# use python3
PYTHON: "python3"
script:
# run the tests
- !reference [".python:pytest", script]
# run --help on each one (with coverage)
- PREFIX=$(${PYTHON} -c "import sys; print(sys.prefix)")
- for entrypoint in $(ls ${PREFIX}/bin/*dqsegdb);
do
echo -e "\x1B[92m$ ${entrypoint} --help...\x1B[0m";
${PYTHON} -m coverage run
--append
--source=dqsegdb
${entrypoint} --help
; done
# upload coverage file as an artifact to be combined later
artifacts:
when: always
paths:
- .coverage-*
# disable coverage total for test jobs, this is reported
# by the coverage job
coverage: null
......@@ -132,7 +132,7 @@ def parse_command_line():
# Data location options
parser.add_option("-t", "--segment-url", metavar = "segment_url", help = "Segment URL. Users have to specify either 'https://' for a secure connection or 'http://' for an insecure connection to the segment database URL. For example, '--segment-url=https://segments.ligo.org'. No need to specify port number. ")
parser.add_option("-d", "--database", metavar = "use_database", action = "store_true", help = "Use database specified by environment variable S6_SEGMENT_SERVER. For example, 'S6_SEGMENT_SERVER=https://segdb.ligo.caltech.edu'")
parser.add_option("-d", "--database", metavar = "use_database", action = "store_true", help = "Use database specified by environment variable DEFAULT_SEGMENT_SERVER. For example, 'DEFAULT_SEGMENT_SERVER=https://segments.ligo.org'")
parser.add_option("-f", "--dmt-files", metavar = "use_files", action = "store_true", help = "Use files in directory specified by environment variable ONLINEDQ, for example, 'ONLINEDQ=file:///path_to_dmt'. 'file://' is the prefix, the actual directory to DMT xml files starts with '/'.")
......@@ -186,9 +186,12 @@ def parse_command_line():
outfile.close()
file_location = tmp_dir
elif options.database:
if 'S6_SEGMENT_SERVER' not in os.environ:
raise ValueError( "--database specified but S6_SEGMENT_SERVER not set" )
database_location = os.environ['S6_SEGMENT_SERVER']
try:
database_location = os.environ['DEFAULT_SEGMENT_SERVER']
except KeyError:
raise ValueError(
"--database specified but DEFAULT_SEGMENT_SERVER not set",
)
elif options.dmt_files:
if 'ONLINEDQ' not in os.environ:
raise ValueError( "--dmt-files specified but ONLINEDQ not set" )
......
......@@ -109,7 +109,7 @@ def parse_command_line():
parser.add_option("-k", "--keep-db", metavar = "keep_db", action = "store_true", help = "Keep sqlite database (don't delete it when the script exits).")
parser.add_option("-r", "--remove-definer-table", action = "store_true", help = "If set, the veto definer table will not be included in the output.")
parser.add_option("-t", "--segment-url", metavar = "segment_url", help = "Segment URL, e.g., https://segments.ligo.org .")
parser.add_option("-d", "--database", metavar = "use_database", action = "store_true", help = "Use database specified by environment variable S6_SEGMENT_SERVER.")
parser.add_option("-d", "--database", metavar = "use_database", action = "store_true", help = "Use database specified by environment variable DEFAULT_SEGMENT_SERVER.")
parser.add_option("-f", "--dmt-file", metavar = "use_files", action = "store_true", help = "Warning: NOT UPDATED FOR DQSEGDB CLIENT YET! use files in directory specified by environment variable ONLINEDQ.")
parser.add_option("-c", "--cumulative-categories", action = "store_true", help = "If set, the category N files will contain all segments in categories <= N.")
parser.add_option("-p", "--separate-categories", action = "store_true", help = "If set, the category N files will contain only category N.")
......@@ -187,9 +187,12 @@ def parse_command_line():
outfile.close()
file_location = tmp_dir
elif options.database:
if 'S6_SEGMENT_SERVER' not in os.environ:
raise ValueError( "--database specified by S6_SEGMENT_SERVER not set" )
database_location = os.environ['S6_SEGMENT_SERVER']
try:
database_location = os.environ['DEFAULT_SEGMENT_SERVER']
except KeyError:
raise ValueError(
"--database specified by DEFAULT_SEGMENT_SERVER not set",
)
elif options.dmt_file:
# Fix!!! Easy to add dmt files and s6 server compatibility, but requires time to test them before deploying them!
raise ValueError("DMT files method not yet suppported for this client in ER6, please use ligolw_segments_from_cats (S6 client)")
......
dqsegdb (2.1.0-1) unstable; urgency=medium
* disable pytest during RPM/Deb builds
* pin lscsoft-glue to >= 3.0.1, < 4.0.0
-- Robert Bruntz <robert.bruntz@ligo.org> Thu, 6 Feb 2024 02:35:00 -0500
dqsegdb (2.0.0-1) unstable; urgency=medium
* 2.0.0 change from Python 2 to Python 3
......
......@@ -20,6 +20,7 @@ Depends:
python3-lal,
python3-ligo-segments,
python3-lscsoft-glue (>= 3.0.1),
python3-lscsoft-glue (<< 4.0.0),
python3-openssl (>= 0.14),
python3-pyrxp,
Description: Python 3 DQSegDB client
......
......@@ -31,9 +31,6 @@ from glue.ligolw import types as ligolwtypes
from glue.segmentdb import segmentdb_utils
from glue.ligolw.utils import ligolw_sqlite
from glue.ligolw import dbtables
from dqsegdb import jsonhelper
......@@ -466,22 +463,6 @@ def run_query_segments(doc, process_id, engine, gps_start_time, gps_end_time, in
#
def setup_files(dir_name, gps_start_time, gps_end_time):
# Filter out the ones that are outside our time range
xml_files = segmentdb_utils.get_all_files_in_range(dir_name, gps_start_time, gps_end_time)
handle, temp_db = tempfile.mkstemp(suffix='.sqlite')
os.close(handle)
target = dbtables.get_connection_filename(temp_db, None, True, False)
connection = ligolw_sqlite.setup(target)
ligolw_sqlite.insert_from_urls(connection, xml_files) # [temp_xml])
segmentdb_utils.ensure_segment_table(connection)
return temp_db, connection
def add_to_segment_ns(xmldoc, proc_id, seg_def_id, sgmtlist):
try:
segtable = table.get_table(xmldoc, lsctables.SegmentTable.tableName)
......
%define srcname dqsegdb
%define version 2.0.0
%define version 2.1.0
%define release 1
Name: python-%{srcname}
......@@ -49,6 +49,7 @@ Requires: python%{python3_pkgversion}-gwdatafind
Requires: python%{python3_pkgversion}-lal
Requires: python%{python3_pkgversion}-ligo-segments
Requires: python%{python3_pkgversion}-lscsoft-glue >= 3.0.1
Requires: python%{python3_pkgversion}-lscsoft-glue < 4.0.0
Requires: python%{python3_pkgversion}-pyOpenSSL >= 0.14
Requires: python%{python3_pkgversion}-pyRXP
%{?python_provide:%python_provide python%{python3_pkgversion}-%{srcname}}
......@@ -96,6 +97,13 @@ rm -rf $RPM_BUILD_ROOT
# -- changelog
%changelog
* Tue Feb 06 2024 Robert Bruntz <robert.bruntz@ligo.org> 2.1.0-1
- update RPM build/install to use wheels
- remove references to (removed) user-env scripts
- disable pytest during RPM/Deb builds
- pin lscsoft-glue to >= 3.0.1, < 4.0.0
* Thu Jan 27 2022 Robert Bruntz <robert.bruntz@ligo.org> 2.0.0-1
- added python3-lal as a requirement for python3-dqsegdb
......
......@@ -2,7 +2,7 @@ gpstime
gwdatafind
lalsuite
ligo-segments
lscsoft-glue >= 3.0.1
lscsoft-glue >= 3.0.1,<4.0.0
pyOpenSSL >= 0.14
pyRXP
six
......@@ -42,7 +42,7 @@ setup(name=PACKAGENAME,
'gwdatafind',
'lalsuite',
'ligo-segments',
'lscsoft-glue>=3.0.1',
'lscsoft-glue>=3.0.1,<4.0.0',
'pyOpenSSL>=0.14',
'pyRXP',
'six',
......