...
 
Commits (112)
stages:
- build
- package
- test
- docs
- deploy
# -- build ------------------
build:sdist:
stage: build
image: ligo/lalsuite-dev:stretch
image: python
script:
- python setup.py sdist -d .
artifacts:
......@@ -41,82 +50,215 @@ build:wheel:macos:
paths:
- '*.whl'
test:stretch:
stage: test
image: ligo/lalsuite-dev:stretch
# -- package ----------------
.package:debian: &package-debian
stage: package
dependencies:
- build:sdist
variables:
GIT_STRATEGY: none
before_script:
- apt-get update
- apt-get install -y lal-python
- apt-get -y install dpkg-dev devscripts lintian
script:
# unpack tarball
- mkdir -p src
- tar -C ./src --strip-components 1 -xf *.tar.*
- cd src
# install build dependencies
- mk-build-deps --tool "apt-get -y" --install --remove
# build debian packages
- dpkg-buildpackage -us -uc -b
# lint
- lintian --pedantic ../*.{deb,changes}
artifacts:
paths:
- '*.deb'
- '*.changes'
package:stretch:
<<: *package-debian
image: ligo/base:stretch
.package:el7: &package-el
stage: package
dependencies:
- build:sdist
variables:
GIT_STRATEGY: none
before_script:
- yum install -y yum-utils rpm-build rpmlint epel-rpm-macros
script:
# prep
- mkdir -p ~/rpmbuild/{SOURCES,SPECS}
- tar -C ~/rpmbuild/SPECS -xf *.tar.* --strip-components 2 '*/etc/glue.spec'
- mv *.tar.* ~/rpmbuild/SOURCES
# install build dependencies
- yum-builddep -y ~/rpmbuild/SPECS/*.spec
# build binary rpms
- rpmbuild -ba ~/rpmbuild/SPECS/*.spec
- mv $(find ~/rpmbuild/{RPMS,SPECS,SRPMS} -type f) .
# lint
- echo "from Config import *; addFilter(\"python-bytecode-wrong-magic-value\")" >> rpmlintrc
- rpmlint -f rpmlintrc *.{rpm,spec}
artifacts:
paths:
- '*.rpm'
- '*.spec'
package:el7:
<<: *package-el
image: ligo/base:el7
# -- test -------------------
.test: &test
stage: test
image: python
dependencies:
- build:sdist
before_script:
- python -m pip install *.tar.*
- python -m pip install lalsuite
script:
- tar xf *.tar.*
- cd $(find . -type d -maxdepth 1 -name 'lscsoft-glue-*')
- python setup.py install
- make -C test
test:el7:
test:python2.7:
<<: *test
image: python:2.7
test:python3.4:
<<: *test
image: python:3.4
test:python3.5:
<<: *test
image: python:3.5
test:python3.6:
<<: *test
image: python:3.6
test:python3.7:
<<: *test
image: python:3.7
test:el7:python2:
stage: test
image: ligo/lalsuite-dev:el7
dependencies:
- package:el7
- build:sdist
variables:
GIT_STRATEGY: none
before_script:
- yum install -y lal-python
script:
- tar xf *.tar.*
- cd $(find . -type d -maxdepth 1 -name 'lscsoft-glue-*')
- python setup.py install
- make -C test
# install glue
- yum -y --nogpgcheck localinstall python2-*glue*.rpm
# install lal
- yum -y --nogpgcheck localinstall python2-lal
# extract and run the tests
- tar -xf *.tar.* --strip-components 1 lscsoft-glue*/test
- make -C test PYTHON=python2
test:stretch:python3:
test:el7:python3:
stage: test
image: ligo/lalsuite-dev:el7
dependencies:
- package:el7
- build:sdist
variables:
GIT_STRATEGY: none
script:
# install glue
- yum -y --nogpgcheck localinstall python3*glue*.rpm
# install lal
- yum -y install python3-lal
# extract and run the tests
- tar -xf *.tar.* --strip-components 1 lscsoft-glue*/test
- make -C test PYTHON=python3
test:stretch:python2:
stage: test
image: ligo/lalsuite-dev:stretch
dependencies:
- package:stretch
- build:sdist
variables:
GIT_STRATEGY: none
before_script:
- apt-get update
- apt-get install -y lal-python3
- apt-get install -y python-lal python-lalburst
script:
- tar xf *.tar.*
- cd $(find . -type d -maxdepth 1 -name 'lscsoft-glue-*')
- python3 setup.py install
- make -C test PYTHON=python3
- dpkg --install python-*.deb || { apt-get -y -f install; dpkg --install python-*.deb; }
# extract and run the tests
- tar -xf *.tar.* --strip-components 1 --wildcards lscsoft-glue*/test
- make -C test PYTHON=python
deploy:stretch:
stage: deploy
test:stretch:python3:
stage: test
image: ligo/lalsuite-dev:stretch
dependencies:
- package:stretch
- build:sdist
variables:
GIT_STRATEGY: none
before_script:
- apt-get update
- apt-get -y install lintian debhelper python-all-dev python3-all-dev
- apt-get install -y python3-lal python3-lalburst
script:
- tar xf *.tar.*
- ln -s *.tar.* $(echo *.tar.* | sed 's/\(.*\)-\(.*\).\(tar.*\)/\1_\2.orig.\3/')
- cd $(find . -type d -maxdepth 1 -name 'lscsoft-glue-*')
- dpkg-buildpackage -us -uc
- lintian --pedantic ../*.{deb,changes}
- dpkg --install python3-*.deb || { apt-get -y -f install; dpkg --install python3-*.deb; }
# extract and run the tests
- tar -xf *.tar.* --strip-components 1 --wildcards lscsoft-glue*/test
- make -C test PYTHON=python3
docs:
stage: docs
image: python:3.7
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
before_script:
# install docs requirements
- python -m pip install sphinx sphinx_rtd_theme
# install glue and lalsuite (everything else is in setup.py)
- python -m pip install . lalsuite
script:
- glue_version=$(python setup.py --version)
- mkdir -v docs
- cd docs
# generate docs skeleton
- python -m sphinx.ext.apidoc
--module-first
--separate
--full
--ext-autodoc
--ext-intersphinx
--doc-project "lscsoft-glue"
--doc-author "The LIGO Scientific Collaboration and The Virgo Collaboration"
--doc-version "${glue_version}"
--output-dir .
../glue
# use module page as index
- mv -v glue.rst index.rst
# use sphinx_rtd_theme
- sed -i 's/alabaster/sphinx_rtd_theme/g' conf.py
# run sphinx to generate docs
- python -m sphinx -M html . build
artifacts:
paths:
- '*.deb'
- '*.changes'
- docs/build
cache:
paths:
- .cache/pip
deploy:el7:
pages:
stage: deploy
image: ligo/base:el7
variables:
GIT_STRATEGY: none
before_script:
- yum install -y gcc make rpm-build rpmlint yum-utils
dependencies:
- docs
only:
- tags
script:
- mkdir -p ~/rpmbuild/{SOURCES,SPECS}
- tar -C ~/rpmbuild/SPECS -xf *.tar.* --strip-components 2 '*/etc/glue.spec'
- mv *.tar.* ~/rpmbuild/SOURCES
- yum-builddep -y ~/rpmbuild/SPECS/*.spec
- rpmbuild -ba ~/rpmbuild/SPECS/*.spec
- mv $(find ~/rpmbuild/{RPMS,SPECS,SRPMS} -type f) .
- rpmlint *.{rpm,spec}
- mv docs/build/html public
artifacts:
paths:
- '*.rpm'
- '*.spec'
- public
include LICENSE README MANIFEST.in
include LICENSE README.md MANIFEST.in
include glue/ligolw/*.c
include glue/ligolw/*.h
include misc/*.py
include misc/*.py.in
include src/conf/db2/*
include src/conf/s6_db2/*.in
include src/conf/s6_db2/change_creator_db.pl
......@@ -15,16 +16,9 @@ include src/php/dq_report/*
include src/php/seginsert/*
include src/php/seginsert/img/*
include src/php/seginsert/scripts/*
include src/segments/*.c
include src/segments/*.h
include src/*.h
include src/nmi/*
include src/nmi/builds/*
include src/nmi/unit/*
include src/nmi/diff/*
include man/man1/*
include etc/*
include test/*
include debian/*
include debian/source/*
exclude src/nmi/builds/lalsuite-build.spec
GLUE: Grid LSC User Environment
Glue is a collection of utilities for running data analysis pipelines
for online and offline analysis as well as accessing various grid
utilities. It also provides the infrastructure for the segment
database.
The Glue web site:
http://www.lsc-group.phys.uwm.edu/daswg/projects/glue.html
contains many useful links for Glue users and developers.
If you have questions or comments about Glue, please send them to the
lal-discuss mailing list. You may join this list at: There are two
LALApps mail lists/archives which are shared with LAL. These are:
http://www.lsc-group.phys.uwm.edu/mailman/listinfo.cgi/lal-discuss
This README file explains how to build and install Glue.
GLUE is distributed under the GNU General Public, version 3. See the
file LICENSE for more information.
If you use GLUE in published scientific work, we request that you
include a reference in your work as follows:
"Workflows were generated using the Grid LSC User Environment (GLUE)"
------------------------------------------------------------------------------
1. Determine your Python version
Python version 2.7 or greater is required to install Glue. Type:
python -V
to determine the version of python installed on your machine. For
example:
[duncan@contra lalapps]$ python -V
Python 2.7
If you do not have version 2.7 or greater installed, contact your system
administrator for help installing Python.
------------------------------------------------------------------------------
2. Get Glue from Git
Glue is part of the LALSuite Git repository. First choose a directory in
which to install the LALSuite sources. In this example, we install the
sources into ${HOME}/src. If you decide to install somewhere else,
change ${HOME}/src to the directory you have chosen. Change
albert.einstein in the git clone below to your ligo.org kerberos
principle, if you have one. Type:
mkdir -p ${HOME}/src
cd ${HOME}/src
git clone git+ssh://albert.einstein@ligo-vcs.phys.uwm.edu/usr/local/git/lalsuite.git
If you do not have a ligo.org kerberos principle you can get the source
code anonymously using:
git clone git://ligo-vcs.phys.uwm.edu/lalsuite.git
This will create a directory ${HOME}/src/lalsuite containing the
LALSuite sources.
------------------------------------------------------------------------------
3. Build and Install Glue
Choose a location in which to install Glue. In this example we choose
/opt/glue. You must have write access to this directory, so you may wish
to install in ${HOME}, for example. Set the environment variable
${GLUE_LOCATION} to point to your chosen install directory. Bash or sh
users should type:
export GLUE_LOCATION=/opt/glue
and csh users should type:
setenv GLUE_LOCATION /opt/glue
Assuming you have installed the glue sources in ${HOME}/src/glue, type:
cd ${HOME}/src/lalsuite/glue
${PYTHON} setup.py install --prefix=${GLUE_LOCATION}
This will build and install all the glue modules and programs.
------------------------------------------------------------------------------
4. Set up your Environment for Using Glue
If you are a bash or sh user, edit your .profile (or .bash_profile) file
and add the lines (changing /opt/glue to wherever you installed Glue):
export GLUE_LOCATION=/opt/glue
if [ -f ${GLUE_LOCATION}/etc/glue-user-env.sh ] ; then
source ${GLUE_LOCATION}/etc/glue-user-env.sh
fi
If you are a csh user, edit your .login file and add the lines (changing
/opt/glue to wherever you installed Glue):
setenv GLUE_LOCATION /opt/glue
if ( -f ${GLUE_LOCATION}/etc/glue-user-env.csh ) then
source ${GLUE_LOCATION}/etc/glue-user-env.csh
endif
Now log out and log back in to ensure that all environment variables
have been updated.
You may now use Glue! Please report any problems via the Gnats database
linked from the Glue home page.
# GLUE: Grid LSC User Environment
Glue is a collection of utilities for running data analysis pipelines
for online and offline analysis as well as accessing various grid
utilities. It also provides the infrastructure for the segment
database.
The Glue web site:
http://www.lsc-group.phys.uwm.edu/daswg/projects/glue.html
contains many useful links for Glue users and developers.
GLUE is distributed under the GNU General Public, version 3. See the
file LICENSE for more information.
## Installation
### Debian Linux
Glue is distributed for Debian in the [LSCSoft Debian Repositories](https://wiki.ligo.org/DASWG/SoftwareOnDebian) for Jessie, Stretch, and Buster; to install:
```bash
apt-get install python-glue
```
or
```bash
apt-get install python3-glue
```
### Scientific Linux
Glue is distributed for Scientific Linux in the [LSCSoft SL7 Repositories](https://wiki.ligo.org/DASWG/ScientificLinux); to install:
```bash
yum install glue
```
or
```bash
yum install python34-glue
```
### PyPI
Glue is registered on PyPI as [`lscsoft-glue`](https://pypi.org/project/lscsoft-glue); to install:
```bash
python -m pip install lscsoft-glue
```
### Conda
Glue is registered on Conda as [`lscsoft-glue`](https://anaconda.org/conda-forge/lscsoft-glue/); to install:
```bash
conda install -c conda-forge lscsoft-glue
```
This diff is collapsed.
#!/bin/bash
########################################################
# Usage:
# sh VnPublish.sh gps_end_time version_number
# for example:
# sh VnPublish.sh 956000000 2
#
# This script calls VnPublish.py to execute gap checking
#########################################################
clear
# Read in start time from file to use in cache file
. last_et
cache_start_time=$last_end_time
cache_end_time=$1
n=$2 # version to publish
echo "gps times to use in cache file: ["$cache_start_time", "$cache_end_time")"
# make a temp dir
tmp_dir=`mktemp -d`
echo "temp dir is: "$tmp_dir
cd ${tmp_dir}
# retrieve version n data from frame file
# retrieve LHO version n data
ligo_data_find -r ldr.ligo.caltech.edu -s $cache_start_time -e $cache_end_time -o H -t H1_LDAS_C02_L2 -l -u file > H-H1_LDAS_C02_L2-${cache_start_time}-$((${cache_end_time} - ${cache_start_time})).cache || exit 1
ligolw_fr_to_science --cache-file=H-H1_LDAS_C02_L2-${cache_start_time}-$((${cache_end_time} - ${cache_start_time})).cache --ifo=H1 --segment-version=${n} --type=H1_LDAS_C02_L2 || exit 1
# retreive LLO version n data
ligo_data_find -r ldr.ligo.caltech.edu -s $cache_start_time -e $cache_end_time -o L -t L1_LDAS_C02_L2 -l -u file > L-L1_LDAS_C02_L2-${cache_start_time}-$((${cache_end_time} - ${cache_start_time})).cache || exit 1
ligolw_fr_to_science --cache-file=L-L1_LDAS_C02_L2-${cache_start_time}-$((${cache_end_time} - ${cache_start_time})).cache --ifo=L1 --segment-version=${n} --type=L1_LDAS_C02_L2 || exit 1
#--------------------------------------------------------------------------------
# Compare segment_summary table between H1:DMT-SCIENCE:1 and H1:DMT-SCIENCE:n
echo "Checking if H1:DMT-SCIENCE:1 segments are a subset of H1:DMT-SCIENCE:${n} segments ..."
# 4.1 Read in version 1 data from the segment database
ligolw_segment_query --segment-url https://segdb.ligo.caltech.edu --query-segments -s $cache_start_time -e $cache_end_time --include-segments 'H1:DMT-SCIENCE:1' > v1h1_sci_segs.xml || exit 1
ligolw_segment_intersect --segment -i H1:RESULT:1,H1:DMT-SCIENCE:$n v1h1_sci_segs.xml H-H1_LDAS_C02_L2_SEGMENTS*.xml > v1vn_intersect.xml
v1vn_diff=`ligolw_segment_diff --segment -i H1:RESULT:1,:RESULT:1 v1h1_sci_segs.xml v1vn_intersect.xml | ligolw_print -t segment -c start_time -c end_time`
if [ "$v1vn_diff" == "" ]; then
echo "test passed"
else
echo "test failed. Discrepancies are: $v1vn_diff"
exit 1
fi
#--------------------------------------------------------------------------------
# check if gaps in Vn summary file equal to LDAS allowed gap list
echo
echo "Checking if gaps of summary intervals for H1:DMT-SCIENCE:$n are a subset of the LDAS allowed gaps ... "
# retrieve LDAS allowed gaps from the database:
ligolw_segment_query --segment-url https://segdb-dev.phy.syr.edu --query-segments --gps-start-time $cache_start_time --gps-end-time $cache_end_time --include-segments H1:DCH-MISSING_LDAS_C02_L2:1 > ldas_gaps.xml || exit 1
result=($(python -c "import sys; sys.path.append('/archive/home/piwei/hoft/test'); import VnPublish; VnPublish.vn_ldas_gap_check('`echo H-H1_LDAS_C02_L2_SEGMENTS_V2-*.xml`','H1:DMT-SCIENCE:$n','ldas_gaps.xml')")) || exit 1
if [ "$result" != "" ]; then
echo $result;
exit 1
else
echo "test passed"
fi
#--------------------------------------------------------------------------------
# if files passed all the checking, insert Vn data into the segment database
echo
echo "Inserting V$n data into the segment database ..."
ldbdc --segment-url https://segdb-dev.phy.syr.edu --dmt-segments --insert H-H1_LDAS_C02_L2_SEGMENTS_V2-*.xml
error=`echo $?`
if [ $error -ne 0 ]; then
echo "Error inserting LHO data into the segment database"
exit 1
fi
ldbdc --segment-url https://segdb-dev.phy.syr.edu --dmt-segments --insert L-L1_LDAS_C02_L2_SEGMENTS_V2-*.xml
error=`echo $?`
if [ $error -ne 0 ]; then
echo "Error inserting LLO data into the segment database"
exit 1
fi
# update end_time to the .ini file for next time use
`echo "last_end_time=$cache_end_time" > /archive/home/piwei/hoft/H-H1_LDAS_C02/last_et` || exit 1
# copy cache file and xml file to central location
cp *.cache /archive/home/piwei/hoft/H-H1_LDAS_C02 || exit 1
cp H-H1_LDAS_C02_L2_SEGMENTS_V2-*.xml /archive/home/piwei/hoft/H-H1_LDAS_C02 || exit 1
cp L-L1_LDAS_C02_L2_SEGMENTS_V2-*.xml /archive/home/piwei/hoft/H-H1_LDAS_C02 || exit 1
# Remove the tmp dir
cd
rm -rf ${tmp_dir}
echo "Done!"
exit 0
#!/usr/bin/python
#
# Copyright (C) 2011 Kipp Cannon
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# =============================================================================
#
# Preamble
#
# =============================================================================
#
from __future__ import print_function
from optparse import OptionParser
import sys
from glue import git_version
from glue import dagfile
__author__ = "Kipp Cannon <kipp.cannon@ligo.org>"
__version__ = "git id %s" % git_version.id
__date__ = git_version.date
#
# =============================================================================
#
# Command Line
#
# =============================================================================
#
def parse_command_line():
parser = OptionParser(
version = "Name: %%prog\n%s" % git_version.verbose_msg,
usage = "%prog [options] node1 [node2 ...] <old_dag >new_dag",
description = "%prog constructs a DAG to re-run all parents of the DAG nodes named on the command line."
)
parser.add_option("--themselves", "-t", action = "store_true", help = "Rerun the named nodes.")
parser.add_option("--ancestors-of", "-a", action = "store_true", help = "Rerun the ancestors of the named nodes.")
parser.add_option("--descendants-of", "-d", action = "store_true", help = "Rerun the descendents of the named nodes.")
parser.add_option("--verbose", "-v", action = "store_true", help = "Be verbose.")
options, nodenames = parser.parse_args()
#
# check that there's something to do
#
if not (options.themselves or options.ancestors_of or options.descendants_of):
raise ValueError("nothing to do!")
if options.ancestors_of and options.descendants_of and not options.themselves:
raise ValueError("cowardly refusing to rerun both the parents and children of the named nodes without also rerunning the named nodes themselves. must include --themselves when both --ancestors-of and --descendants-of have been selected.")
#
# uniqueify the node names
#
nodenames = set(nodenames)
#
# done
#
return options, nodenames
#
# =============================================================================
#
# Process DAG
#
# =============================================================================
#
#
# command line
#
options, nodenames = parse_command_line()
#
# read original dag from stdin
#
if options.verbose:
def progress(f, n, done):
print("reading original dag from stdin ... %d lines\r" % n, end=' ', file=sys.stderr)
if done:
print(file=sys.stderr)
else:
progress = None
dag = dagfile.DAG.parse(sys.stdin, progress = progress)
if not nodenames.issubset(set(dag.nodes)):
raise ValueError("node(s) %s not found in dag" % ", ".join(sorted(nodenames - set(dag.nodes))))
#
# extract graph
#
if options.verbose:
print("extracting graph ...", file=sys.stderr)
names_to_rerun = set()
if options.ancestors_of:
names_to_rerun |= dag.get_all_parent_names(nodenames)
if options.descendants_of:
names_to_rerun |= dag.get_all_child_names(nodenames)
if options.themselves:
names_to_rerun |= nodenames
assert names_to_rerun # must not be empty
dag = dagfile.DAG.select_nodes_by_name(dag, names_to_rerun)
#
# set nodes matching the requested patterns to done, everything else to not
# done
#
if options.verbose:
print("setting jobs to done/not-done ...", file=sys.stderr)
for nodename, node in dag.nodes.items():
# FIXME: feature not implemented; all nodes marked "not done"
node.done = False
#
# write new dag to stdout
#
if options.verbose:
def progress(f, n, done):
print("writing new dag to stdout ... %d lines\r" % n, end=' ', file=sys.stderr)
if done:
print(file=sys.stderr)
else:
progress = None
dag.write(sys.stdout, progress = progress)
#!/usr/bin/env python
#
# generates and submits a Metronome build of lalsuite from a given git URL and id
from __future__ import print_function
import os
import sys
import commands
import glue.nmi.opts
# define common options (e.g., -v -q -h)
parser = glue.nmi.opts.OptionParserInit()
# define custom options
parser.add_option("-r", "--git-repo", dest="git_repo",
default="git://ligo-vcs.phys.uwm.edu/lalsuite.git",
help="git repository", metavar="URL (git:// or file://)")
parser.add_option("-i", "--git-id", dest="git_id",
help="git id", metavar="ID")
parser.add_option("-b", "--git-branch", dest="git_branch",
help="git branch", metavar="NAME")
parser.add_option("--harness-git-repo", dest="harness_git_repo",
help="build harness git repository",
metavar="URL (git:// or file://)")
parser.add_option("--harness-git-id", dest="harness_git_id",
help="build harness git id", metavar="ID")
parser.add_option("--harness-git-branch", dest="harness_git_branch",
help="build harness git branch", metavar="NAME")
parser.add_option("--template-dir", dest="template_dir",
help="Metronome submit file template location", metavar="DIR")
(options, args) = parser.parse_args()
# copy options to local vars so we can modify them
git_repo = options.git_repo
git_id = options.git_id
git_branch = options.git_branch
harness_git_repo = options.harness_git_repo
harness_git_id = options.harness_git_id
harness_git_branch = options.harness_git_branch
template_dir = options.template_dir
# if unspecified, harness should use the same repo/branch/tag as code
if harness_git_repo is None:
harness_git_repo = git_repo
if harness_git_id is None:
harness_git_id = git_id
if harness_git_branch is None:
harness_git_branch = git_branch
if template_dir is None:
template_dir = "%s/share/nmi" % os.getenv("GLUE_PREFIX")
# sanity-check our options
if git_id is None:
print("ERROR: git_id (-i) required", file=sys.stderr)
sys.exit(2)
if git_branch is None:
print('WARNING: no git_branch specified, using "unknown_branch" in metadata', file=sys.stderr)
git_branch = "unknown_branch"
# TODO: confirm that the given git id exists on the given git branch
# (currently the git branch is unused except as informational
# metadata, so user error can lead to wrong and confusing metadata).
# See <http://stackoverflow.com/questions/2444100> for how to do this.
# propagate relevant options to Metronome via special environment vars
# (this allows us to use a single static Metronome submit file
# template, located in glue/lib/nmi, for all lalsuite builds)
os.environ["_NMI_GIT_REPO"] = git_repo
os.environ["_NMI_GIT_ID"] = git_id
os.environ["_NMI_GIT_BRANCH"] = git_branch
os.environ["_NMI_HARNESS_GIT_REPO"] = harness_git_repo
os.environ["_NMI_HARNESS_GIT_ID"] = harness_git_id
os.environ["_NMI_HARNESS_GIT_BRANCH"] = harness_git_branch
os.environ["_NMI_HARNESS_INPUT_SPEC_FILE"] = "%s/lalsuite-build-scripts.location" % template_dir
if options.verbose: os.system("env|fgrep _NMI|sort")
# we need to tell Metronome how & where to find the build harness code
# depending on whether it was specified as a git:// or file:// URL
if 'git://' not in harness_git_repo and 'file://' not in harness_git_repo:
print("ERROR: invalid harness repository URL (\"%s\"): only git:// and file:// schemes are currently supported" % harness_git_repo, file=sys.stderr)
sys.exit(2)
(status, output) = commands.getstatusoutput("nmi_submit %s/lalsuite-build.spec" % template_dir)
print(output)
sys.exit(status)
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/python
#
# Copyright (C) 2006 Kipp Cannon
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# =============================================================================
#
# Preamble
#
# =============================================================================
#
"""
Add (merge) LIGO LW XML files containing LSC tables.
"""
from optparse import OptionParser
import os
import sys
from glue import git_version
from glue.lal import CacheEntry
from glue.ligolw import ligolw
from glue.ligolw import array as ligolw_array
from glue.ligolw import lsctables
from glue.ligolw import utils as ligolw_utils
from glue.ligolw.utils import ligolw_add
__author__ = "Kipp Cannon <kipp.cannon@ligo.org>"
__version__ = "git id %s" % git_version.id
__date__ = git_version.date
#
# =============================================================================
#
# Command Line
#
# =============================================================================
#
def parse_command_line():
"""
Parse the command line, return an options object and a list of URLs.
"""
parser = OptionParser(
version = "Name: %%prog\n%s" % git_version.verbose_msg,
usage = "%prog [options] [url ...]",
description = "Combines one or more LIGO Light Weight XML files into a single output file. The output is written to stdout or to the filename specified by --output. In addition to regular files, many common URL types can be read such as http:// and ftp://. Input documents that are gzip-compressed are automatically detected and decompressed. If the output file's name ends in \".gz\", the output document will be gzip-compressed. Table elements contained in the document will be merged so that there is not more than one table of any given name in the output. To accomplish this, any tables in the input documents that share the same name must have compatible columns, meaning the same column names with matching types (but not necessarily in the same order)."
)
parser.add_option("-i", "--input-cache", metavar = "filename", action = "append", default = [], help = "Get input files from the LAL cache named filename.")
parser.add_option("--add-lfn-table", action = "store_true", help = "Add an lfn entry for each process.")
parser.add_option("--lfn-start-time", metavar = "GPS seconds", help = "Set lfn start_time (optional).")
parser.add_option("--lfn-end-time", metavar = "GPS seconds", help = "Set lfn end_time (optional).")
parser.add_option("--lfn-comment", metavar = "string", help = "Set lfn comment (optional).")
parser.add_option("--non-lsc-tables-ok", action = "store_true", help = "OK to merge documents containing non-LSC tables.")
parser.add_option("-o", "--output", metavar = "filename", help = "Write output to filename (default = stdout).")
parser.add_option("-v", "--verbose", action = "store_true", help = "Be verbose.")
parser.add_option("--remove-input", action = "store_true", help = "Remove input files after writing output (an attempt is made to not delete the output file in the event that it overwrote one of the input files).")
parser.add_option("--remove-input-except", metavar = "filename", action = "append", default = [], help = "When deleting input files, do not delete this file.")
options, urls = parser.parse_args()
if options.lfn_start_time:
options.lfn_start_time = int(options.lfn_start_time)
if options.lfn_end_time:
options.lfn_end_time = int(options.lfn_end_time)
urls += [CacheEntry(line).url for cache in options.input_cache for line in open(cache)]
if len(urls) < 1:
raise ValueError("no input files!")
return options, urls
#
# =============================================================================
#
# LFN Table
#
# =============================================================================
#
def update_lfn_table(xmldoc, pathname, start_time = None, end_time = None, comment = None):
"""
Update the LFN table in the document, adding it if needed.
"""
# determine the columns required by the user
cols = ["process_id", "lfn_id", "name"]
if start_time is not None:
cols.append("start_time")
if end_time is not None:
cols.append("end_time")
if comment is not None:
cols.append("comment")
# look for an existing LFN table. while at it, sync the next_id
# attribute and collect the process IDs that already have entries
# in the tables
lfn_table = None
existing_pids = set()
for lfn_table in lsctables.LfnTable.getTablesByName(xmldoc, lsctables.LfnTable.tableName):
lfn_table.sync_next_id()
existing_pids.update(lfn_table.getColumnByName("process_id"))
if lfn_table is None:
# didn't find an LFN table, add one
lfn_table = lsctables.New(lsctables.LfnTable, cols)
xmldoc.getElementsByTagName(ligolw.LIGO_LW.tagName)[0].appendChild(lfn_table)
else:
# already has an LFN table, are the columns OK?
if set(cols) != set(lfn_table.columnnames):
raise Exception("document contains an LFN table with columns %s, but columns %s are required" % (", ".join(lfn_table.columnnames), ", ".join(cols)))
pathname = os.path.basename(pathname)
# add a row to the LFN table for every process in every process
# table
for process_table in lsctables.ProcessTable.getTablesByName(xmldoc, lsctables.ProcessTable.tableName):
for pid in process_table.getColumnByName("process_id"):
if pid in existing_pids:
# LFN table already has an entry for this
# one, ignore
continue
row = lsctables.Lfn()
row.process_id = pid
row.lfn_id = lfn_table.get_next_id()
row.name = pathname
row.start_time = start_time
row.end_time = end_time
row.comment = comment
lfn_table.append(row)
return lfn_table
#
# =============================================================================
#
# Main
#
# =============================================================================
#
#
# Command line
#
options, urls = parse_command_line()
#
# Input
#
class ContentHandler(ligolw.LIGOLWContentHandler):
pass
ligolw_array.use_in(ContentHandler)
lsctables.use_in(ContentHandler)
lsctables.table.TableStream.RowBuilder = lsctables.table.InterningRowBuilder
xmldoc = ligolw_add.ligolw_add(ligolw.Document(), urls, non_lsc_tables_ok = options.non_lsc_tables_ok, verbose = options.verbose, contenthandler = ContentHandler)
#
# LFN table
#
if options.add_lfn_table:
if not options.output:
raise Exception("cannot add LFN table when no output filename is given")
update_lfn_table(xmldoc, options.output, options.lfn_start_time, options.lfn_end_time, options.lfn_comment)
#
# Output
#
ligolw_utils.write_filename(xmldoc, options.output, verbose = options.verbose, gz = (options.output or "stdout").endswith(".gz"))
#
# Remove input
#
if options.remove_input:
ligolw_add.remove_input(urls, [options.output] + options.remove_input_except, options.verbose)
......@@ -27,7 +27,7 @@ import os
import glob
import tempfile
from glue.segments import segmentlist, segment
from ligo.segments import segmentlist, segment
from glue.ligolw import ligolw
from glue.ligolw import lsctables
......
......@@ -23,8 +23,8 @@ segment list that represents the intersection.
from __future__ import print_function
import copy
from optparse import OptionParser
from ligo.segments import segmentlist
from glue.ligolw import ligolw, lsctables, utils
from glue.segments import segmentlist
from glue.ligolw.utils import process, segments, ligolw_add
from glue import git_version
......
#!/usr/bin/python
#
# Copyright (C) 2006 Kipp Cannon
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# =============================================================================
#
# Preamble
#
# =============================================================================
#
"""
Cut pieces out of LIGO LW XML files containing LSC tables.
"""
from __future__ import print_function
from optparse import OptionParser
import sys
from glue import git_version
from glue.ligolw import ligolw
from glue.ligolw import table
from glue.ligolw import utils
__author__ = "Kipp Cannon <kipp.cannon@ligo.org>"
__version__ = "git id %s" % git_version.id
__date__ = git_version.date
#
# =============================================================================
#
# Command Line
#
# =============================================================================
#
def parse_command_line():
parser = OptionParser(
version = "Name: %%prog\n%s" % git_version.verbose_msg,
usage = "%prog [options] [file ...]",
description = "%prog removes XML elements from a LIGO Light Weight XML file. If file names are given on the command line, those files are read, processed, and rewritten one at a time, otherwise input is read from stdin and output written to stdout. Gzipped files are automatically detected on input, if the file's name ends in \".gz\" it will be gzip-compressed when written."
)
parser.add_option("-v", "--verbose", action = "store_true", help = "Be verbose.")
parser.add_option("--delete-column", metavar = "name", action = "append", default = [], help = "Delete matching columns. Example \"--delete-column sim_burst:dtminus\".")
parser.add_option("--delete-element", metavar = "tag[,attr=value[,...]]", action = "append", default = [], help = "Delete matching elements. Example \"--delete-element Table,Name=search_summary\".")
parser.add_option("--delete-table", metavar = "name", action = "append", default = [], help = "Delete matching tables. Example \"--delete-table search_summary\".")
options, filenames = parser.parse_args()
# strip column names
options.delete_column = set(map(table.Column.ColumnName, options.delete_column))
# parse element specs
def parse_delete_element(arg):
arg = arg.split(",")
return arg[0], tuple(a.split("=") for a in arg[1:])
options.delete_element = set(map(parse_delete_element, options.delete_element))
# strip table names
options.delete_table = set(map(table.Table.TableName, options.delete_table))
return options, (filenames or [None])
#
# =============================================================================
#
# Input
#
# =============================================================================
#
class ElementFilter(object):
"""
Class implementing any cuts that can be performed during document
parsing.
"""
def __init__(self, delete_tables):
self.delete_tables = delete_tables
def element_filter(self, name, attrs):
# check for unwanted tables
return name != ligolw.Table.tagName or table.Table.TableName(attrs["Name"]) not in self.delete_tables
class ContentHandler(ligolw.FilteringLIGOLWContentHandler):
def __init__(self, xmldoc):
super(ContentHandler, self).__init__(xmldoc, ElementFilter(options.delete_table).element_filter)
table.use_in(ContentHandler)
#
# Enable attribute interning
#
table.TableStream.RowBuilder = table.InterningRowBuilder
#
# =============================================================================
#
# Cut
#
# =============================================================================
#
#
# Remove unwanted columns
#
def RemoveColumns(doc, columns):
for table_elem in doc.getElementsByTagName(ligolw.Table.tagName):
for name in columns:
for column in table.Column.getColumnsByName(table_elem, name):
table_elem.removeChild(column)
#
# Remove unwanted elements
#
def CompareDeleteElement(elem, name, attrvalues):
"""
Return 1 for !=, 0 for ==.
"""
if elem.tagName != name:
return 1
for attr, value in attrvalues:
try:
if elem.getAttribute(attr) != value:
return 1
except KeyError:
return 1
return 0
def IsDeleteElement(elem, specs):
"""
Return True if elem matches an elem spec for deleting.
"""
for name, attrvalues in specs:
if CompareDeleteElement(elem, name, attrvalues) == 0:
return True
return False
def RemoveElements(doc, specs):
for elem in doc.getElements(lambda e: IsDeleteElement(e, specs)):
elem.parentNode.removeChild(elem)
#
# =============================================================================
#
# Main
#
# =============================================================================
#
options, filenames = parse_command_line()
for n, filename in enumerate(filenames):
if options.verbose:
print("%d/%d:" % (n + 1, len(filenames)), end=' ', file=sys.stderr)
doc = utils.load_filename(filename, verbose = options.verbose, contenthandler = ContentHandler)
table.InterningRowBuilder.strings.clear()
if options.verbose:
print("processing", file=sys.stderr)
if options.delete_column:
RemoveColumns(doc, options.delete_column)
if options.delete_element:
RemoveElements(doc, options.delete_element)
utils.write_filename(doc, filename, verbose = options.verbose, gz = (filename or "stdout").endswith(".gz"))
doc.unlink()
......@@ -62,8 +62,6 @@ import tempfile
import urllib
import pwd
import glue.segments
from glue.ligolw import ligolw
from glue.ligolw import table
from glue.ligolw import lsctables
......
......@@ -30,9 +30,9 @@ import pwd
from optparse import OptionParser
from ligo import segments
from glue import lal
from glue import segments
from glue import segmentsUtils
from ligo.segments import utils as segmentsUtils
from glue import gpstime
import glue.pipeline
......@@ -111,7 +111,7 @@ class diskcacheEntry(object):
# group the integers by two and turn those tuples into segments
assert len(times) % 2 is 0
self.segments = glue.segments.segmentlist([ glue.segments.segment(t) for t in zip(times[::2],times[1::2]) ])
self.segments = segments.segmentlist([ segments.segment(t) for t in zip(times[::2],times[1::2]) ])
except Exception as e:
msg = "Error parsing frame cache file with line %s: %s" % (line, e)
......@@ -127,9 +127,9 @@ class diskcacheEntry(object):
"""
if self.__frame_segments__:
return self.__frame_segments__
self.__frame_segments__ = glue.segments.segmentlist([])
self.__frame_segments__ = segments.segmentlist([])
for seg in self.segments:
self.__frame_segments__.extend(glue.segmentsUtils.segmentlist_range(seg[0], seg[1], self.duration))
self.__frame_segments__.extend(segmentsUtils.segmentlist_range(seg[0], seg[1], self.duration))
return self.__frame_segments__
......
#!/usr/bin/python
#
# Copyright (C) 2006 Kipp Cannon
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# =============================================================================
#
# Preamble
#
# =============================================================================
#
"""
Print things from LIGO LW XML files. Inspired by lwtprint from LIGOTools.
"""
from __future__ import print_function
from optparse import OptionParser
from glue import git_version
from glue import segmentsUtils
from glue.lal import CacheEntry
from glue.ligolw import ligolw
from glue.ligolw import table
from glue.ligolw import lsctables
from glue.ligolw import array
from glue.ligolw import utils
from glue import git_version
import six
__author__ = "Kipp Cannon <kipp.cannon@ligo.org>"
__version__ = "git id %s" % git_version.id
__date__ = git_version.date
#
# =============================================================================
#
# Command Line
#
# =============================================================================
#
def parse_command_line():
parser = OptionParser(
version = "Name: %%prog\n%s" % git_version.verbose_msg,
usage = "%prog [options] [url ...]",
description = "Prints the contents of table elements from one or more LIGO Light Weight XML files to stdout in delimited ASCII format. In addition to regular files, the program can read from many common URLs such as http:// and ftp://. Gzipped files will be automatically detected and decompressed. If no filenames or URLs are given, then input is read from stdin."
)
parser.add_option("-i", "--input-cache", metavar = "name", action = "append", default = [], help = "Get URLs from the LAL cache file. Can be provided multiple times to name several caches to iterate over.")
parser.add_option("-c", "--column", metavar = "name", action = "append", help = "Print only the contents of the given column. Can be provided multiple times to print multiple columns. The default is to print all columns from each table.")
parser.add_option("-d", "--delimiter", metavar = "string", default = ",", help = "Delimit output with the given string. The default is \",\".")
parser.add_option("-r", "--rows", metavar = "rowspec", default = ":", help = "Print rows in the given range(s). The format is first:last[,first:last,...]. Rows are numbered from 0. A single first:last pair requests rows in the range [first, last). If the first or last value of a pair is omited it means 0 or infinity respectively. The default is \":\", or to print all rows.")
parser.add_option("-t", "--table", metavar = "name", action = "append", default = [], help = "Print rows from this table. Can be provided multiple times to print rows from multiple tables. The default is to print the contents of all tables.")
parser.add_option("-a", "--array", metavar = "name", action = "append", default = [], help = "Print the contents of this array. Can be provided multiple times to print the elements from multiple arrays. The default is to print the contents of all arrays.")
parser.add_option("-v", "--verbose", action = "store_true", help = "Be verbose.")
parser.add_option("--constrain-lsc-tables", action = "store_true", help = "Impose additional constraints on official LSC tables. Provides format validation and allows RAM requirements to be reduced.")
options, urls = parser.parse_args()
# Ensure the delimiter is unicode compliant as well
options.delimiter = six.text_type(options.delimiter)
# add urls from cache files
urls += [CacheEntry(line).url for cache in options.input_cache for line in open(cache)]
# strip table names
if options.table:
options.table = list(map(table.Table.TableName, options.table))
if options.array:
options.array = list(map(array.Array.ArrayName, options.array))
# turn row requests into a segment list
try:
options.rows = segmentsUtils.from_range_strings(options.rows.split(","))
except ValueError as e:
raise ValueError("invalid rowspec: %s" % str(e))
# success
return options, (urls or [None])
#
# =============================================================================
#
# Input
#
# =============================================================================
#
#
# Enable column interning
#
table.TableStream.RowBuilder = table.InterningRowBuilder
#
# =============================================================================
#
# How to find things to print
#
# =============================================================================
#
#
# How to print a table
#
def print_table(table_elem, columns, rows):
if not columns:
columns = table_elem.columnnames