...
 
Commits (179)
# This is a Dockerfile to build an EL7 MKL-optimized GstLAL
# runtime container.
FROM containers.ligo.org/alexander.pace/gstlal-dev/gstlal-dev:el7
# Labeling/packaging stuff:
LABEL name="GstLAL Runtime Package, EL7" \
maintainer="Alexander Pace <alexander.pace@ligo.org>" \
date="2019-10-16" \
support="Reference Platform, EL7"
USER root
## Copy Optimized RPMs to container
COPY rpms /rpms
# Install Optimized RPMs, delete old RPMs
RUN yum makecache && \
yum -y localinstall /rpms/*.rpm
# Update gstreamer registry cache:
ENV GST_REGISTRY_1_0=/usr/share/gstreamer-1.0/.registry.bin
RUN gst-inspect-1.0
# Clean up and close-out
#RUN rm -rf /rpms && \
RUN yum clean all
# Export MKL environment variables:
ENV MKL_INTERFACE_LAYER LP64
ENV MKL_THREADING_LAYER SEQUENTIAL
ENV GSTLAL_FIR_WHITEN 0
ENV TMPDIR /tmp
ENTRYPOINT source /opt/intel/bin/compilervars.sh intel64 && \
#source /opt/rh/devtoolset-7/enable && \
bash
This diff is collapsed.
# GstLAL Project
![gstlal](https://git.ligo.org/lscsoft/gstlal/raw/master/doc/source/_static/gstlal_small.png "gstlal")
Welcome to the GstLAL project. Please find doc pages here: https://lscsoft.docs.ligo.org/gstlal/
\ No newline at end of file
GStreamer elements for gravitational-wave data analysis
==================================================================
[![pipeline status](https://git.ligo.org/lscsoft/gstlal/badges/master/pipeline.svg)](https://git.ligo.org/lscsoft/gstlal/commits/master)
| | version | source |
| :------------:| :------: | :------: |
| `gstlal` | 1.5.0 | [gstlal-1.5.0.tar.gz](http://software.ligo.org/lscsoft/source/gstlal-1.5.0.tar.gz) |
| `gstlal-ugly` | 1.6.4 | [gstlal-ugly-1.6.4.tar.gz](http://software.ligo.org/lscsoft/source/gstlal-ugly-1.6.4.tar.gz) |
| `gstlal-inspiral` | 1.6.5 | [gstlal-inspiral-1.6.5.tar.gz](http://software.ligo.org/lscsoft/source/gstlal-inspiral-1.6.5.tar.gz) |
| `gstlal-calibration` | 1.2.10 | [gstlal-calibration-1.2.10.tar.gz](http://software.ligo.org/lscsoft/source/gstlal-calibration-1.2.10.tar.gz) |
| `gstlal-burst` | 0.1.1 | [gstlal-burst-0.1.1.tar.gz](http://software.ligo.org/lscsoft/source/gstlal-burst-0.1.1.tar.gz) |
Full documentation is provided [here](https://lscsoft.docs.ligo.org/gstlal/).
**GstLAL** provides a suite of GStreamer elements that expose gravitational-wave data analysis tools from the LALSuite library for use in GStreamer signal-processing pipelines.
Examples include an element to add simulated gravitational waves to an h(t) stream, and a source element to provide the contents of .gwf frame files to a GStreamer pipeline.
Overview
The **GstLAL** software package is used for the following activities:
* GstLAL: Provides core Gstreamer plugins for signal processing workflows with LIGO data and core python bindings for constructing such workflows.
* GstLAL Calibration: Provides real-time calibration of LIGO control system data into strain data.
* GstLAL Inspiral: Provides additional signal processing plugins that are specific for LIGO / Virgo searches for compact binaries as well as a substantial amount of python code for post-processing raw signal processing results into gravitational wave candidate lists. Several publications about the methodology and workflow exist, see publications
* GstLAL Ugly: An incubator project for gradual inclusion in the other packages.
......@@ -3,33 +3,43 @@ gstlal-modules :
#mv ../gstlal/python/__init__.py ../gstlal/python/__init__.py.bk
sphinx-apidoc -e -o source/gstlal/python-modules ../gstlal/python ../gstlal/python/misc.py ../gstlal/python/bottle.py ../gstlal/python/coherent_null.py ../gstlal/python/matplotlibhelper.py
#mv ../gstlal/python/__init__.py.bk ../gstlal/python/__init__.py
sed -i "1s/python/\`\`gstlal\`\`/" source/gstlal/python-modules/modules.rst
gstlal-inspiral-modules :
sphinx-apidoc -e -o source/gstlal-inspiral/python-modules ../gstlal-inspiral/python ../gstlal-inspiral/python/cbc_template_iir.py ../gstlal-inspiral/python/spawaveform.py ../gstlal-inspiral/python/emcee.py ../gstlal-inspiral/python/spiirparts.py ../gstlal-inspiral/python/snglinspiraltable.py
sed -i "1s/python/\`\`gstlal-inspiral\`\`/" source/gstlal-inspiral/python-modules/modules.rst
gstlal-calibration-modules :
sphinx-apidoc -e -o source/gstlal-calibration/python-modules ../gstlal-calibration/python
sed -i "1s/python/\`\`gstlal-calibration\`\`/" source/gstlal-calibration/python-modules/modules.rst
gstlal-burst-modules :
sphinx-apidoc -e -o source/gstlal-burst/python-modules ../gstlal-burst/python ../gstlal-burst/python/excesspower ../gstlal-burst/python/fxtools/sngltriggertable.py
sed -i "1s/python/\`\`gstlal-burst\`\`/" source/gstlal-burst/python-modules/modules.rst
gstlal-ugly-modules :
sphinx-apidoc -e -o source/gstlal-ugly/python-modules ../gstlal-ugly/python ../gstlal-ugly/python/gviz_api.py ../gstlal-ugly/python/coherent_inspiral_metric.py ../gstlal-ugly/python/coherent_inspiral_metric_plots.py ../gstlal-ugly/python/coherent_inspiral_metric_detector_details.py ../gstlal-ugly/python/aggregator.py
sed -i "1s/python/\`\`gstlal-ugly\`\`/" source/gstlal-ugly/python-modules/modules.rst
gstlal-bin :
./sphinx-bindoc source/gstlal/bin ../gstlal/bin gstlal_ligo_data_find_check
sed -i "1s/bin/\`\`gstlal\`\`/" source/gstlal/bin/bin.rst
gstlal-inspiral-bin :
./sphinx-bindoc source/gstlal-inspiral/bin ../gstlal-inspiral/bin gstlal_inspiral_flopulator gstlal_inspiral_lvalert_omegascan gstlal_ll_inspiral_calculate_range gstlal_ll_inspiral_daily_page_online gstlal_iir_inspiral gstlal_inspiral_iir_bank_pipe gstlal_ll_inspiral_save_state gstlal_inspiral_marginalize_likelihoods_online gstlal_inspiral_plot_extrinsic_params gstlal_ll_inspiral_get_urls gstlal_inspiral_plot_svd_bank gstlal_inspiral_lvalert_background_plotter
sed -i "1s/bin/\`\`gstlal-inspiral\`\`/" source/gstlal-inspiral/bin/bin.rst
gstlal-calibration-bin :
./sphinx-bindoc source/gstlal-calibration/bin ../gstlal-calibration/bin
sed -i "1s/bin/\`\`gstlal-calibration\`\`/" source/gstlal-calibration/bin/bin.rst
gstlal-burst-bin :
./sphinx-bindoc source/gstlal-burst/bin ../gstlal-burst/bin gstlal_excesspower gstlal_excesspower_trigvis
sed -i "1s/bin/\`\`gstlal-burst\`\`/" source/gstlal-burst/bin/bin.rst
gstlal-ugly-bin :
./sphinx-bindoc source/gstlal-ugly/bin ../gstlal-ugly/bin gstlal_h1h2_coh_frames gstlal_geo_player gstlal_ll_inspiral_daily_summary_page gstlal_cache_to_segments gstlal_inspiral_plot_efficiency gstlal_burst_injections_by_rate_plot gstlal_display_from_logs gstlal_inspiral_bankviz gstlal_ll_inspiral_regen_reg_from_condor_q gstlal_cbc_sensitivity_vs_flow_at_fixed_computational_cost gstlal_inspiral_check_livetimes gstlal_rds_gen gstlal_injections_by_local_rate_plot gstlal_ninja_smooth_reference_psd gstlal_shift_frames
sed -i "1s/bin/\`\`gstlal-ugly\`\`/" source/gstlal-ugly/bin/bin.rst
html : gstlal-modules gstlal-inspiral-modules gstlal-calibration-modules gstlal-ugly-modules gstlal-burst-modules gstlal-bin gstlal-inspiral-bin gstlal-calibration-bin gstlal-ugly-bin gstlal-burst-bin
sphinx-build -b html source _build
......
/* Uses skorch docs theme from https://skorch.readthedocs.io */
.wy-side-nav-search {
background-color: #fcfcfc;
}
.wy-side-nav-search > div.version {
color: black;
}
.wy-side-nav-search > a {
color: black;
}
/* Adds logo in header for mobile */
.wy-nav-top {
background-color: #fcfcfc;
background-image: url('../logo.svg');
background-repeat: no-repeat;
background-position: bottom;
padding: 0;
margin: 0.4045em 0.809em;
color: #333;
}
.wy-nav-top > a {
display: none;
}
{%- set logo = "gstlal.png" %}
{% extends "!layout.html" %}
GstLAL API
============
.. toctree::
:maxdepth: 2
:glob:
gstlal/python-modules/*modules
gstlal-inspiral/python-modules/*modules
gstlal-calibration/python-modules/*modules
gstlal-burst/python-modules/*modules
gstlal-ugly/python-modules/*modules
.. _cbc-analysis:
CBC Analysis
================
WRITEME
......@@ -25,6 +25,9 @@ sys.path.insert(0, os.path.abspath('../../gstlal-burst/python'))
sys.path.insert(0, os.path.abspath('../../gstlal-calibration/python'))
sys.path.insert(0, os.path.abspath('../../gstlal-ugly/python'))
# on_rtd is whether we are on readthedocs.org, this line of code grabbed
# from docs.readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
# -- General configuration ------------------------------------------------
......@@ -35,12 +38,13 @@ sys.path.insert(0, os.path.abspath('../../gstlal-ugly/python'))
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc',
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.autosummary',
'sphinx.ext.intersphinx',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.imgmath',
'sphinx.ext.pngmath',
'sphinx.ext.ifconfig',
'sphinx.ext.viewcode',
'sphinx.ext.githubpages',
......@@ -98,28 +102,29 @@ todo_include_todos = True
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'#'classic'
html_logo = "gstlal_small.png"
html_theme = 'default'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
html_theme_options = {
'fixed_sidebar': 'true',
'sidebar_width': '200px',
'page_width': '95%',
'show_powered_by': 'false',
'logo_name': 'true',
}
#html_theme_options = {}
def setup(app):
app.add_stylesheet('css/my_theme.css')
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
if not on_rtd: # only import and set the theme if we're building docs locally
import sphinx_rtd_theme
html_theme = 'sphinx_rtd_theme'
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# Custom sidebar templates, maps document names to template names.
html_sidebars = { '**': ['navigation.html', 'relations.html', 'searchbox.html'] }
html_last_updated_fmt = None
#html_sidebars = { '**': ['navigation.html', 'relations.html', 'searchbox.html'] }
#html_last_updated_fmt = None
# -- Options for HTMLHelp output ------------------------------------------
......
Executables
===============
.. toctree::
:maxdepth: 2
gstlal/bin/bin
gstlal-inspiral/bin/bin
gstlal-calibration/bin/bin
gstlal-burst/bin/bin
gstlal-ugly/bin/bin
.. _fake-data:
Fake Data Generation
=========================
WRITEME
####################################################################################################
Overview
####################################################################################################
.. _burst-overview-feature_extraction:
.. _feature_extraction:
Feature Extraction
====================================================================================================
The `fxtools` module and related feature-based executables contain relevant libraries to identify
glitches in low-latency using auxiliary channel data.
SNAX (Signal-based Noise Acquisition and eXtraction), the `snax` module and related SNAX executables
contain relevant libraries to identify glitches in low-latency using auxiliary channel data.
`gstlal_feature_extractor` functions as a modeled search for data quality by applying matched filtering
SNAX functions as a modeled search for data quality by applying matched filtering
on auxiliary channel timeseries using waveforms that model a large number of glitch classes. Its primary
purpose is to whiten incoming auxiliary channels and extract relevant features in low-latency.
There are two different modes of output `gstlal_feature_extractor` can function in:
.. _feature_extraction-intro:
Introduction
------------
There are two different modes of feature generation:
1. **Timeseries:**
1. **Timeseries:** Production of regularly-spaced feature rows, containing the SNR, waveform parameters,
and the time of the loudest event in a sampling time interval.
2. **ETG:** This produces output that resembles that of a traditional event trigger generator (ETG), in
which only feature rows above an SNR threshold will be produced.
Production of regularly-spaced feature rows, containing the SNR, waveform parameters,
and the time of the loudest event in a sampling time interval.
2. **ETG:**
This produces output that resembles that of a traditional event trigger generator (ETG), in
which only feature rows above an SNR threshold will be produced.
One useful feature in using a matched filter approach to detect glitches is the ability to switch between
different glitch templates or generate a heterogeneous bank of templates.. Currently, there are Sine-Gaussian
and half-Sine-Gaussian waveforms implemented for use in detecting glitches, but the feature extractor was
designed to be fairly modular and so it isn't difficult to design and add new waveforms for use.
different glitch templates or generate a heterogeneous bank of templates. Currently, there are Sine-Gaussian,
half-Sine-Gaussian, and tapered Sine-Gaussian waveforms implemented for use in detecting glitches, but the feature
extractor is designed to be fairly modular and so it isn't difficult to design and add new waveforms for use.
Since the GstLAL feature extractor uses time-domain convolution to matched filter auxiliary channel timeseries
Since SNAX uses time-domain convolution to matched filter auxiliary channel timeseries
with glitch waveforms, this allows latencies to be much lower than in traditional ETGs. The latency upon writing
features to disk are O(5 s) in the current layout when using waveforms where the peak occurs at the edge of the
template (zero-latency templates). Otherwise, there is extra latency incurred due to the non-causal nature of
......@@ -36,7 +42,7 @@ the waveform itself.
digraph llpipe {
labeljust = "r";
label="gstlal_feature_extractor"
label="gstlal_snax_extract"
rankdir=LR;
graph [fontname="Roman", fontsize=24];
edge [ fontname="Roman", fontsize=10 ];
......@@ -132,17 +138,19 @@ the waveform itself.
}
.. _feature_extraction-highlights:
**Highlights:**
Highlights
----------
* Launch feature extractor jobs in online or offline mode:
* Launch SNAX jobs in online or offline mode:
* Online: Using /shm or framexmit protocol
* Offline: Read frames off disk
* Online/Offline DAGs available for launching jobs.
* Offline DAG parallelizes by time, channels are processed sequentially by subsets to reduce I/O concurrency issues. There are options to allow flexibility in choosing this, however.
* Offline DAG parallelizes by time, channels are processed sequentially by subsets to reduce I/O concurrency issues.
* On-the-fly PSD generation (or take in a prespecified PSD)
......@@ -165,3 +173,73 @@ the waveform itself.
* Waveform type (currently Sine-Gaussian and half-Sine-Gaussian only)
* Specify parameter ranges (frequency, Q for Sine-Gaussian based)
* Min mismatch between templates
.. _feature_extraction-online:
Online Operation
----------------
An online DAG is provided in /gstlal-burst/share/snax/Makefile.gstlal_feature_extractor_online
in order to provide a convenient way to launch online feature extraction jobs as well as auxiliary jobs as
needed (synchronizer/hdf5 file sinks). A condensed list of instructions for use is also provided within the Makefile itself.
There are four separate modes that can be used to launch online jobs:
1. Auxiliary channel ingestion:
a. Reading from framexmit protocol (DATA_SOURCE=framexmit).
This mode is recommended when reading in live data from LHO/LLO.
b. Reading from shared memory (DATA_SOURCE=lvshm).
This mode is recommended for reading in data for O2 replay (e.g. UWM).
2. Data transfer of features:
a. Saving features directly to disk, e.g. no data transfer.
This will save features to disk directly from the feature extractor,
and saves features periodically via hdf5.
b. Transfer of features via Kafka topics.
This requires a Kafka/Zookeeper service to be running (can be existing LDG
or your own). Features get transferred via Kafka from the feature extractor,
parallel instances of the extractor get synchronized, and then sent downstream
where it can be read by other processes (e.g. iDQ). In addition, an streaming
hdf5 file sink is launched where it'll dump features periodically to disk.
In order to start up online runs, you'll need an installation of gstlal. An installation Makefile that
includes Kafka dependencies are located at: gstlal/gstlal-burst/share/feature_extractor/Makefile.gstlal_idq_icc
To run, making sure that the correct environment is sourced:
$ make -f Makefile.gstlal_feature_extractor_online
Then launch the DAG with:
$ condor_submit_dag feature_extractor_pipe.dag
.. _feature_extraction-offline:
Offline Operation
-----------------
An offline DAG is provided in /gstlal-burst/share/snax/Makefile.gstlal_feature_extractor_offline
in order to provide a convenient way to launch offline feature extraction jobs. A condensed list of
instructions for use is also provided within the Makefile itself.
For general use cases, the only configuration options that need to be changed are:
* User/Accounting tags: GROUP_USER, ACCOUNTING_TAG
* Analysis times: START, STOP
* Data ingestion: IFO, CHANNEL_LIST
* Waveform parameters: WAVEFORM, MISMATCH, QHIGH
In order to start up offline runs, you'll need an installation of gstlal. An installation Makefile that
includes Kafka dependencies are located at: gstlal/gstlal-burst/share/feature_extractor/Makefile.gstlal_idq_icc
To generate a DAG, making sure that the correct environment is sourced:
$ make -f Makefile.gstlal_feature_extractor_offline
Then launch the DAG with:
$ condor_submit_dag feature_extractor_pipe.dag
GstLAL burst code
=================
.. toctree::
:maxdepth: 2
bin/bin
python-modules/modules
####################################################################################################
GstLAL burst
####################################################################################################
`GstLAL burst` contains several projects targeting a variety of different searches. These include:
* **Feature extraction:** Identify noise transient bursts (glitches) in auxiliary channel data.
* **Cosmic string search**
* **Excess power**
Contents
-------------------------
.. toctree::
:maxdepth: 2
overview
tutorials/tutorials
code
####################################################################################################
Running Offline Jobs
####################################################################################################
An offline DAG is provided in /gstlal-burst/share/feature_extractor/Makefile.gstlal_feature_extractor_offline
in order to provide a convenient way to launch offline feature extraction jobs. A condensed list of
instructions for use is also provided within the Makefile itself.
For general use cases, the only configuration options that need to be changed are:
* User/Accounting tags: GROUP_USER, ACCOUNTING_TAG
* Analysis times: START, STOP
* Data ingestion: IFO, CHANNEL_LIST
* Waveform parameters: WAVEFORM, MISMATCH, QHIGH
Launching DAGs
====================================================================================================
In order to start up offline runs, you'll need an installation of gstlal. An installation Makefile that
includes Kafka dependencies are located at: gstlal/gstlal-burst/share/feature_extractor/Makefile.gstlal_idq_icc
To generate a DAG, making sure that the correct environment is sourced:
$ make -f Makefile.gstlal_feature_extractor_offline
Then launch the DAG with:
$ condor_submit_dag feature_extractor_pipe.dag
Configuration options
====================================================================================================
Analysis times:
* START: set the analysis gps start time
* STOP: set the analysis gps stop time
Data ingestion:
* IFO: select the IFO for auxiliary channels to be ingested (H1/L1).
* CHANNEL_LIST: a list of channels for the feature extractor to process. Provided
lists for O1/O2 and H1/L1 lists are in gstlal/gstlal-burst/share/feature_extractor.
* MAX_SERIAL_STREAMS: Maximum # of streams that a single gstlal_feature_extractor job will
process at once. This is determined by sum_i(channel_i * # rates_i). Number of rates for a
given channels is determined by log2(max_rate/min_rate) + 1.
* MAX_PARALLEL_STREAMS: Maximum # of streams that a single job will run in the lifespan of a job.
This is distinct from serial streams since when a job is first launched, it will cache
auxiliary channel frames containing all channels that meet the criterion here, and then process
each channel subset sequentially determined by the serial streams. This is to save on input I/O.
* CONCURRENCY: determines the maximum # of concurrent reads from the same frame file. For most
purposes, it will be set to 1. Use this at your own risk.
Waveform parameters:
* WAVEFORM: type of waveform used to perform matched filtering (sine_gaussian/half_sine_gaussian).
* MISMATCH: maximum mismatch between templates (corresponding to Omicron's mismatch definition).
* QHIGH: maximum value of Q
Data transfer/saving:
* OUTPATH: directory in which to save features.
* SAVE_CADENCE: span of a typical dataset within an hdf5 file.
* PERSIST_CADENCE: span of a typical hdf5 file.
Setting the number of streams (ADVANCED USAGE)
====================================================================================================
NOTE: This won't have to be changed for almost all use cases, and the current configuration has been
optimized to aim for short run times.
Definition: Target number of streams (N_channels x N_rates_per_channel) that each cpu will process.
* if max_serial_streams > max_parallel_streams, all jobs will be parallelized by channel
* if max_parallel_streams > num_channels in channel list, all jobs will be processed serially,
with processing driven by max_serial_streams.
* any other combination will produce a mix of parallelization by channels and processing channels serially per job.
Playing around with combinations of MAX_SERIAL_STREAMS, MAX_PARALLEL_STREAMS, CONCURRENCY, will entirely
determine the structure of the offline DAG. Doing so will also change the memory usage for each job, and so you'll
need to tread lightly. Changing CONCURRENCY in particular may cause I/O locks due to jobs fighting to read from the same
frame file.
####################################################################################################
Running Online Jobs
####################################################################################################
An online DAG is provided in /gstlal-burst/share/feature_extractor/Makefile.gstlal_feature_extractor_online
in order to provide a convenient way to launch online feature extraction jobs as well as auxiliary jobs as
needed (synchronizer/hdf5 file sinks). A condensed list of instructions for use is also provided within the Makefile itself.
There are four separate modes that can be used to launch online jobs:
1. Auxiliary channel ingestion:
a. Reading from framexmit protocol (DATA_SOURCE=framexmit).
This mode is recommended when reading in live data from LHO/LLO.
b. Reading from shared memory (DATA_SOURCE=lvshm).
This mode is recommended for reading in data for O2 replay (e.g. UWM).
2. Data transfer of features:
a. Saving features directly to disk, e.g. no data transfer.
This will save features to disk directly from the feature extractor,
and saves features periodically via hdf5.
b. Transfer of features via Kafka topics.
This requires a Kafka/Zookeeper service to be running (can be existing LDG
or your own). Features get transferred via Kafka from the feature extractor,
parallel instances of the extractor get synchronized, and then sent downstream
where it can be read by other processes (e.g. iDQ). In addition, an streaming
hdf5 file sink is launched where it'll dump features periodically to disk.
Launching DAGs
====================================================================================================
In order to start up online runs, you'll need an installation of gstlal. An installation Makefile that
includes Kafka dependencies are located at: gstlal/gstlal-burst/share/feature_extractor/Makefile.gstlal_idq_icc
To run, making sure that the correct environment is sourced:
$ make -f Makefile.gstlal_feature_extractor_online
Then launch the DAG with:
$ condor_submit_dag feature_extractor_pipe.dag
Configuration options
====================================================================================================
General:
* TAG: sets the name used for logging purposes, Kafka topic naming, etc.
Data ingestion:
* IFO: select the IFO for auxiliary channels to be ingested.
* CHANNEL_LIST: a list of channels for the feature extractor to process. Provided
lists for O1/O2 and H1/L1 lists are in gstlal/gstlal-burst/share/feature_extractor.
* DATA_SOURCE: Protocol for reading in auxiliary channels (framexmit/lvshm).
* MAX_STREAMS: Maximum # of streams that a single gstlal_feature_extractor process will
process. This is determined by sum_i(channel_i * # rates_i). Number of rates for a
given channels is determined by log2(max_rate/min_rate) + 1.
Waveform parameters:
* WAVEFORM: type of waveform used to perform matched filtering (sine_gaussian/half_sine_gaussian).
* MISMATCH: maximum mismatch between templates (corresponding to Omicron's mismatch definition).
* QHIGH: maximum value of Q
Data transfer/saving:
* OUTPATH: directory in which to save features.
* SAVE_FORMAT: determines whether to transfer features downstream or save directly (kafka/hdf5).
* SAVE_CADENCE: span of a typical dataset within an hdf5 file.
* PERSIST_CADENCE: span of a typical hdf5 file.
Kafka options:
* KAFKA_TOPIC: basename of topic for features generated from feature_extractor
* KAFKA_SERVER: Kafka server address where Kafka is hosted. If features are run in same location,
as in condor's local universe, setting localhost:port is fine. Otherwise you'll need to determine
the IP address where your Kafka server is running (using 'ip addr show' or equivalent).
* KAFKA_GROUP: group for which Kafka producers for feature_extractor jobs report to.
Synchronizer/File sink options:
* PROCESSING_CADENCE: cadence at which incoming features are processed, so as to limit polling
of topics repeatedly, etc. Default value of 0.1s is fine.
* REQUEST_TIMEOUT: timeout for waiting for a single poll from a Kafka consumer.
* LATENCY_TIMEOUT: timeout for the feature synchronizer before older features are dropped. This
is to prevent a single feature extractor job from holding up the online pipeline. This will
also depend on the latency induced by the feature extractor, especially when using templates
that have latencies associated with them such as Sine-Gaussians.
####################################################################################################
Tutorials
####################################################################################################
.. toctree::
:maxdepth: 2
running_online_jobs
running_offline_jobs
GstLAL calibration code
=======================
.. toctree::
:maxdepth: 2
bin/bin
python-modules/modules
GstLAL calibration
==========================
.. toctree::
:maxdepth: 2
code
GstLAL inspiral code
====================
.. toctree::
:maxdepth: 2
bin/bin
python-modules/modules
GstLAL inspiral
=======================
.. toctree::
:maxdepth: 2
tutorials/tutorials
code
Tutorials
=========
.. toctree::
:maxdepth: 1
online_analysis
GstLAL ugly code
================
.. toctree::
:maxdepth: 2
bin/bin
python-modules/modules
GstLAL ugly
===================
.. toctree::
:maxdepth: 2
code
GstLAL Code
===========
.. toctree::
:maxdepth: 2
bin/bin
python-modules/modules
GstLAL
==============
.. toctree::
:maxdepth: 2
tutorials/tutorials
code
.. GstLAL documentation master file, created by
sphinx-quickstart on Wed Apr 25 21:10:13 2018.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
The GstLAL project documentation
GstLAL documentation
================================
`GstLAL` provides a suite of GStreamer elements that expose gravitational-wave data analysis tools from the LALSuite library for use in GStreamer signal-processing pipelines.
Examples include an element to add simulated gravitational waves to an h(t) stream, and a source element to provide the contents of .gwf frame files to a GStreamer pipeline.
.. _welcome-contents:
Contents
Overview
-------------------------
The GstLAL software package is used for the following activities:
- ``gstlal`` provides core Gstreamer plugins for signal processing workflows with LIGO data and core python bindings for constructing such workflows.
- ``gstlal-calibration`` provides real-time calibration of LIGO control system data into strain data.
- ``gstlal-inspiral`` provides additional signal processing plugins that are specific for LIGO / Virgo searches for compact binaries as well as a substantial amount of python code for post-processing raw signal processing results into gravitational wave candidate lists. Several publications about the methodology and workflow exist, see :ref:`publications`
- ``gstlal-burst`` provides additional signal processing plugins for use in astrophysical and noise transient burst searches.
- ``gstlal-ugly`` is an incubator project for gradual inclusion in the other packages.
.. toctree::
:maxdepth: 2
:caption: Getting Started
:maxdepth: 2
overview
getting-started
projects
publications
installation
quickstart
tutorials/tutorials
.. _welcome-indices:
.. toctree::
:caption: User Guide
:maxdepth: 2
Indices and tables
-------------------------
cbc_analysis
feature_extraction
fake_data
psd_estimation
publications
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
.. toctree::
:caption: API Reference
:maxdepth: 2
executables
api
Build/Test Results
-------------------------
......@@ -39,3 +53,13 @@ Results pages for the `Offline Tutorial Test <https://git.ligo.org/lscsoft/gstla
* `gstlal_offline_tutorial test dag <gstlal_offline_tutorial/1000000000-1000002048-test_dag-run_1/>`_
* `gstlal_offline_tutorial test dag lite <gstlal_offline_tutorial/1000000000-1000002048-test_dag-run_1_lite/>`_
.. _welcome-indices:
Indices and tables
-------------------------
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
Getting started
Installation
===============
You can get a development copy of the gstlal software suite from git. Doing this at minimum will require a development copy of lalsuite.
......
Overview
========
The GstLAL software package is used for the following activities:
- **GstLAL:** The package `GstLAL <http://software.ligo.org/lscsoft/source/gstlal-1.4.1.tar.gz>`_ provides core Gstreamer plugins for signal processing workflows with LIGO data and core python bindings for constructing such workflows.
- **GstLAL Calibration:** The package `GstLAL Calibration <http://software.ligo.org/lscsoft/source/gstlal-calibration-1.2.4.tar.gz>`_ provides real-time calibration of LIGO control system data into strain data.
- **GstLAL Inspiral:** The package `GstLAL Inspiral <http://software.ligo.org/lscsoft/source/gstlal-inspiral-1.5.1.tar.gz>`_ provides additional signal processing plugins that are specific for LIGO / Virgo searches for compact binaries as well as a substantial amount of python code for post-processing raw signal processing results into gravitational wave candidate lists. Several publications about the methodology and workflow exist, see :ref:`publications`
- **GstLAL Ugly:** The package `GstLAL Inspiral <http://software.ligo.org/lscsoft/source/gstlal-inspiral-1.5.1.tar.gz>`_ is an incubator project for gradual inclusion in the other packages.
Projects
========
.. toctree::
:maxdepth: 1
gstlal/gstlal
gstlal-inspiral/gstlal-inspiral
gstlal-calibration/gstlal-calibration
gstlal-burst/gstlal-burst
gstlal-ugly/gstlal-ugly
.. _psd-generation:
PSD Generation
================
WRITEME
.. _publications:
Publications
============
......
.. _quickstart:
Quickstart
============
WRITEME
......@@ -17,13 +17,13 @@ Consult :any:`gstlal_fake_frames` for more details
The basic steps to generate and validate LIGO colored noise are:
1. use gstlal_fake_frames to make the data (examples in the documenation include this)
2. verify that the PSD is as you would expect with gstlal_reference_psd (examples in the documentation include this)
3. plot the resulting PSD with gstlal_plot_psd (examples in the documentation include this)
1. Use ``gstlal_fake_frames`` to make the data
2. Verify that the PSD is as you would expect with ``gstlal_reference_psd``
3. Plot the resulting PSD with ``gstlal_plot_psd``
An example PSD plot:
.. image:: ../images/H1L1fakedataexamplepsd.png
.. image:: ../gstlal/images/H1L1fakedataexamplepsd.png
:width: 400px
Custom colored noise, i.e. simulate your own detector
......@@ -31,16 +31,11 @@ Custom colored noise, i.e. simulate your own detector
Consult :any:`gstlal_fake_frames` for more details
1. Start by obtaining a reference PSD that you wish to have as the target for
recoloring. If you actually have a text file ASD such as this one: e.g. <a
href=http://www.lsc-group.phys.uwm.edu/cgit/gstlal/plain/gstlal/share/v1_early_asd.txt>here</a>,
then you will need to first use gstlal_psd_xml_from_asd_txt to convert it
(examples in the documentation include this)
1. Next use gstlal_fake_frames to make the data with the desired PSD (examples
in the documentation include this)
1. Repeat the same validation steps as above to obtain, e.g.:
1. Start by obtaining a reference PSD that you wish to have as the target for recoloring. If you actually have a text file ASD such as this one: e.g. `here <https://git.ligo.org/lscsoft/gstlal/raw/master/gstlal/share/v1_early_asd.txt>`_, then you will need to first use ``gstlal_psd_xml_from_asd_txt`` to convert it
2. Next use ``gstlal_fake_frames`` to make the data with the desired PSD
3. Repeat the same validation steps as above to obtain, e.g.:
.. image:: ../images/V1fakedataexamplepsd.png
.. image:: ../gstlal/images/V1fakedataexamplepsd.png
:width: 400px
......@@ -53,25 +48,24 @@ This procedure assumes you are on an LDG cluster which has the data you wish to
recolor. Note that some of the tools required on not gstlal based. Please
consult the documentation for the external tools should you have questions.
1. First obtain segments for the data using ligolw_segment_query
2. Next obtain the frame file cache from ligo_data_find
3. Then create the PSD you wish to recolor to (perhaps using gstlal_psd_xml_from_asd_txt)
4. compute a reference spectrum from the frame data that you wish to recolor using gstlal_reference_psd
5. You might choose to optionally "smooth" the reference spectrum in order to leave lines in the underlying data. You can try using gstlal_psd_polyfit
6. Now with segments, a frame cache, a PSD (possibly smoothed) measured from the frame cache, and a PSD that is the target for the recolored spectrum, you are free to use gstlal_fake_frames according to the documentation.
1. First obtain segments for the data using ``ligolw_segment_query_dqsegdb``
2. Next obtain the frame file cache from ``gw_data_find``
3. Then create the PSD you wish to recolor to (perhaps using ``gstlal_psd_xml_from_asd_txt``)
4. compute a reference spectrum from the frame data that you wish to recolor using ``gstlal_reference_psd``
5. You might choose to optionally "smooth" the reference spectrum in order to leave lines in the underlying data. You can try using ``gstlal_psd_polyfit``
6. Now with segments, a frame cache, a PSD (possibly smoothed) measured from the frame cache, and a PSD that is the target for the recolored spectrum, you are free to use ``gstlal_fake_frames`` according to the documentation.
Recoloring existing data with a HTCondor dag
--------------------------------------------
Some of the steps required to automate the batch processing of recoloring a
large data set has been automated in a script that generates a condor DAG. The
input to the condor dag script has itself been automated in makefiles such as:
<a
href=http://www.lsc-group.phys.uwm.edu/cgit/gstlal/plain/gstlal/share/Makefile.2015recolored>this</a>.
input to the condor dag script has itself been automated in makefiles such as
`this <https://git.ligo.org/lscsoft/gstlal/raw/master/gstlal/share/Makefile.2015recolored>`_.
As an example try this::
$ wget http://www.lsc-group.phys.uwm.edu/cgit/gstlal/plain/gstlal/share/Makefile.2015recolored
$ wget https://git.ligo.org/lscsoft/gstlal/raw/master/gstlal/share/Makefile.2015recolored
$ make -f Makefile.2015recolored
$ condor_submit_dag gstlal_fake_frames_pipe.dag
......@@ -79,10 +73,6 @@ You can monitor the dag progress with::
$ tail -f gstlal_fake_frames_pipe.dag.dagman.out
You should have directories called LIGO and Virgo that contain the recolored frame data. Try changing values in the Makefile to match what you need
TODO
----
1. Add support for making colored noise in the gstlal_fake_frames_pipe
You should have directories called LIGO and Virgo that contain the recolored frame data. Experiment
with changing parameters in the Makefile to generate different PSDs, create frames over different stretches
of data, etc.
This diff is collapsed.
Documentation for starting an online compact binary coalescence analysis
Running an online compact binary coalescence analysis
========================================================================
Prerequisites
......
......@@ -5,3 +5,5 @@ Tutorials
:maxdepth: 1
gstlal_fake_data_overview
online_analysis
offline_analysis
......@@ -23,7 +23,7 @@ outdir = sys.argv[1]
tocf = open(os.path.join(outdir, "bin.rst"), "w")
tocf.write("""bin
===
=====================
.. toctree::
:maxdepth: 1
......@@ -58,10 +58,13 @@ for prog in sorted(os.listdir(indir)):
# write the output of --help
f.write("%s\n%s\n\n" % ("Command line options", "".join(["-"] * len("Command line options"))))
f.write("\n\n.. code-block:: none\n\n")
proc = subprocess.Popen([path_to_prog, "--help"], stdout = subprocess.PIPE)
helpmessage = proc.communicate()[0]
helpmessage = "\n".join([" %s" % l for l in helpmessage.split("\n")])
f.write(helpmessage)
try:
proc = subprocess.Popen([path_to_prog, "--help"], stdout = subprocess.PIPE)
helpmessage = proc.communicate()[0]
helpmessage = "\n".join([" %s" % l for l in helpmessage.split("\n")])
f.write(helpmessage)
except OSError:
pass
# close the file
f.close()
......
......@@ -3,7 +3,7 @@
#
AC_INIT([gstlal-burst],[0.1.0],[gstlal-discuss@ligo.org],[gstlal-burst])
AC_INIT([gstlal-burst],[0.1.1],[gstlal-discuss@ligo.org],[gstlal-burst])
AC_COPYRIGHT([Copyright (C) The authors (see source code for details)])
# a file whose existance can be used to use to check that we are in the
# top-level directory of the source tree
......@@ -97,7 +97,8 @@ case `cat /etc/redhat-release 2> /dev/null` in
;;
esac
LT_INIT
PKG_PROG_PKG_CONFIG()
AC_SUBST([MIN_PKG_CONFIG_VERSION], [0.18.0])
PKG_PROG_PKG_CONFIG("$MIN_PKG_CONFIG_VERSION")
#
......
gstlal-burst (0.1.1) unstable; urgency=low
* Updated gstlal_feature_aggregator, gstlal_feature_monitor to deal with
ligo-scald API change
-- Patrick Godwin <patrick.godwin@ligo.org> Sun, 03 Mar 2019 21:27:15 -0500
gstlal-burst (0.1.0) unstable; urgency=low
* Add feature extraction toolkit
......
......@@ -13,7 +13,7 @@ Build-Depends:
lalmetaio-dev (>= @MIN_LALMETAIO_VERSION@),
python-glue (>= @MIN_GLUE_VERSION@),
fakeroot,
pkg-config,
pkg-config (>= @MIN_PKG_CONFIG_VERSION@),
gstlal-dev (>= @MIN_GSTLAL_VERSION@),
gstlal-ugly-dev (>= @MIN_GSTLAL_UGLY_VERSION@)
......
......@@ -11,7 +11,7 @@ AC_DEFUN([AX_PYTHON_GLUE],[
AC_MSG_ERROR(["cannot determine version"])
])
minversion=$1
AX_COMPARE_VERSION([$LIGO_SEGMENTS_VERSION], [ge], [${minversion:-0}], [
AX_COMPARE_VERSION([$GLUE_VERSION], [ge], [${minversion:-0}], [
AC_MSG_RESULT([$GLUE_VERSION])
], [
AC_MSG_WARN([found $GLUE_VERSION, require at least $1])
......
......@@ -7,39 +7,39 @@ Summary: GSTLAL Burst
License: GPL
Group: LSC Software/Data Analysis
Requires: gstlal-ugly >= @MIN_GSTLAL_UGLY_VERSION@
Requires: gstlal >= @MIN_GSTLAL_VERSION@
Requires: python >= @MIN_PYTHON_VERSION@
Requires: fftw >= 3
Requires: glue >= @MIN_GLUE_VERSION@
Requires: python2-ligo-segments >= @MIN_LIGO_SEGMENTS_VERSION@
Requires: gobject-introspection >= @MIN_GOBJECT_INTROSPECTION_VERSION@
Requires: fftw >= 3
Requires: python-%{gstreamername}
Requires: gstlal >= @MIN_GSTLAL_VERSION@
Requires: gstlal-ugly >= @MIN_GSTLAL_UGLY_VERSION@
Requires: %{gstreamername} >= @MIN_GSTREAMER_VERSION@
Requires: %{gstreamername}-plugins-base >= @MIN_GSTREAMER_VERSION@
Requires: %{gstreamername}-plugins-good >= @MIN_GSTREAMER_VERSION@
Requires: gsl
Requires: h5py
Requires: numpy
Requires: scipy
Requires: lal >= @MIN_LAL_VERSION@
Requires: lalmetaio >= @MIN_LALMETAIO_VERSION@
Requires: lalburst >= @MIN_LALBURST_VERSION@
Requires: numpy
Requires: orc >= @MIN_ORC_VERSION@
Requires: gsl
Requires: python >= @MIN_PYTHON_VERSION@
Requires: python-%{gstreamername}
Requires: python2-ligo-segments >= @MIN_LIGO_SEGMENTS_VERSION@
Requires: scipy
BuildRequires: fftw-devel >= 3
BuildRequires: gobject-introspection-devel >= @MIN_GOBJECT_INTROSPECTION_VERSION@
BuildRequires: graphviz
BuildRequires: gsl-devel
BuildRequires: gstlal-devel >= @MIN_GSTLAL_VERSION@
BuildRequires: python-devel >= @MIN_PYTHON_VERSION@
BuildRequires: fftw-devel >= 3
BuildRequires: %{gstreamername}-devel >= @MIN_GSTREAMER_VERSION@
BuildRequires: %{gstreamername}-plugins-base-devel >= @MIN_GSTREAMER_VERSION@
BuildRequires: lal-devel >= @MIN_LAL_VERSION@
BuildRequires: lal-python >= @MIN_LAL_VERSION@
BuildRequires: lalburst-devel >= @MIN_LALBURST_VERSION@
BuildRequires: lalmetaio-devel >= @MIN_LALMETAIO_VERSION@
BuildRequires: gsl-devel
BuildRequires: graphviz
BuildRequires: orc >= @MIN_ORC_VERSION@
BuildRequires: pkgconfig >= @MIN_PKG_CONFIG_VERSION@
BuildRequires: python-devel >= @MIN_PYTHON_VERSION@
BuildRequires: python2-lal >= @MIN_LAL_VERSION@
Conflicts: gstlal-ugly < 0.6.0
Source: @PACKAGE_NAME@-%{version}.tar.gz
URL: https://www.lsc-group.phys.uwm.edu/daswg/projects/gstlal.html
......
......@@ -6,6 +6,6 @@ includedir=@includedir@
Name: libgstlalburst
Description: LAL/Gstreamer Library (the bursty bits)
Version: @VERSION@
Requires: lal lalmetaio lalburst gstlal
Requires: lal lalmetaio glib-2.0 gsl gstlal
Libs: -L${libdir} -lgstlalburst
Cflags: -I${includedir}
......@@ -96,7 +96,8 @@ case `cat /etc/redhat-release 2> /dev/null` in
;;
esac
LT_INIT
PKG_PROG_PKG_CONFIG()
AC_SUBST([MIN_PKG_CONFIG_VERSION], [0.18.0])
PKG_PROG_PKG_CONFIG("$MIN_PKG_CONFIG_VERSION")
#
......@@ -239,7 +240,7 @@ AX_PYTHON_LIGO_SEGMENTS([$MIN_LIGO_SEGMENTS_VERSION])
AC_SUBST([MIN_GSTLAL_VERSION], [1.0.0])
AC_SUBST([MIN_GSTLALUGLY_VERSION], [1.0.0])
AC_SUBST([MIN_GSTLALUGLY_VERSION], [1.5.6])
AC_SUBST([MIN_GSTPLUGINSMATH_VERSION], [1.0.1])
PKG_CHECK_MODULES([GSTLAL], [gstlal >= ${MIN_GSTLAL_VERSION}])
AC_SUBST([GSTLAL_VERSION], [`$PKG_CONFIG --modversion gstlal`])
......
......@@ -12,7 +12,7 @@ Build-Depends:
lalmetaio-dev (>= @MIN_LALMETAIO_VERSION@),
libgstreamer1.0-dev (>= @MIN_GSTREAMER_VERSION@),
libgstreamer-plugins-base1.0-dev (>= @MIN_GSTREAMER_VERSION@),
pkg-config,
pkg-config (>= @MIN_PKG_CONFIG_VERSION@),
python-all-dev (>= @MIN_PYTHON_VERSION@),
python-numpy
......@@ -22,7 +22,6 @@ Depends: ${shlibs:Depends}, ${misc:Depends}, ${python:Depends},
gir1.2-glib-2.0,
gir1.2-gstreamer-1.0 (>= @MIN_GSTREAMER_VERSION@),
gir1.2-gst-plugins-base-1.0 (>= @MIN_GSTREAMER_VERSION@),
gst-plugins-math (>= 1.0.1),
gstlal (>= @MIN_GSTLAL_VERSION@),
gstlal-ugly (>= @MIN_GSTLALUGLY_VERSION@),
gstreamer1.0-plugins-base (>= @MIN_GSTREAMER_VERSION@),
......
......@@ -11,7 +11,7 @@ AC_DEFUN([AX_PYTHON_GLUE],[
AC_MSG_ERROR(["cannot determine version"])
])
minversion=$1
AX_COMPARE_VERSION([$LIGO_SEGMENTS_VERSION], [ge], [${minversion:-0}], [
AX_COMPARE_VERSION([$GLUE_VERSION], [ge], [${minversion:-0}], [
AC_MSG_RESULT([$GLUE_VERSION])
], [
AC_MSG_WARN([found $GLUE_VERSION, require at least $1])
......
......@@ -19,8 +19,8 @@ Requires: numpy
Requires: scipy
Requires: lal >= @MIN_LAL_VERSION@
Requires: lalmetaio >= @MIN_LALMETAIO_VERSION@
Requires: gst-plugins-math >= @MIN_GSTPLUGINSMATH_VERSION@
BuildRequires: pkgconfig >= @MIN_PKG_CONFIG_VERSION@
BuildRequires: gstlal-devel >= @MIN_GSTLAL_VERSION@
BuildRequires: python-devel >= @MIN_PYTHON_VERSION@
BuildRequires: fftw-devel >= 3
......
......@@ -9,6 +9,7 @@ dist_bin_SCRIPTS = \
gstlal_inspiral_calc_likelihood \
gstlal_inspiral_calc_rank_pdfs \
gstlal_inspiral_coinc_extractor \
gstlal_inspiral_compress_ranking_stat \
gstlal_inspiral_add_dt_dphi_snr_ratio_pdfs \
gstlal_inspiral_create_dt_dphi_snr_ratio_pdfs \
gstlal_inspiral_create_dt_dphi_snr_ratio_pdfs_dag \
......@@ -25,6 +26,7 @@ dist_bin_SCRIPTS = \
gstlal_inspiral_lvalert_sim_equiv \
gstlal_inspiral_lvalert_sngls_plotter \
gstlal_inspiral_lvalert_snrtimeseries_plotter \
gstlal_inspiral_lvalert_uberplotter \
gstlal_inspiral_make_snr_pdf \
gstlal_inspiral_marginalize_likelihood \
gstlal_inspiral_marginalize_likelihoods_online \
......@@ -47,6 +49,7 @@ dist_bin_SCRIPTS = \
gstlal_ll_inspiral_calculate_range \
gstlal_ll_inspiral_daily_page \
gstlal_ll_inspiral_daily_page_online \
gstlal_ll_inspiral_event_uploader \
gstlal_ll_inspiral_get_urls \
gstlal_ll_inspiral_gracedb_min_instruments \
gstlal_ll_inspiral_gracedb_threshold \
......
......@@ -88,7 +88,7 @@ def group_templates(templates, n, overlap = 0):
def parse_command_line():
parser = OptionParser()
parser.add_option("--output-path", metavar = "path", default = ".", help = "Set the path to the directory where output files will be written. Default is \".\".")
parser.add_option("--output-full-bank-file", metavar = "path", default = "gstlal_bank.xml.gz", help = "Set the path to output the bank. Default is gstlal_bank.xml.gz")
parser.add_option("--output-full-bank-file", metavar = "path", help = "Set the path to output the bank.")
parser.add_option("--output-cache", metavar = "file", help = "Set the file name for the output cache.")
parser.add_option("--n", metavar = "count", type = "int", help = "Set the number of templates per output file (required). It will be rounded to make all sub banks approximately the same size.")
parser.add_option("--overlap", default = 0, metavar = "count", type = "int", help = "overlap the templates in each file by this amount, must be even")
......@@ -146,7 +146,8 @@ sngl_inspiral_table = lsctables.SnglInspiralTable.get_table(xmldoc)
# FIXME
#process = ligolw_process.register_to_xmldoc(xmldoc, program = "gstlal_bank_splitter", paramdict = options.__dict__, comment = "Assign template IDs")
ligolw_utils.write_filename(xmldoc, options.output_full_bank_file, gz = options.output_full_bank_file.endswith('gz'), verbose = options.verbose)
if options.output_full_bank_file is not None:
ligolw_utils.write_filename(xmldoc, options.output_full_bank_file, gz = options.output_full_bank_file.endswith('gz'), verbose = options.verbose)
# Bin by Chi
sngl_inspiral_table.sort(key = lambda row: spawaveform.computechi(row.mass1, row.mass2, row.spin1z, row.spin2z))
......
......@@ -253,6 +253,8 @@ def parse_command_line():
parser.add_option_group(group)
group = OptionGroup(parser, "Ranking Statistic Options", "Adjust ranking statistic behaviour")
group.add_option("--cap-singles", action = "store_true", help = "Cap singles to 1 / livetime if computing FAR. No effect otherwise")
group.add_option("--FAR-trialsfactor", metavar = "trials", type = "float", default = 1.0, help = "Add trials factor to FAR before uploading to gracedb")
group.add_option("--chisq-type", metavar = "type", default = "autochisq", help = "Choose the type of chisq computation to perform. Must be one of (autochisq|timeslicechisq). The default is autochisq.")
group.add_option("--coincidence-threshold", metavar = "seconds", type = "float", default = 0.005, help = "Set the coincidence window in seconds (default = 0.005 s). The light-travel time between instruments will be added automatically in the coincidence test.")
group.add_option("--min-instruments", metavar = "count", type = "int", default = 2, help = "Set the minimum number of instruments that must contribute triggers to form a candidate (default = 2).")
......@@ -271,6 +273,7 @@ def parse_command_line():
group.add_option("--gracedb-pipeline", metavar = "name", default = "gstlal", help = "Name of pipeline to provide in GracedB uploads (default is gstlal).")
group.add_option("--gracedb-search", metavar = "name", default = "LowMass", help = "Name of search to provide in GracedB uploads (default is LowMass).")
group.add_option("--gracedb-service-url", metavar = "url", default = gracedb_default_service_url, help = "Override default GracedB service url (optional, default is %s)." % gracedb_default_service_url)
group.add_option("--delay-uploads", action = "store_true", help = "Choose whether to delay uploads to allow aggregation of events downstream from many gstlal_inspiral jobs (default is False).")
parser.add_option_group(group)
group = OptionGroup(parser, "Program Behaviour")
......@@ -281,6 +284,7 @@ def parse_command_line():
group.add_option("--job-tag", metavar = "tag", help = "Set the string to identify this job and register the resources it provides on a node. Should be 4 digits of the form 0001, 0002, etc.; may not contain \".\" nor \"-\".")
group.add_option("--local-frame-caching", action = "store_true", help = "Pre-reads frame data, performs downsampling, and stores to local filespace. ")
group.add_option("--nxydump-segment", metavar = "start:stop", default = ":", help = "Set the time interval to dump from nxydump elments (optional). The default is \":\", i.e. dump all time.")
group.add_option("--reconstruction-segment", metavar = "start:stop", action = "append", help = "Only reconstruct the SNRs for this time interval (optional). Can be provided multiple times.")
group.add_option("-t", "--tmp-space", metavar = "path", help = "Path to a directory suitable for use as a work area while manipulating the database file. The database file will be worked on in this directory, and then moved to the final location when complete. This option is intended to improve performance when running in a networked environment, where there might be a local disk with higher bandwidth than is available to the filesystem on which the final output will reside.")
group.add_option("-v", "--verbose", action = "store_true", help = "Be verbose (optional).")
group.add_option("--write-pipeline", metavar = "filename", help = "Write a DOT graph description of the as-built pipeline to this file (optional). The environment variable GST_DEBUG_DUMP_DOT_DIR must be set for this option to work.")
......@@ -717,6 +721,11 @@ for output_file_number, (svd_bank_url_dict, output_url, ranking_stat_output_url,
if options.injections:
offset_padding = max(math.ceil(abs(row.end))+1 for bank in banks.items()[0][-1] for row in bank.sngl_inspiral_table)
reconstruction_segment_list = simulation.sim_inspiral_to_segment_list(options.injections, pad = offset_padding)
elif options.reconstruction_segment:
reconstruction_segment_list = segments.segmentlist()
for reconstruction_segment in options.reconstruction_segment:
t0, t1 = reconstruction_segment.split(":")
reconstruction_segment_list.append(segments.segment(LIGOTimeGPS(int(t0)), LIGOTimeGPS(int(t1))))
else:
reconstruction_segment_list = None
......@@ -823,6 +832,8 @@ for output_file_number, (svd_bank_url_dict, output_url, ranking_stat_output_url,
search = options.gracedb_search,
pipeline = options.gracedb_pipeline,
service_url = options.gracedb_service_url,
delay_uploads = options.delay_uploads,
kafka_server = options.output_kafka_server,
upload_auxiliary_data = True,
verbose = options.verbose
),
......@@ -835,6 +846,8 @@ for output_file_number, (svd_bank_url_dict, output_url, ranking_stat_output_url,
tag = options.job_tag,
kafka_server = options.output_kafka_server,
cluster = True,#options.data_source in ("lvshm", "framexmit"),# If uncommented, we only cluster when running online
cap_singles = options.cap_singles,
FAR_trialsfactor = options.FAR_trialsfactor,
verbose = options.verbose
)
if options.verbose:
......
......@@ -48,8 +48,8 @@ N = sum(len(lsctables.SnglInspiralTable.get_table(xmldoc)) for fname, xmldoc in
# N get too close to the number of available IDs to keep the probability of
# two ID sequences being the same small.
assert N < 50000000, "too many templates: increase size of draw space"
ids = sorted(random.sample(xrange(99999999), N))
assert N < 5000000, "too many templates: increase size of draw space"
ids = sorted(random.sample(xrange(9999999), N))
# assign the IDs and write back to disk
......
#!/usr/bin/env python
#
# Copyright (C) 2019 Kipp Cannon
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
#
# =============================================================================
#
# Preamble
#
# =============================================================================
#
import math
from optparse import OptionParser
import sys
import numpy
from ligo.lw import utils as ligolw_utils
from gstlal import far
__author__ = "Kipp Cannon <kipp.cannon@ligo.org>"
__version__ = "git id %s" % "" # FIXME
__date__ = "" # FIXME
#
# =============================================================================
#
# Command Line
#
# =============================================================================
#
def parse_command_line():
parser = OptionParser(
version = "Name: %%prog\n%s" % "" # FIXME
)
parser.add_option("-t", "--threshold", type = "float", default = 0.03, help = "Only keep horizon distance values that differ by this much, fractionally, from their neighbours (default = 0.03).")
parser.add_option("--remove-horizon-deviations", action = "store_true", help = "Remove horizon entries that display an uncharacteristic deviation in sensitivity from the non-zero mean.")
parser.add_option("--deviation-percent", type = "float", default = 0.50, help = "Remove horizon entries that deviate by this fraction from the non-zero mean.")
parser.add_option("-v", "--verbose", action = "store_true", help = "Be verbose.")
options, filenames = parser.parse_args()
process_params = dict(options.__dict__)
return options, process_params, filenames
#
# =============================================================================
#
# Main
#
# =============================================================================
#
#
# command line
#
options, process_params, filenames = parse_command_line()
#
# loop over ranking statistic files
#
for filename in filenames:
#
# load file
#
xmldoc = ligolw_utils.load_filename(filename, verbose = options.verbose, contenthandler = far.RankingStat.LIGOLWContentHandler)
#
# extract ranking statistic object, and erase from XML tree
#
rankingstat, rankingstatpdf = far.parse_likelihood_control_doc(xmldoc)
if rankingstatpdf is not None and options.verbose:
print >>sys.stderr, "WARNING: \"%s\" contains a RankingStatPDF object, it is not a pure ranking statistic file, you might be using this program on the wrong files." % filename
# FIXME: don't hard-code object name
name = u"gstlal_inspiral_likelihood"
elem = rankingstat.get_xml_root(xmldoc, name)
elem.parentNode.removeChild(elem)
elem.unlink()
elem = rankingstatpdf.get_xml_root(xmldoc, name)
elem.parentNode.removeChild(elem)
elem.unlink()
#
# compress horizon distance history. the outer loop makes a list
# to ensure no problems modifying the object being iterated over
#
abs_ln_thresh = math.log1p(options.threshold)
for instrument, horizon_history in list(rankingstat.numerator.horizon_history.items()):
# GPS time / distance pairs
items = horizon_history.items()
if options.remove_horizon_deviations:
values = numpy.array(items)[:,1]
mean_horizon = values[values!=0].mean()
items = [item for item in items if item[1] < (mean_horizon * (1. + options.deviation_percent))]
# compress array
j = 1
for i in range(1, len(items) - 1):
values = items[j - 1][1], items[i][1], items[i + 1][1]
# remove distances that are non-zero and differ
# fractionally from both neighbours by less than
# the selected threshold. always keep the first
# and last values
if values