Commit d6325c07 authored by Adam Mercer's avatar Adam Mercer

Revert "Adding iDQ scripts to laldetchar. This commit includes addition of new...

Revert "Adding iDQ scripts to laldetchar. This commit includes addition of new files in laldetchar/python and a new directory laldetchar/pythonlaldetchar/idq. Makefiles have been modified to ensure that the new codes are configured and installed correctly."
This reverts commit 3e6ef0e4b71b99a13b7ae00330de31e3876a1d1e.
Original: 7c668494a00a8e5fe817e9da5b1415a7fed1f31c
parent 79de15b2
......@@ -25,18 +25,6 @@ libtool
octave/
python/laldetchar-hveto
python/laldetchar-hveto-summary-page
python/laldetchar-idq-add-excluded-vars
python/laldetchar-idq-evaluate-times
python/laldetchar-idq-fap-calibration
python/laldetchar-idq-gdb-glitch-tables
python/laldetchar-idq-gdb-timeseries
python/laldetchar-idq-prepare-training-auxmvc-samples
python/laldetchar-idq-realtime
python/laldetchar-idq-select-channels-from-kw-config
python/laldetchar-idq-summary
python/laldetchar-idq-svm-evaluate
python/laldetchar-idq-svm-train
python/laldetchar-idq-train
python/laldetchar-segments-from-data
python/laldetchar/__init__.py
python/laldetchar/_laldetchar*
......
......@@ -20,7 +20,6 @@ AC_CONFIG_FILES([ \
python/laldetchar/__init__.py \
python/laldetchar/triggers/Makefile \
python/laldetchar/hveto/Makefile \
python/laldetchar/idq/Makefile \
swig/Makefile \
test/Makefile
])
......
......@@ -5,27 +5,11 @@ include $(top_srcdir)/gnuscripts/lalsuite_python.am
SUBDIRS = laldetchar
dist_bin_SCRIPTS = \
laldetchar-idq-append-file-to-cache \
laldetchar-idq-post-svm-train
if HAVE_PYTHON
pybin_scripts = \
laldetchar-hveto \
laldetchar-hveto-summary-page \
laldetchar-idq-add-excluded-vars \
laldetchar-idq-evaluate-times \
laldetchar-idq-fap-calibration \
laldetchar-idq-gdb-glitch-tables \
laldetchar-idq-gdb-timeseries \
laldetchar-idq-prepare-training-auxmvc-samples \
laldetchar-idq-realtime \
laldetchar-idq-select-channels-from-kw-config \
laldetchar-idq-summary \
laldetchar-idq-svm-evaluate \
laldetchar-idq-svm-train \
laldetchar-idq-train \
laldetchar-segments-from-data \
$(END_OF_LIST)
......
# Copyright (C) 2013 Ruslan Vaulin
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# =============================================================================
#
# Preamble
#
# =============================================================================
#
from optparse import *
from laldetchar.idq import auxmvc_utils
import numpy
from laldetchar import git_version
__author__ = 'Ruslan Vaulin <ruslan.vaulin@ligo.org>'
__version__ = git_version.id
__date__ = git_version.date
######################################################################################
#
# MAIN
#
######################################################################################
description = \
"""
This code takes *.pat and *.dat files (input and output of the MLA classifier evaluation) and combines them into a single output file adding the variables that were excluded during classification. By default it overwrites the original *.dat file.
"""
parser = OptionParser(version='Name: %%prog\n%s'
% git_version.verbose_msg, usage='%prog [options]'
, description=description)
parser.add_option('', '--pat-file', type='string',
help='*.pat file, containing unclassified samples'
)
parser.add_option('', '--dat-file', type='string',
help='*.dat file, containing classified samples'
)
parser.add_option('', '--excluded-variables', type='string',
help='comma separated list of variables that were excluded during classification'
)
parser.add_option('', '--output-file', type='string', default=None,
help='output file')
(opts, args) = parser.parse_args()
# load samples from pat file
auxmvc_pat_samples = auxmvc_utils.ReadMVSCTriggers([opts.pat_file],
Classified=False)
# load samples from dat file
auxmvc_dat_samples = auxmvc_utils.ReadMVSCTriggers([opts.dat_file],
Classified=True)
excluded_vars = opts.excluded_variables.split(',')
vars = []
if 'GPS_s' and 'GPS_ms' in excluded_vars:
vars.append('GPS')
excluded_vars.remove('GPS_s')
excluded_vars.remove('GPS_ms')
vars.extend(['i', 'w'])
if 'unclean' in excluded_vars:
vars.append('unclean')
excluded_vars.remove('unclean')
for var in excluded_vars:
vars.append(var)
excluded_vars.append('unclean')
vars.append('rank')
for var in auxmvc_dat_samples.dtype.names:
if not var in ['index', 'i', 'w', 'Bagger']:
vars.append(var)
datformats = []
for var in vars:
if var in ['i', 'unclean']:
datformats.append('i')
else:
datformats.append('g8')
dat_samples = numpy.empty(auxmvc_dat_samples.shape,
dtype={'names': vars, 'formats': datformats})
dat_samples['GPS'] = auxmvc_pat_samples['GPS_s'] \
+ auxmvc_pat_samples['GPS_ms'] * 10 ** -3
vars.remove('GPS')
dat_samples['rank'] = auxmvc_dat_samples['Bagger']
vars.remove('rank')
for var in vars:
if var in excluded_vars:
dat_samples[var] = auxmvc_pat_samples[var]
else:
dat_samples[var] = auxmvc_dat_samples[var]
if not opts.output_file:
opts.output_file = opts.dat_file
auxmvc_utils.WriteMVSCTriggers(dat_samples,
output_filename=opts.output_file,
Classified=True)
#!/bin/bash
# usage: appends a file names to a cache file.
# example: idq_append_file_to_cache.sh cachefile file1 file2 ...
# file1 is the full path to the file1 to be appended to cache file
# cachefile is the full path to the cache file
for i in ${@:2}
do
echo $i >> $1
done
This diff is collapsed.
## Copyright (C) 2014 Reed Essick
##
## This program is free software; you can redistribute it and/or modify it
## under the terms of the GNU General Public License as published by the
## Free Software Foundation; either version 2 of the License, or (at your
## option) any later version.
##
## This program is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
## Public License for more details.
##
## You should have received a copy of the GNU General Public License along
## with this program; if not, write to the Free Software Foundation, Inc.,
## 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
#
description = \
"""
This code checks the calibration of the iDQ pipeline using the FAP time-series data.
We expect that segments generated by thresholding on the FAP time-series at FAP=0.XYZ should correspond to an amount of time DURATION=0.XYZ*LIVETIME
"""
# =================================================
import numpy as np
from laldetchar.idq import event
from laldetchar.idq import idq
from laldetchar.idq import idq_gdb_utils
from laldetchar import git_version
__author__ = 'Reed Essick <reed.essick@ligo.org>'
__version__ = git_version.id
__date__ = git_version.date
# =================================================
def timeseries_to_segments(t, ts, thr):
"""
....computes segements from t = time stamps
.... ts = time series (values)
.... thr=threshold on time series
....so that t \in segments iff ts(t) >= thr
....pad is added to the end of the time-series points when generating segments
...."""
segs = []
in_seg = False
min_TS = np.infty
for (T, TS) in zip(t, ts):
if TS < thr:
if in_seg:
segs.append([start, T])
in_seg = False
else:
pass
else:
# TS >= thr
if min_TS > TS:
min_TS = TS
if not in_seg:
start = T
in_seg = True
if in_seg:
dt = t[1] - t[0]
segs.append([start, T])
if len(segs):
return (segs, min_TS)
else:
return ([], None)
# ========================
def check_calibartion(
realtimedir,
start,
end,
classifier,
FAPthrs,
verbose=False,
):
"""
....checks the pipeline's calibration at each "FAPthr in FAPThrs"
"""
# =================================================
# grab idq_segments so we compute meaningful livetimes
# =================================================
if verbose:
print 'getting idq_segements'
idq_segs = idq.get_idq_segments(realtimedir, start, end,
suffix='.npy.gz')
idq_livetime = event.livetime(idq_segs)
# =================================================
# grab relevant data
# =================================================
if verbose:
print 'looking for *_fap_*.npy.gz files in', opts.realtimedir
fapfilenames = [filename for filename in
idq.get_all_files_in_range(realtimedir, start, end,
pad=0, suffix='.npy.gz') if '_fap_' in filename
and classifier in filename]
if opts.verbose:
print 'discovered %d files' % len(fapfilenames)
print 'building time-series'
(times, timeseries) = idq_gdb_utils.combine_ts(fapfilenames)
# =================================================
# check calibration
# =================================================
segments = []
deadtimes = []
statedFAPs = []
for FAPthr in FAPthrs:
if verbose:
print 'computing segements for FAPthr =', FAPthr
segs = []
max_statedFAP = None
for (t, ts) in zip(times, timeseries):
(_segs, _min_ts) = timeseries_to_segments(t, -ts, -FAPthr) # we want FAP <= FAPthr <--> -FAP >= FAPthr
segs += _segs
if _min_ts != None:
statedFAP = -_min_ts
if max_statedFAP < statedFAP:
max_statedFAP = statedFAP
segs = event.andsegments([segs, idq_segs])
segments.append(segs)
deadtimes.append(1.0 * event.livetime(segs) / idq_livetime)
statedFAPs.append(statedFAP)
return (idq_segs, segments, deadtimes, statedFAPs)
# =================================================
#
# MAIN
#
# =================================================
if __name__ == '__main__':
from optparse import OptionParser
parser = OptionParser(version='Name: %%prog\n%s'
% git_version.verbose_msg,
usage='%prog [options]',
description=description)
parser.add_option('-v', '--verbose', default=False,
action='store_true')
parser.add_option('-r', '--realtimedir', default='./', type='string'
,
help='the iDQ realtimedir in which FAP time-series files are stored'
)
parser.add_option('-s', '--start', default=-np.infty, type='float'
,
help='gps start time'
)
parser.add_option('-e', '--end', default=np.infty, type='float'
,
help='gps end time'
)
parser.add_option('-c', '--classifier', default='ovl', type='string'
,
help='classifier for which we test the calibration'
)
parser.add_option('', '--FAPthrs', default='0.01', type='string',
help='check calibration at these FAP values (must be supplied as a string. eg: "0.001 0.005 0.01"). DEFAULT="0.01"'
)
(opts, args) = parser.parse_args()
FAPthrs = [float(l) for l in opts.FAPthrs.split()]
(idq_segs, segments, deadtimes, statedFAPs) = check_calibartion(
opts.realtimedir,
opts.start,
opts.end,
opts.classifier,
FAPthrs,
verbose=opts.verbose,
)
# ## Report!
print 'idq_livetime = %.3f' % event.livetime(idq_segs)
report_str = \
"""
FAPthr = %.5f
stated FAP = %.5f
deadtime = %.5f
%s difference = %.3f%s"""
for (FAPthr, deadtime, statedFAP) in zip(FAPthrs, deadtimes,
statedFAPs):
# ## Report!
print report_str % (
FAPthr,
statedFAP,
deadtime,
'%',
100 * (deadtime / FAPthr - 1),
'%',
)
# Copyright (C) 2014 Reed Essick
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
description = \
"""The program generates a summary of iDQ output during a short time period. Its output is written into a new xml file(s)"""
from laldetchar import git_version
__author__ = 'Reed Essick <reed.essick@ligo.org>'
__version__ = git_version.id
__date__ = git_version.date
##################################################
# main
##################################################
if __name__ == '__main__':
from laldetchar.idq import idq
from glue.ligolw import ligolw
from glue.ligolw.utils import ligolw_add
from optparse import OptionParser
parser = OptionParser(version='Name: %%prog\n%s'
% git_version.verbose_msg,
usage='%prog [options]',
description=description)
parser.add_option('-v', '--verbose', default=False,
action='store_true')
parser.add_option(
'-s',
'--gps-start',
dest='start',
default=0,
type='int',
help='the gps start time of the time range of interest',
)
parser.add_option(
'-e',
'--gps-end',
dest='end',
default=0,
type='int',
help='the gps end time of the time range of interest',
)
parser.add_option('', '--ifo', type='string',
help='the ifo for which predictions were made')
parser.add_option('-c', '--classifier', default='ovl', type='string'
,
help='the classifier that was used to generate the data. Default="ovl"'
)
parser.add_option('-i', '--input-dir', default='./', type='string',
help='the directory through which is searched for relevant *glitch*.xml files. Assumes directory structure generated by laldetchar-idq-realtime.py'
)
parser.add_option('-o', '--output-dir', default='./', type='string'
,
help='the output directory'
)
parser.add_option('-t', '--usertag', dest='tag', default='', type='string'
,
help='user tag'
)
(opts, args) = parser.parse_args()
if not opts.ifo:
opts.ifo = raw_input('ifo = ')
if opts.tag != '':
opts.tag = opts.tag + '-'
# #########################################
# ## Find relevant files
# #########################################
if opts.verbose:
print 'Finding relevant *glitch*.xml files'
gchxml_filenames = sorted([filename for filename in
idq.get_all_files_in_range(opts.input_dir,
opts.start, opts.end, pad=0, suffix='.xml'
) if opts.classifier in filename.split('/'
)[-1] and 'glitch' in filename
and opts.ifo in filename])
if opts.verbose:
print 'Found:'
for filename in gchxml_filenames:
print '\t' + filename
# #########################################
# ## Merge xml_docs and write merged file
# #########################################
# ## merge filenames into a single xmldoc
if opts.verbose:
print 'merging files'
merged_xmldoc = ligolw_add.ligolw_add(ligolw.Document(),
gchxml_filenames)
# ....merged_xmldoc = ligolw_add.ligolw_add( ligolw.Document(), gchxml_filenames, non_lsc_tables_ok=False, verbose=False, contenthandler=DefaultContentHandler)
# ## remove any rwo outside of [opts.start, opts.end]
print 'WRITE step that removes glitches outside [opts.start, opts.end]'
# ## save filename
merged_xmldoc_filename = '%s/%s_idq_%s-glitch-%s%d-%d.xml.gz' % (
opts.output_dir,
opts.ifo,
opts.classifier,
opts.usertag,
opts.start,
opts.end - opts.start,
)
if opts.verbose:
print 'saving ' + merged_xmldoc_filename
ligolw_utils.write_filename(merged_xmldoc, merged_xmldoc_filename,
gz=True) # write file
This diff is collapsed.
#!/bin/csh -f
/bin/rm $argv[1] $argv[2]
/bin/echo $argv[3] $argv[4] >> $argv[5]
# Copyright (C) 2013 Ruslan Vaulin
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# =============================================================================
#
# Preamble
#
# =============================================================================
#
import os
import sys
from optparse import *
from laldetchar.idq import auxmvc_utils
import numpy
from laldetchar.idq import event
from laldetchar.idq import idq
from laldetchar import git_version
__author__ = 'Ruslan Vaulin <ruslan.vaulin@ligo.org>'
__version__ = git_version.id
__date__ = git_version.date
######################################################################################
#
# MAIN
#
######################################################################################
description = \
"""This program finds *.pat files that were generated during the low-latency evaluation step and that fall within the specified time segment. It merges them into a single file used for training of classifiers."""
parser = OptionParser(version='Name: %%prog\n%s'
% git_version.verbose_msg, usage='%prog [options]'
, description=description)
parser.add_option('', '--source-directory',
help='path to where *.pat files of low-latency evaluation step are located'
)
parser.add_option('', '--dq-segments', type='string', default='',
help='xml file with dq segments')
parser.add_option('', '--dq-segments-name', type='string',
help='name of dq segments, --include-segments option given to ligolw_segment_query during generation of the xml dq segment file.'
)
parser.add_option('', '--gps-start-time', default='', type='string',
help='gps start time of the period to be processed')
parser.add_option('', '--gps-end-time', default='', type='string',
help='gps end time of the period to be processed')
parser.add_option('', '--max-clean-samples', default=None, type='int',
help='upper limit on number of clean samples in the training set'
)
parser.add_option('', '--max-glitch-samples', default=None, type='int',
help='upper limit on number of glitch samples in the training set.'
)
parser.add_option('', '--output-file', type='string',
default='training_samples.pat',
help='full path and name of the output file into which the training samples will be saved'
)
parser.add_option('', '--verbose', action='store_true', default=False,
help='run in verbose mode')
(opts, args) = parser.parse_args()
gps_start_time = int(opts.gps_start_time)
gps_end_time = int(opts.gps_end_time)
# get all *.pat files in the specififed range
patfiles = idq.get_all_files_in_range(opts.source_directory,
gps_start_time, gps_end_time, suffix='.pat')
if len(patfiles) == 0:
print 'No *.pat files found in the gps range ' \
+ str(gps_start_time) + ' - ' + str(gps_end_time)
print 'Exiting with status 2.'
sys.exit(2)
# load auxmvc vector samples
auxmvc_samples = auxmvc_utils.ReadMVSCTriggers(patfiles,
Classified=False)
if opts.dq_segments:
# load dq segments
(dq_segments, covered_segments) = \
idq.extract_dq_segments(open(opts.dq_segments, 'r'),
opts.dq_segments_name)
# sort and merge segments
dq_segments = event.fixsegments(dq_segments)
# keep only samples that fall in the dq_segments
auxmvc_samples = \
auxmvc_utils.get_samples_in_segments(auxmvc_samples,
dq_segments)
# get clean and glitch samples
random_samples = auxmvc_samples[numpy.nonzero(auxmvc_samples['i']
== 0)[0], :]
clean_samples = auxmvc_utils.get_clean_samples(random_samples)
# apply upper limit on the number of clean samples if given
if opts.max_clean_samples:
if len(clean_samples) > opts.max_clean_samples:
clean_samples = clean_samples[-opts.max_clean_samples:]