Commit 4678544d authored by Kipp Cannon's avatar Kipp Cannon

gstlal-inspiral: partially restore the online variant

parent 198f25c2
......@@ -11,7 +11,6 @@ dist_bin_SCRIPTS = \
gstlal_inspiral_create_p_of_ifos_given_horizon \
gstlal_inspiral_create_prior_diststats \
gstlal_inspiral_dlrs_diag \
gstlal_inspiral_fake_zerolag_counts \
gstlal_inspiral_iir_bank_pipe \
gstlal_inspiral_injection_snr \
gstlal_inspiral_lvalert_background_plotter \
......
This diff is collapsed.
#!/usr/bin/env python
#
# Copyright (C) 2017 Kipp Cannon, Chad Hanna
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# =============================================================================
#
# Preamble
#
# =============================================================================
#
from optparse import OptionParser
import sys
from glue.ligolw import ligolw
from glue.ligolw import utils as ligolw_utils
from glue.ligolw.utils import process as ligolw_process
from gstlal import far
process_name = u"gstlal_inspiral_fake_zerolag_counts"
__author__ = "Kipp Cannon <kipp.cannon@ligo.org>"
__version__ = "git id %s" % "" # FIXME
__date__ = "" # FIXME
#
# =============================================================================
#
# Command Line
#
# =============================================================================
#
def parse_command_line():
parser = OptionParser(
version = "Name: %%prog\n%s" % "" # FIXME
)
parser.add_option("-i", "--input", metavar = "filename", help = "Set name of input file (default = stdin).")
parser.add_option("-o", "--output", metavar = "filename", help = "Set name of output file (default = stdout). May be same as input file (input will be descructively overwritten).")
parser.add_option("-v", "--verbose", action = "store_true", help = "Be verbose.")
options, urls = parser.parse_args()
paramdict = options.__dict__.copy()
required_options = []
missing_options = [opt for opt in required_options if getattr(options, opt) is None]
if missing_options:
raise ValueError("missing required option(s) %s" % ", ".join("--%s" % opt for opt in missing_options))
if urls:
raise ValueError("unrecognized options \"%s\"" % " ".join(urls))
return options, paramdict, urls
#
# =============================================================================
#
# Main
#
# =============================================================================
#
#
# command line
#
options, paramdict, urls = parse_command_line()
#
# load parameter distribution data
#
_, rankingstatpdf = far.parse_likelihood_control_doc(ligolw_utils.load_filename(options.input, contenthandler = far.RankingStat.LIGOLWContentHandler, verbose = options.verbose))
if rankingstatpdf is None:
raise ValueError("'%s' does not contain a RankingStatPDF object" % options.input)
if options.verbose:
print >>sys.stderr, "total livetime: %s s" % str(abs(rankingstatpdf.segments))
#
# copy background histograms to zero-lag histograms, and zero the
# background histograms
#
# FIXME 100.0 is a rough coincidence fraction loss, can probably do better.
rankingstatpdf.zero_lag_lr_lnpdf.array[:] = rankingstatpdf.noise_lr_lnpdf.array / 100.0
rankingstatpdf.zero_lag_lr_lnpdf.normalize()
rankingstatpdf.noise_lr_lnpdf.array[:] = 0.
rankingstatpdf.noise_lr_lnpdf.normalize()
#
# write result
#
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
process = ligolw_process.register_to_xmldoc(xmldoc, process_name, paramdict)
far.gen_likelihood_control_doc(xmldoc, None, rankingstatpdf)
ligolw_utils.write_filename(xmldoc, options.output, gz = (options.output or "stdout").endswith(".gz"), verbose = options.verbose)
......@@ -63,8 +63,8 @@ shift
# paths to data objects on each job's web management interface
#
LIKELIHOOD_PATH="likelihood.xml"
ZEROLAG_COUNTS_PATH="zero_lag_ranking_stats.xml"
LIKELIHOOD_PATH="rankingstat.xml"
ZEROLAG_COUNTS_PATH="zerolag_rankingstatpdf.xml"
#
# pause for each iteration (seconds)
......@@ -98,7 +98,7 @@ while true ; do
# sum the noise and signal model ranking statistic histograms
# across jobs
date +"%H:%M:%S" >&2
gstlal_inspiral_marginalize_likelihood --verbose --ranking-stat-pdf --output ${BACKGROUND_COUNTS} ${RANKING_PDF_FILES} || break
gstlal_inspiral_marginalize_likelihood --verbose --marginalize ranking-stat-pdf --output ${BACKGROUND_COUNTS} ${RANKING_PDF_FILES} || break
rm -vf ${RANKING_PDF_FILES}
# collect and sum the current observed zero-lag ranking statistic
......@@ -114,7 +114,7 @@ while true ; do
ZEROLAG_COUNTS_URLS="${ZEROLAG_COUNTS_URLS} ${SERVER}${ZEROLAG_COUNTS_PATH}"
done || break
date +"%H:%M:%S" >&2
gstlal_inspiral_marginalize_likelihood --verbose --ranking-stat-pdf --output ${OUTPUT}.next.gz ${BACKGROUND_COUNTS} ${ZEROLAG_COUNTS_URLS} || break
gstlal_inspiral_marginalize_likelihood --verbose --marginzlize ranking-stat-pdf --output ${OUTPUT}.next.gz ${BACKGROUND_COUNTS} ${ZEROLAG_COUNTS_URLS} || break
mv -f ${OUTPUT}.next.gz ${OUTPUT} || break
done
rm -vf ${BACKGROUND_COUNTS}
......
......@@ -248,6 +248,8 @@ def parse_command_line():
if options.injection_file:
inj_name_dict = datasource.injection_dict_from_channel_list_with_node_range(options.injection_file)
else:
inj_name_dict = {}
#FIXME add consistency check?
bankcache = inspiral_pipe.parse_cache_str(options.bank_cache)
......@@ -421,13 +423,17 @@ for num_insp_nodes, (svd_banks, likefile, zerolikefile) in enumerate(zip(bank_gr
"min-instruments":options.min_instruments,
"min-log-L":options.min_log_L,
"time-slide-file":options.time_slide_file
},
input_files = {"ranking-stat-pdf":options.marginalized_likelihood_file},
output_files = {"output":"not_used.xml.gz",
"ranking-stat-output":likefile,
"zerolag-rankingstatpdf-filename":zerolikefile,
}
)
},
input_files = {
"ranking-stat-input":[likefile],
"ranking-stat-pdf":options.marginalized_likelihood_file
},
output_files = {
"output":"/dev/null",
"ranking-stat-output":likefile,
"zerolag-rankingstat-pdf":zerolikefile
}
)
if str("%04d" %num_insp_nodes) in inj_channel_dict:
# FIXME The node number for injection jobs currently follows the same
......@@ -467,12 +473,15 @@ for num_insp_nodes, (svd_banks, likefile, zerolikefile) in enumerate(zip(bank_gr
"min-instruments":options.min_instruments,
"min-log-L":options.min_log_L,
"time-slide-file":options.time_slide_file
},
input_files = {"ranking-stat-pdf":options.marginalized_likelihood_file,
"ranking-stat-input":[likefile]},
output_files = {"output":"not_used.xml.gz",
}
)
},
input_files = {
"ranking-stat-input":[likefile],
"ranking-stat-pdf":options.marginalized_likelihood_file
},
output_files = {
"output":"/dev/null"
}
)
def groups(l, n):
for i in xrange(0, len(l), n):
......@@ -538,13 +547,15 @@ dag.write_cache()
#
shutil.copy2(dagparts.which('gstlalcbcsummary'), os.path.expanduser("~/public_html/cgi-bin"))
shutil.copy2(dagparts.which('gstlalcbcnode'), os.path.expanduser("~/public_html/cgi-bin"))
cgibinpath = os.path.expanduser("~/public_html/cgi-bin")
if not os.path.isdir(cgibinpath):
os.makedirs(cgibinpath)
shutil.copy2(dagparts.which('gstlalcbcsummary'), cgibinpath)
shutil.copy2(dagparts.which('gstlalcbcnode'), cgibinpath)
query = "id=%s,%s&dir=%s&ifos=%s" % (jobTags[0], jobTags[-1], os.getcwd(), ",".join(sorted(bank_cache.keys())))
# Write the analysis to a special file that the summary page can find by default
webfile = open(os.path.join(os.path.expanduser("~/public_html/cgi-bin"), "gstlalcbc_analysis.txt"), "w")
webfile.write(query)
webfile.close()
with open(os.path.join(cgibinpath, "gstlalcbc_analysis.txt"), "w") as webfile:
webfile.write(query)
print >>sys.stderr, "\n\n NOTE! You can monitor the analysis at this url: %s/~%s/cgi-bin/gstlalcbcsummary?%s \n\n" % (inspiral_pipe.webserver_url(), os.environ['USER'], query)
if inj_jobTags:
print >>sys.stderr, "\n\n NOTE! You can monitor the injection analysis at this url: %s/~%s/cgi-bin/gstlalcbcsummary?id=%s,%s&dir=%s&ifos=%s \n\n" % (inspiral_pipe.webserver_url(), os.environ['USER'], inj_jobTags[0], inj_jobTags[-1], os.getcwd(), ",".join(sorted(bank_cache.keys())))
......@@ -215,6 +215,7 @@ class RankingStat(snglcoinc.LnLikelihoodRatioMixin):
self.numerator.finish()
self.denominator.finish()
self.zerolag.finish()
return self
@classmethod
def get_xml_root(cls, xml, name):
......@@ -270,6 +271,35 @@ class DatalessRankingStat(RankingStat):
# no zero-lag
self.numerator.finish()
self.denominator.finish()
return self
class OnlineFrakensteinRankingStat(RankingStat):
"""
Version of RankingStat with horizon distance history and trigger
rate history spliced in from another instance. Used to solve a
chicken-or-egg problem and assign ranking statistic values in an
aonline anlysis. NOTE: the donor data is not copied, instances of
this class hold references to the donor's data, so as it is
modified those modifications are immediately reflected here.
For safety's sake, instances cannot be written to or read from
files, cannot be marginalized together with other instances, nor
accept updates from new data.
"""
# NOTE: .__iadd__(), .copy() and I/O are forbidden, but these
# operations will be blocked by the .numerator and .denominator
# instances, no need to add extra code here to prevent these
# operations
def __init__(self, src, donor):
self.numerator = inspiral_lr.OnlineFrakensteinLnSignalDensity.splice(src.numerator, donor.numerator)
self.denominator = inspiral_lr.OnlineFrakensteinLnNoiseDensity.splice(src.denominator, donor.denominator)
def finish(self):
# no zero-lag
self.numerator.finish()
self.denominator.finish()
return self
#
......@@ -361,6 +391,13 @@ class RankingStatPDF(object):
raise ValueError("cannot be initialized from a RankingStat that is not for a specific set of templates")
self.template_ids = rankingstat.template_ids
#
# bailout used by codes that want all-zeros histograms
#
if not nsamples:
return
#
# run importance-weighted random sampling to populate
# binnings.
......
This diff is collapsed.
......@@ -116,7 +116,7 @@ def webserver_url():
#FIXME add more hosts as you need them
if "cit" in host or "ligo.caltech.edu" in host:
return "https://ldas-jobs.ligo.caltech.edu"
if "phys.uwm.edu" in host or "cgca.uwm.edu" in host:
if ".phys.uwm.edu" in host or ".cgca.uwm.edu" in host or ".nemo.uwm.edu" in host:
return "https://ldas-jobs.cgca.uwm.edu"
raise NotImplementedError("I don't know where the webserver is for this environment")
......
......@@ -532,6 +532,52 @@ class DatalessLnSignalDensity(LnSignalDensity):
raise NotImplementedError
class OnlineFrakensteinLnSignalDensity(LnSignalDensity):
"""
Version of LnSignalDensity with horizon distance history spliced in
from another instance. Used to solve a chicken-or-egg problem and
assign ranking statistic values in an aonline anlysis. NOTE: the
horizon history is not copied from the donor, instances of this
class hold a reference to the donor's data, so as it is modified
those modifications are immediately reflected here.
For safety's sake, instances cannot be written to or read from
files, cannot be marginalized together with other instances, nor
accept updates from new data.
"""
@classmethod
def splice(cls, src, Dh_donor):
self = cls(src.template_ids, src.instruments, src.delta_t, src.min_instruments)
for key, lnpdf in self.densities.items():
self.densities[key] = lnpdf.copy()
# NOTE: not a copy. we hold a reference to the donor's
# data so that as it is updated, we get the updates.
self.horizon_history = Dh_donor.horizon_history
return self
def __iadd__(self, other):
raise NotImplementedError
def increment(self, *args, **kwargs):
raise NotImplementedError
def copy(self):
raise NotImplementedError
def to_xml(self, name):
# I/O not permitted: the on-disk version would be
# indistinguishable from a real ranking statistic and could
# lead to accidents
raise NotImplementedError
@classmethod
def from_xml(cls, xml, name):
# I/O not permitted: the on-disk version would be
# indistinguishable from a real ranking statistic and could
# lead to accidents
raise NotImplementedError
#
# =============================================================================
#
......@@ -865,3 +911,49 @@ class DatalessLnNoiseDensity(LnNoiseDensity):
# indistinguishable from a real ranking statistic and could
# lead to accidents
raise NotImplementedError
class OnlineFrakensteinLnNoiseDensity(LnNoiseDensity):
"""
Version of LnNoiseDensity with trigger rate data spliced in from
another instance. Used to solve a chicken-or-egg problem and
assign ranking statistic values in an aonline anlysis. NOTE: the
trigger rate data is not copied from the donor, instances of this
class hold a reference to the donor's data, so as it is modified
those modifications are immediately reflected here.
For safety's sake, instances cannot be written to or read from
files, cannot be marginalized together with other instances, nor
accept updates from new data.
"""
@classmethod
def splice(cls, src, rates_donor):
self = cls(src.template_ids, src.instruments, src.delta_t, src.min_instruments)
for key, lnpdf in self.densities.items():
self.densities[key] = lnpdf.copy()
# NOTE: not a copy. we hold a reference to the donor's
# data so that as it is updated, we get the updates.
self.triggerrates = rates_donor.triggerrates
return self
def __iadd__(self, other):
raise NotImplementedError
def increment(self, *args, **kwargs):
raise NotImplementedError
def copy(self):
raise NotImplementedError
def to_xml(self, name):
# I/O not permitted: the on-disk version would be
# indistinguishable from a real ranking statistic and could
# lead to accidents
raise NotImplementedError
@classmethod
def from_xml(cls, xml, name):
# I/O not permitted: the on-disk version would be
# indistinguishable from a real ranking statistic and could
# lead to accidents
raise NotImplementedError
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment