gstlal_inspiral_marginalize_likelihoods_online

#!/bin/bash
#
# Copyright (C) 2012,2014  Kipp Cannon
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

### This program runs gstlal_inspiral_marginalize_likelihood in a while True
### loop; See gstlal_inspiral_marginalize_likelihoods_online for more details
###
###
### This program is not meant to be executed standalone by a user. It should be
### part of a DAG managing a running gstlal_inspiral online analysis.
###
### This program takes two or more arguments;
###
### 1. The path of the output file name
### 2. One or more paths to text files each containing a single line giving
###    the root URL of a web server from which to retrieve event parameter
###    distribution data (e.g., "http://node001.ligo.caltech.edu")
###
### This program queries each running gstlal_inspiral job via the URL,
### computes PDFs of the likelihood ratio ranking statistics from the
### parameter distribution data, then marginalizes the ranking statistic PDFs
### across jobs and writes the result to the given output file.
###
### It continues to do that in an infinite loop with a 10 minute pause on
### each iteration.  Files are not overwritten directly, but rather via a
### temporary file and mv operation to ensure that no files are corrupted in
### a POSIX file environment.
###
### Review status
### -------------
###
### +---------------------------------------------+---------------------------------------------+------------+
### | Names                                       | Hash                                        | Date       |
### +=============================================+=============================================+============+
### | Florent, Jolien, Kipp, Chad                 | 0e96523b8846e5a4597ba3477c8462443470cd94    | 2015-05-15 |
### +---------------------------------------------+---------------------------------------------+------------+
###

#
# get the output file name
#

OUTPUT="${1}"
shift

#
# paths to data objects on each job's web management interface
#

LIKELIHOOD_PATH="rankingstat.xml"
ZEROLAG_COUNTS_PATH="zerolag_rankingstatpdf.xml"

#
# pause for each iteration (seconds)
#

SLEEP="600"

#
# loop forever
#

while true ; do
	echo "... sleeping for ${SLEEP} seconds ..."
	sleep ${SLEEP}

	# collect the current coinc parameter PDF data from each job, and
	# run some iterations of the ranking statistic sampler to generate
	# noise and signal model ranking statistic histograms for each job.
	# NOTE:  the zero-lag ranking statistic histograms in the files
	# generated here are all 0.
	RANKING_PDF_FILES=
	for REG in "$@" ; do
		date +"%H:%M:%S" >&2
		SERVER=$(cat ${REG}) || exit 1
		RANKING_PDF_FILE=$(mktemp --suffix .xml.gz) || exit 1
		RANKING_PDF_FILES="${RANKING_PDF_FILES} ${RANKING_PDF_FILE}"
		# get 4,000,000 samples total
		echo gstlal_inspiral_calc_rank_pdfs --verbose --ranking-stat-samples $(python -c "print 10000000 / $#") --output ${RANKING_PDF_FILE} ${SERVER}${LIKELIHOOD_PATH} || exit 1
		gstlal_inspiral_calc_rank_pdfs --verbose --ranking-stat-samples $(python -c "print 10000000 / $#") --output ${RANKING_PDF_FILE} ${SERVER}${LIKELIHOOD_PATH} || exit 1
	done || break
	date +"%H:%M:%S" >&2

	# sum the noise and signal model ranking statistic histograms
	# across jobs, and collect and sum the current observed zero-lag
	# ranking statistic histograms from all jobs. combine with the
	# noise and signal model ranking statistic histograms.  NOTE:  the
	# noise and signal model ranking statistic histograms in the
	# zero-lag counts files downloaded from the jobs must be all 0, and
	# the zero-lag counts in the output generated by
	# gstlal_inspiral_calc_rank_pdfs must be 0.  NOTE:  this is where
	# the zero-lag counts have the density estimation transform
	# applied.
	ZEROLAG_COUNTS_URLS=
	for REG in "$@" ; do
		SERVER=$(cat ${REG}) || exit 1
		ZEROLAG_COUNTS_URLS="${ZEROLAG_COUNTS_URLS} ${SERVER}${ZEROLAG_COUNTS_PATH}"
	done || break
	# NOTE we mix in previous samples
	cp -v ${OUTPUT} prev_${OUTPUT}
	gstlal_inspiral_marginalize_likelihood --verbose --marginalize ranking-stat-pdf --density-estimate-zero-lag --output ${OUTPUT} ${RANKING_PDF_FILES} prev_${OUTPUT} ${ZEROLAG_COUNTS_URLS} || break
	rm -vf prev_${OUTPUT}
	date +"%H:%M:%S" >&2
	rm -vf ${RANKING_PDF_FILES}
done

#
# if this program ends its an error, always, so condor will restart it
#

exit 1