Verified Commit 6f7f2e00 authored by Thomas Almeida's avatar Thomas Almeida
Browse files

cohfar: Reduce the size of outputted XML files by only using active IFOs

Previously, every possible combination of existing IFO was outputted as
a part of the XML dump from `cohfar`. This meant that the size of the
XML dump wasn't tied to the actual number of IFOs in use, and would
continue to exponentially increase in size as we increased the number of
IFOs that might be in play (e.g. KAGRA, LIGO-India, etc.).

To solve this, we have moved to using the total combination (as an index
into IFOComboMap) as a store of the active IFOs, and using
`__builtin_popcount()` to get the total number of IFOs that are in use
in that combination. This means that we can quickly and efficiently
determine the maximum number of detectors that we actually need to print
stats for, with no additional information required.

-- Tom Almeida
parent b4f563eb
......@@ -96,14 +96,7 @@ typedef struct {
TriggerStats **multistats;
GString *rank_xmlname;
GString *feature_xmlname;
int ncombo;
int icombo;
} TriggerStatsXML;
typedef TriggerStats **TriggerStatsPointer;
typedef struct {
TriggerStatsPointer *plist;
int size;
int pos;
} TriggerStatsPointerList;
#endif /* __BACKGROUND_STATS_H__ */
/*
* Copyright (C) 2015 Qi Chu <qi.chu@uwa.edu.au>
* Copyright (C) 2015 Qi Chu <qi.chu@uwa.edu.au>,
* 2020 Tom Almeida <tom@tommoa.me>,
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
......@@ -48,7 +49,7 @@ Bins2D *bins2D_create_long(double cmin_x,
double cmax_y,
int nbin_y);
TriggerStats **trigger_stats_create(int ncombo);
TriggerStats **trigger_stats_create(int icombo);
int bins1D_get_idx(double val, Bins1D *bins);
......
/*
* Copyright (C) 2015 Qi Chu <qi.chu@uwa.edu.au>
* Copyright (C) 2015 Qi Chu <qi.chu@uwa.edu.au>,
* 2020 Tom Almeida <tom@tommoa.me>,
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
......@@ -103,30 +104,27 @@ static gboolean cohfar_accumbackground_sink_event(GstPad *pad, GstEvent *event);
static void cohfar_accumbackground_dispose(GObject *object);
static void update_stats_icombo(PostcohInspiralTable *intable,
int icombo,
int ncombo,
TriggerStatsXML *stats) {
int nifo, isingle, write_ifo_mapping[MAX_NIFO];
// update the multi-IFO background at the last bin.
if (icombo > -1) {
int nifo, ifo;
nifo = __builtin_popcount(stats->icombo + 1);
if (stats->icombo > -1) {
// update the multi-IFO background at the last bin.
trigger_stats_feature_rate_update(
(double)(intable->cohsnr), (double)intable->cmbchisq,
stats->multistats[ncombo - 1]->feature,
stats->multistats[ncombo - 1]);
stats->multistats[nifo]->feature, stats->multistats[nifo]);
nifo = strlen(intable->ifos) / IFO_LEN;
/* add single detector stats */
get_write_ifo_mapping(IFOComboMap[icombo].name, nifo,
write_ifo_mapping);
// update single-IFO background according the single-IFO decomposition
for (isingle = 0; isingle < nifo; isingle++) {
int write_isingle = write_ifo_mapping[isingle];
trigger_stats_feature_rate_update(
(double)(*(&(intable->snglsnr_H) + write_isingle)),
(double)(*(&(intable->chisq_H) + write_isingle)),
stats->multistats[write_isingle]->feature,
stats->multistats[write_isingle]);
int index;
for (ifo = 0, index = 0; ifo < MAX_NIFO; ifo++) {
if ((stats->icombo + 1) & (1 << ifo)) {
trigger_stats_feature_rate_update(
(double)((&intable->snglsnr_H)[ifo]),
(double)((&intable->chisq_H)[ifo]),
stats->multistats[index]->feature, stats->multistats[index]);
++index;
}
}
}
}
......@@ -171,14 +169,14 @@ static GstFlowReturn cohfar_accumbackground_chain(GstPad *pad,
// TriggerStats **stats_prompt = element->stats_prompt;
// TriggerStatsPointerList *stats_list = element->stats_list;
// /* reset stats_prompt */
// trigger_stats_reset(stats_prompt, element->ncombo);
// trigger_stats_reset(stats_prompt, element->nifo);
/*
* reset stats in the stats_list in order to input new background points
*/
// int pos = stats_list->pos;
// TriggerStats **cur_stats_in_list = stats_list->plist[pos];
// trigger_stats_reset(cur_stats_in_list, element->ncombo);
// trigger_stats_reset(cur_stats_in_list, element->nifo);
/*
* calculate number of output postcoh entries
......@@ -203,7 +201,7 @@ static GstFlowReturn cohfar_accumbackground_chain(GstPad *pad,
/* allocate extra space for prompt stats */
// int out_size = sizeof(PostcohInspiralTable) * outentries +
// sizeof(TriggerStats) * ncombo;
// sizeof(TriggerStats) * (nifo + 1);
int out_size = sizeof(PostcohInspiralTable) * outentries;
result = gst_pad_alloc_buffer(srcpad, 0, out_size, caps, &outbuf);
if (result != GST_FLOW_OK) {
......@@ -235,13 +233,13 @@ static GstFlowReturn cohfar_accumbackground_chain(GstPad *pad,
}
if (intable->is_background == FLAG_BACKGROUND) {
update_stats_icombo(
intable, icombo, element->ncombo,
bgstats); // update the last ncombo and single IFO stats
intable,
bgstats); // update the last combination and single IFO stats
} else if (intable->is_background
== FLAG_FOREGROUND) { /* coherent trigger entry */
update_stats_icombo(
intable, icombo, element->ncombo,
zlstats); // update the last ncombo and single IFO stats
intable,
zlstats); // update the last combination and single IFO stats
memcpy(outtable, intable, sizeof(PostcohInspiralTable));
outtable++;
} else {
......@@ -252,22 +250,15 @@ static GstFlowReturn cohfar_accumbackground_chain(GstPad *pad,
//
// Note that this is inverted because icombo is sum(1 << index) - 1
if ((icombo + 1) & icombo) {
nifo = strlen(intable->ifos) / IFO_LEN;
nifo = __builtin_popcount(icombo + 1);
/* add single detector stats */
get_write_ifo_mapping(IFOComboMap[icombo].name, nifo,
element->write_ifo_mapping);
for (isingle = 0; isingle < nifo; isingle++) {
int write_isingle = element->write_ifo_mapping[isingle];
trigger_stats_livetime_inc(bgstats->multistats,
write_isingle);
trigger_stats_livetime_inc(zlstats->multistats,
write_isingle);
for (isingle = 0; isingle <= nifo; isingle++) {
trigger_stats_livetime_inc(bgstats->multistats, isingle);
trigger_stats_livetime_inc(zlstats->multistats, isingle);
}
trigger_stats_livetime_inc(bgstats->multistats,
element->ncombo - 1);
trigger_stats_livetime_inc(zlstats->multistats,
element->ncombo - 1);
}
memcpy(outtable, intable, sizeof(PostcohInspiralTable));
outtable++;
......@@ -422,7 +413,7 @@ static void cohfar_accumbackground_set_property(GObject *object,
case PROP_IFOS:
element->ifos = g_value_dup_string(value);
element->nifo = strlen(element->ifos) / IFO_LEN;
element->ncombo = get_ncombo(element->nifo);
element->icombo = get_icombo(element->ifos);
element->bgstats =
trigger_stats_xml_create(element->ifos, STATS_XML_TYPE_BACKGROUND);
element->zlstats =
......
......@@ -32,8 +32,8 @@
#include <glib.h>
#include <gst/base/gstbasetransform.h>
#include <gst/gst.h>
#include <postcohtable.h>
#include <libxml/xmlwriter.h>
#include <postcohtable.h>
G_BEGIN_DECLS
#define COHFAR_ACCUMBACKGROUND_TYPE (cohfar_accumbackground_get_type())
......@@ -59,7 +59,7 @@ typedef struct {
char *ifos;
int nifo;
int ncombo; // ifo combination
int icombo; // ifo combination
int write_ifo_mapping[MAX_NIFO];
TriggerStatsXML *bgstats;
TriggerStatsXML *zlstats;
......
......@@ -293,9 +293,9 @@ static GstFlowReturn cohfar_assignfar_transform_ip(GstBaseTransform *trans,
fprintf(stderr, "icombo not found, cohfar_assignfar\n");
exit(0);
}
cur_stats = element->bgstats_1w->multistats[element->ncombo - 1];
cur_stats = element->bgstats_1w->multistats[element->nifo];
if (icombo > -1 && cur_stats->nevent > MIN_BACKGROUND_NEVENT) {
update_trigger_fars(table, element->ncombo - 1, element);
update_trigger_fars(table, element->nifo, element);
}
}
}
......@@ -344,7 +344,7 @@ static void cohfar_assignfar_set_property(GObject *object,
case PROP_IFOS:
element->ifos = g_value_dup_string(value);
element->nifo = strlen(element->ifos) / IFO_LEN;
element->ncombo = get_ncombo(element->nifo);
element->icombo = get_icombo(element->ifos);
element->bgstats_1w =
trigger_stats_xml_create(element->ifos, STATS_XML_TYPE_BACKGROUND);
element->bgstats_1d =
......
......@@ -54,7 +54,7 @@ typedef struct {
char *ifos;
int nifo;
int ncombo; // ifo combination
int icombo; // ifo combination
int hist_trials;
TriggerStatsXML *bgstats_2h;
TriggerStatsXML *bgstats_1d;
......
/*
* Copyright (C) 2015 Qi Chu <qi.chu@uwa.edu.au>
* Copyright (C) 2015 Qi Chu <qi.chu@uwa.edu.au>,
* 2020 Tom Almeida <tom@tommoa.me>,
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
......@@ -44,13 +45,21 @@ static void parse_opts(int argc,
*ptype = g_strdup("all");
*update_pdf = 0;
int option_index = 0;
struct option long_opts[] = { { "input", required_argument, 0, 'i' },
{ "input-format", required_argument, 0, 'f' },
{ "output", required_argument, 0, 'o' },
{ "ifos", required_argument, 0, 'd' },
{ "type", required_argument, 0, 'u' },
{ "update-pdf", no_argument, 0, 'p' },
{ 0, 0, 0, 0 } };
struct option long_opts[] = {
// A comma separated list of files to use for input.
{ "input", required_argument, 0, 'i' },
// The format of the input files. One of "data" or "stats".
{ "input-format", required_argument, 0, 'f' },
// The name of the file to output.
{ "output", required_argument, 0, 'o' },
// The IFOs to use.
{ "ifos", required_argument, 0, 'd' },
// The type of stat. One of "background", "zerolag", "signal" or "all".
{ "type", required_argument, 0, 'u' },
// Should we update the PDF?
{ "update-pdf", no_argument, 0, 'p' },
{ 0, 0, 0, 0 }
};
int opt;
while (
(opt = getopt_long(argc, argv, "i:f:o:d:u:p:", long_opts, &option_index))
......@@ -131,19 +140,18 @@ void cohfar_get_stats_from_file(gchar **in_fnames,
TriggerStatsXML *stats_out,
int *hist_trials) {
gchar **ifname;
int icombo;
int ifo;
for (ifname = in_fnames; *ifname; ifname++) {
#ifdef __DEBUG__
printf("%s\n", *ifname);
#endif
trigger_stats_xml_from_xml(stats_in, hist_trials, *ifname);
for (icombo = 0; icombo < stats_in->ncombo; icombo++) {
trigger_stats_feature_rate_add(
stats_out->multistats[icombo]->feature,
stats_in->multistats[icombo]->feature,
stats_out->multistats[icombo]);
for (ifo = 0; ifo <= __builtin_popcount(stats_in->icombo + 1); ifo++) {
trigger_stats_feature_rate_add(stats_out->multistats[ifo]->feature,
stats_in->multistats[ifo]->feature,
stats_out->multistats[ifo]);
trigger_stats_livetime_add(stats_out->multistats,
stats_in->multistats, icombo);
stats_in->multistats, ifo);
}
}
}
......@@ -157,7 +165,7 @@ static int get_type(gchar **ptype) {
static int process_stats_full(
gchar **in_fnames, int nifo, gchar **pifos, gchar **pout, int *update_pdf) {
int icombo, ncombo = get_ncombo(nifo), hist_trials;
int ifo, hist_trials;
TriggerStatsXML *zlstats_in =
trigger_stats_xml_create(*pifos, STATS_XML_TYPE_ZEROLAG);
TriggerStatsXML *zlstats_out =
......@@ -180,24 +188,21 @@ static int process_stats_full(
cohfar_get_stats_from_file(in_fnames, bgstats_in, bgstats_out,
&hist_trials);
if (*update_pdf == 1) {
for (icombo = 0; icombo < ncombo; icombo++) {
for (ifo = 0; ifo < nifo; ifo++) {
trigger_stats_feature_rate_to_pdf(
sgstats_out->multistats[icombo]->feature);
trigger_stats_feature_to_rank(
sgstats_out->multistats[icombo]->feature,
sgstats_out->multistats[icombo]->rank);
sgstats_out->multistats[ifo]->feature);
trigger_stats_feature_to_rank(sgstats_out->multistats[ifo]->feature,
sgstats_out->multistats[ifo]->rank);
trigger_stats_feature_rate_to_pdf(
zlstats_out->multistats[icombo]->feature);
trigger_stats_feature_to_rank(
zlstats_out->multistats[icombo]->feature,
zlstats_out->multistats[icombo]->rank);
zlstats_out->multistats[ifo]->feature);
trigger_stats_feature_to_rank(zlstats_out->multistats[ifo]->feature,
zlstats_out->multistats[ifo]->rank);
trigger_stats_feature_rate_to_pdf(
bgstats_out->multistats[icombo]->feature);
trigger_stats_feature_to_rank(
bgstats_out->multistats[icombo]->feature,
bgstats_out->multistats[icombo]->rank);
bgstats_out->multistats[ifo]->feature);
trigger_stats_feature_to_rank(bgstats_out->multistats[ifo]->feature,
bgstats_out->multistats[ifo]->rank);
}
}
......@@ -234,18 +239,17 @@ static int process_stats_single(gchar **in_fnames,
gchar **pout,
int type,
int *update_pdf) {
int icombo, ncombo = get_ncombo(nifo), hist_trials;
int ifo, hist_trials;
TriggerStatsXML *stats_in = trigger_stats_xml_create(*pifos, type);
TriggerStatsXML *stats_out = trigger_stats_xml_create(*pifos, type);
cohfar_get_stats_from_file(in_fnames, stats_in, stats_out, &hist_trials);
if (*update_pdf == 1) {
for (icombo = 0; icombo < ncombo; icombo++) {
for (ifo = 0; ifo < nifo; ifo++) {
trigger_stats_feature_rate_to_pdf(
stats_out->multistats[icombo]->feature);
trigger_stats_feature_to_rank(
stats_out->multistats[icombo]->feature,
stats_out->multistats[icombo]->rank);
stats_out->multistats[ifo]->feature);
trigger_stats_feature_to_rank(stats_out->multistats[ifo]->feature,
stats_out->multistats[ifo]->rank);
}
}
xmlTextWriterPtr stats_writer = NULL;
......@@ -288,24 +292,18 @@ int main(int argc, char *argv[]) {
trigger_stats_xml_create(*pifos, STATS_XML_TYPE_BACKGROUND);
TriggerStatsXML *bgstats_out =
trigger_stats_xml_create(*pifos, STATS_XML_TYPE_BACKGROUND);
int ncombo = get_ncombo(nifo);
// FIXME: hardcoded to only update the last stats
trigger_stats_feature_rate_update_all(
data_dim1, data_dim2, bgstats_out->multistats[ncombo - 1]->feature,
bgstats_out->multistats[ncombo - 1]);
data_dim1, data_dim2, bgstats_out->multistats[nifo]->feature,
bgstats_out->multistats[nifo]);
trigger_stats_feature_rate_to_pdf(
bgstats_out->multistats[ncombo - 1]->feature);
trigger_stats_feature_to_rank(
bgstats_out->multistats[ncombo - 1]->feature,
bgstats_out->multistats[ncombo - 1]->rank);
bgstats_out->multistats[nifo]->feature);
trigger_stats_feature_to_rank(bgstats_out->multistats[nifo]->feature,
bgstats_out->multistats[nifo]->rank);
if (data_dim1) {
free(data_dim1);
free(data_dim2);
}
// trigger_stats_pdf_from_data(data_dim1, data_dim2,
// stats_out[ncombo-1]->rate->lgsnr_bins,
// stats_out[ncombo-1]->rate->lgchisq_bins, stats_out[ncombo-1]->pdf);
} else if (g_strcmp0(*pfmt, "stats") == 0) {
if (type == STATS_XML_TYPE_ALL) {
rc = process_stats_full(in_fnames, nifo, pifos, pout, update_pdf);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment