diff --git a/gstlal-inspiral/bin/gstlal_inspiral_plotsummary b/gstlal-inspiral/bin/gstlal_inspiral_plotsummary index 207b70e2414fba900616df3bb09acc6dfef2c524..c3f318e7d6e42882dd1c249ac91ef10d0d2ea1d6 100755 --- a/gstlal-inspiral/bin/gstlal_inspiral_plotsummary +++ b/gstlal-inspiral/bin/gstlal_inspiral_plotsummary @@ -29,6 +29,7 @@ import bisect import itertools +import json import math import matplotlib import matplotlib.figure @@ -64,16 +65,22 @@ from ligo.lw import lsctables from ligo.lw.utils import segments as ligolw_segments from gstlal import far from gstlal import dagparts -from gstlal import gviz_api from gstlal.plotutil import golden_ratio -def to_google_json(fname, description, data): - json = open(fname,"w") - data_table = gviz_api.DataTable(description) - data_table.LoadData(data) - print >> json, data_table.ToJSon() - json.close() +def format_column(column): + return column.lower().replace(' ', '_').replace(':', '').replace(';', '') + + +def save_table(fname, fields, rows): + ### format data + fields = [{'key': format_column(col), 'label': col, 'sortable': True} for col in fields] + colnames = [field['key'] for field in fields] + items = [{col: val for col, val in zip(colnames, row)} for row in rows] + + ### write to disk + with open(fname, 'w') as f: + f.write(json.dumps({'fields': fields, 'items': items})) class SimInspiral(lsctables.SimInspiral): @@ -311,7 +318,7 @@ class CoincDatabase(object): for description, n in cursor.execute("SELECT description, COUNT(*) FROM coinc_definer NATURAL JOIN coinc_event GROUP BY coinc_def_id"): data.append([description, str(n)]) if self.sim_inspiral_table is None: - to_google_json("%s_%s" % (self.base, "plotsummary.json"), [('Summary of database:', 'string'), (plotsummary_json, 'string')], data) + save_table("%s_plot_summary.json" % self.base, ['Summary of database:', plotsummary_json], data) # # ============================================================================= @@ -585,35 +592,35 @@ SELECT distinct_ifos.ifos, count(*) FROM coinc_inspiral JOIN distinct_ifos ON (d def write_candidate_json(self, candidates, fname): data = [] # FIXME generalize for virgo - description = [ - ("Rank", "number"), - ("FAR (Hz)", "number"), - ("FAP", "number"), - ("ln Λ", "number"), - ("ρ", "number"), - ("<i>t</i>", "number"), - ("<i>M</i><sub>total</sub> / M<sub>⊙</sub>", "number"), - ("<i>M</i><sub>chirp</sub> / M<sub>⊙</sub>", "number"), - ("Participating<br>Instruments", "string"), - ("On<br>Instruments", "string"), - ("H1 ρ", "string"), - ("H1 χ<sup>2</sup>", "number"), - ("H1 m<sub>1</sub>", "number"), - ("H1 m<sub>2</sub>", "number"), - ("H1 s<sub>1z</sub>", "number"), - ("H1 s<sub>2z</sub>", "number"), - ("L1 ρ", "string"), - ("L1 χ<sup>2</sup>", "number"), - ("L1 m<sub>1</sub>", "number"), - ("L1 m<sub>2</sub>", "number"), - ("L1 s<sub>1z</sub>", "number"), - ("L1 s<sub>2z</sub>", "number"), - ("V1 ρ", "string"), - ("V1 χ<sup>2</sup>", "number"), - ("V1 m<sub>1</sub>", "number"), - ("V1 m<sub>2</sub>", "number"), - ("V1 s<sub>1z</sub>", "number"), - ("V1 s<sub>2z</sub>", "number") + fields = [ + "rank", + "FAR (Hz)", + "FAP", + "ln L", + "SNR", + "time", + "M total", + "M chirp", + "Participating IFOs", + "On IFOs", + "H1 SNR", + "H1 chisq", + "H1 m1", + "H1 m2", + "H1 S_1z", + "H1 S_2z", + "L1 SNR", + "L1 chisq", + "L1 m1", + "L1 m2", + "L1 S_1z", + "L1 S_2z", + "V1 SNR", + "V1 chisq", + "V1 m1", + "V1 m2", + "V1 S_1z", + "V1 S_2z" ] for rank, values in enumerate(candidates, 1): if values[0] is None: @@ -646,7 +653,7 @@ SELECT distinct_ifos.ifos, count(*) FROM coinc_inspiral JOIN distinct_ifos ON (d row.extend(ifodict["L1"]) row.extend(ifodict["V1"]) data.append(row) - to_google_json(fname, description, data) + save_table(fname, fields, data) def finish(self): self.candidates.sort() @@ -889,8 +896,8 @@ FROM for fig, filename_fragment, is_open_box in plot.finish(): yield fig, "%s_%s" % (filename_fragment, "".join(sorted(on_instruments))), is_open_box - to_google_json(self.base + "_injection_summary.json", [("Found||Missed", "string"), ("On Instruments", "string"), ("Participating Instruments", "string"), ("count", "number")], self.injection_summary_data) - to_google_json(self.base + "_missed_summary.json", [("On Instruments", "string"), ("Waveform", "string"), ("H1 <i>t</i>", "number"), ("L1 <i>t</i>", "number"), ("V1 <i>t</i>", "number"), ("<i>m</i><sub>1</sub>", "number"), ("<i>m</i><sub>2</sub>", "number"), ("<i>s</i><sub>1x</sub>", "number"), ("<i>s</i><sub>1y</sub>", "number"), ("<i>s</i><sub>1z</sub>", "number"), ("<i>s</i><sub>2x</sub>", "number"), ("<i>s</i><sub>2y</sub>", "number"), ("<i>s</i><sub>2z</sub>", "number"), ("D (Mpc)", "number"), ("Decisive D<sub>chirp,eff</sub> (Mpc)" , "number"), ("Inclination", "number"), ("H1 ρ", "number"), ("L1 ρ", "number"), ("V1 ρ", "number"), ("Decisive ρ", "number")], self.missed_summary_data) + save_table(self.base + "_injection_summary.json", ["Found || Missed", "On Instruments", "Participating IFOs", "count"], self.injection_summary_data) + save_table(self.base + "_missed_summary.json", ["On Instruments", "Waveform", "H1 time", "L1 time", "V1 time", "m1", "m2", "S_1x", "S_1y", "S_1z", "S_2x", "S_2y", "S_2z", "D (Mpc)", "Decisive D chirp,eff (Mpc)" , "Inclination", "H1 SNR", "L1 SNR", "V1 SNR", "Decisive SNR"], self.missed_summary_data) # @@ -1709,7 +1716,7 @@ class Segments(object): f = open("%s_segments.json" % self.base, "w") f.write(repr(self.data)) f.close() - to_google_json("%s_segment_table.json" % self.base, [("name","string"), ("ifos", "string"), ("start","number"), ("end", "number"), ("extent", "number")], self.table) + save_table("%s_segment_table.json" % self.base, ["name", "ifos", "start", "end", "extent"], self.table) # diff --git a/gstlal-inspiral/bin/gstlal_inspiral_summary_page b/gstlal-inspiral/bin/gstlal_inspiral_summary_page index 5c0db382d67e61218c353e43cb81fdc656b41706..ebb20e217343b11c31088c28c65df252b2dae8fd 100755 --- a/gstlal-inspiral/bin/gstlal_inspiral_summary_page +++ b/gstlal-inspiral/bin/gstlal_inspiral_summary_page @@ -33,32 +33,10 @@ SCHEDULE_URL = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vR1XwazhZYoVbMXF IMAGE_URL = 'https://results.ligo.uwm.edu/~gstlalcbc/share/vis/gstlal.png' -def sanitizetext(item): - # FIXME: hack to clean up formatted columns downstream, - # should really be handling this at the source - item = item.replace('⊙', '') - htmlr = re.compile('<.*?>') - alphanumr = re.compile('[\W_]+') - item = re.sub(htmlr, '', item) - return re.sub(alphanumr, '', item) - -def read_format_json(path): - """ - formats tabular summaries into a format readable by ligo-scald - FIXME: remove once formatting is changed upstream - """ - table = '' +def load_json(path): with open(path, 'r') as f: - table = json.loads(f.read()) + return json.loads(f.read()) - ### format data - # FIXME: hack to clean up formatted columns downstream, - # should really be handling this at the source - fields = [{'key': sanitizetext(col['id']), 'label': sanitizetext(col['label']), 'sortable': True} for col in table['cols']] - colnames = [field['key'] for field in fields] - items = [{col: val['v'] for col, val in zip(colnames, row['c'])} for row in table['rows']] - - return {'fields': fields, 'items': items} def parse_command_line(): parser = OptionParser(usage = "%prog [options] [file ...]", description = "%prog summary web page") @@ -111,10 +89,10 @@ sched_doc = report.Tab('Schedule', url=SCHEDULE_URL) summary += report.Header('Summary') -tabledata = read_format_json(os.path.join(opts.glob_path, '%s_plotsummary.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%s_plot_summary.json' % opts.output_user_tag[0])) summary += report.Table('Search Summary').save(tabledata, opts.webserver_dir) -tabledata = read_format_json(os.path.join(opts.glob_path, '%s_segment_table.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%s_segment_table.json' % opts.output_user_tag[0])) summary += report.Table('Segments').save(tabledata, opts.webserver_dir) summary += report.Description("This page was generated on %s by %s in the working directory: %s" % (time.strftime("%Y-%m-%d %H:%M"), os.environ["USER"], os.environ["PWD"])) @@ -177,7 +155,7 @@ injrecovery += report.Description("""Measuring the found and missed injections a injrecovery += report.Header("Found / Missed Summary Table") for tag in opts.output_user_tag: - tabledata = read_format_json(os.path.join(opts.glob_path, '%s_injection_summary.json' % tag)) + tabledata = load_json(os.path.join(opts.glob_path, '%s_injection_summary.json' % tag)) injrecovery += report.Description("Summary of missed and found injections broken up by detector time for %s" % tag) injrecovery += report.Table('Injection Summary Table for %s' % tag).save(tabledata, opts.webserver_dir) @@ -251,7 +229,7 @@ for tag in opts.output_user_tag: injrecovery += report.Header("Missed Table") for tag in opts.output_user_tag: - tabledata = read_format_json(os.path.join(opts.glob_path, '%s_missed_summary.json' % tag)) + tabledata = load_json(os.path.join(opts.glob_path, '%s_missed_summary.json' % tag)) injrecovery += report.Table('Missed Injection Table for %s' % tag).save(tabledata, opts.webserver_dir) thisreport += injrecovery @@ -331,7 +309,7 @@ money += report.Header('Closed Box Results') imgfooter = "Comparison of single time slide event rate to event rate expected from background as a function of threshold." money += report.ImageGrid("Rate vs. Threshold", grid_size=3, footer=imgfooter).glob(os.path.join(opts.glob_path, '*5_count_vs_*_closedbox*.png')) -tabledata = read_format_json(os.path.join(opts.glob_path, '%sbgsummary_table.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%sbgsummary_table.json' % opts.output_user_tag[0])) money += report.Table('Closed Box Summary Table').save(tabledata, opts.webserver_dir) if opts.open_box: @@ -339,7 +317,7 @@ if opts.open_box: imgfooter = "Comparison of single time slide event rate to event rate expected from background as a function of threshold." money += report.ImageGrid("Rate vs. Threshold", grid_size=3, footer=imgfooter).glob(os.path.join(opts.glob_path, '*5_count_vs_*_openbox*.png')) - tabledata = read_format_json(os.path.join(opts.glob_path, '%ssummary_table.json' % opts.output_user_tag[0])) + tabledata = load_json(os.path.join(opts.glob_path, '%ssummary_table.json' % opts.output_user_tag[0])) money += report.Table('Open Box Summary Table').save(tabledata, opts.webserver_dir) thisreport += money @@ -373,7 +351,7 @@ sched_doc = report.Tab('Schedule', url=SCHEDULE_URL) summary += report.Header('Summary') -tabledata = read_format_json(os.path.join(opts.glob_path, '%s_plotsummary.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%s_plot_summary.json' % opts.output_user_tag[0])) summary += report.Table('Search Summary').save(tabledata, opts.webserver_dir+'_lite') ## Include horizon distance plots @@ -384,7 +362,7 @@ summary += report.ImageGrid("Horizon distance", grid_size=1, footer=imgfooter).g summary += report.Header("Found / Missed Expected SNR") summary += report.Description("""Measuring the found and missed injections as a function of various parameters aids in diagnosing the pipeline as well as providing the expected sensitivity of the pipeline to real signals. The plots in this section show the missed and found injections for the various IFO times for coincident triggers.""") -tabledata = read_format_json(os.path.join(opts.glob_path, '%s_injection_summary.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%s_injection_summary.json' % opts.output_user_tag[0])) footer = "Summary of missed and found injections broken up by detector time for %s" % opts.output_user_tag[0] summary += report.Table('Injection Summary Table', footer=footer).save(tabledata, opts.webserver_dir+'_lite') @@ -394,7 +372,7 @@ summary += report.ImageGrid("Expected SNR vs. chirp mass", grid_size=4, footer=i ## Include missed injection table summary += report.Header("Missed Table") -tabledata = read_format_json(os.path.join(opts.glob_path, '%s_missed_summary.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%s_missed_summary.json' % opts.output_user_tag[0])) summary += report.Table('Missed Injection Table for %s' % opts.output_user_tag[0]).save(tabledata, opts.webserver_dir+'_lite') ## Include search sensitivity binned by source type @@ -428,14 +406,14 @@ summary += report.Description("This section provides the detection statistic plo imgfooter = "Comparison of single time slide event rate to event rate expected from background as a function of threshold." summary += report.ImageGrid("Rate vs. Threshold", grid_size=3, footer=imgfooter).glob(os.path.join(opts.glob_path, '*5_count_vs_*_closedbox*.png')) -tabledata = read_format_json(os.path.join(opts.glob_path, '%sbgsummary_table.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%sbgsummary_table.json' % opts.output_user_tag[0])) summary += report.Table('Closed Box Summary Table').save(tabledata, opts.webserver_dir+'_lite') if opts.open_box: imgfooter = "Comparison of single time slide event rate to event rate expected from background as a function of threshold." summary += report.ImageGrid("Rate vs. Threshold: Zero lag", grid_size=3, footer=imgfooter).glob(os.path.join(opts.glob_path, '*5_count_vs_*_openbox*.png')) - tabledata = read_format_json(os.path.join(opts.glob_path, '%ssummary_table.json' % opts.output_user_tag[0])) + tabledata = load_json(os.path.join(opts.glob_path, '%ssummary_table.json' % opts.output_user_tag[0])) summary += report.Table('Open Box Summary Table').save(tabledata, opts.webserver_dir+'_lite') reportlite += summary diff --git a/gstlal-inspiral/bin/gstlal_inspiral_summary_page_lite b/gstlal-inspiral/bin/gstlal_inspiral_summary_page_lite index e8a5a630a3cc2af47396930e9b1899adb1c50c15..55fd19729404adcd081dc24f697148058dc6cd92 100755 --- a/gstlal-inspiral/bin/gstlal_inspiral_summary_page_lite +++ b/gstlal-inspiral/bin/gstlal_inspiral_summary_page_lite @@ -30,32 +30,10 @@ SCHEDULE_URL = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vR1XwazhZYoVbMXF IMAGE_URL = 'https://results.ligo.uwm.edu/~gstlalcbc/share/vis/gstlal.png' -def sanitizetext(item): - # FIXME: hack to clean up formatted columns downstream, - # should really be handling this at the source - item = item.replace('⊙', '') - htmlr = re.compile('<.*?>') - alphanumr = re.compile('[\W_]+') - item = re.sub(htmlr, '', item) - return re.sub(alphanumr, '', item) - -def read_format_json(path): - """ - formats tabular summaries into a format readable by ligo-scald - FIXME: remove once formatting is changed upstream - """ - table = '' +def load_json(path): with open(path, 'r') as f: - table = json.loads(f.read()) + return json.loads(f.read()) - ### format data - # FIXME: hack to clean up formatted columns downstream, - # should really be handling this at the source - fields = [{'key': sanitizetext(col['id']), 'label': sanitizetext(col['label']), 'sortable': True} for col in table['cols']] - colnames = [field['key'] for field in fields] - items = [{col: val['v'] for col, val in zip(colnames, row['c'])} for row in table['rows']] - - return {'fields': fields, 'items': items} def parse_command_line(): parser = OptionParser(usage = "%prog [options] [file ...]", description = "%prog summary web page") @@ -111,7 +89,7 @@ summary += report.Header('Closed Box Results') imgfooter = "Comparison of single time slide event rate to event rate expected from background as a function of threshold." summary += report.ImageGrid("Rate vs. Threshold", grid_size=3, footer=imgfooter).glob(os.path.join(opts.glob_path, '*5_count_vs_*_closedbox*.png')) -tabledata = read_format_json(os.path.join(opts.glob_path, '%sbgsummary_table.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%sbgsummary_table.json' % opts.output_user_tag[0])) summary += report.Table('Closed Box Summary Table').save(tabledata, reportdir) if opts.open_box: @@ -119,14 +97,14 @@ if opts.open_box: imgfooter = "Comparison of single time slide event rate to event rate expected from background as a function of threshold." summary += report.ImageGrid("Rate vs. Threshold", grid_size=3, footer=imgfooter).glob(os.path.join(opts.glob_path, '*5_count_vs_*_openbox*.png')) - tabledata = read_format_json(os.path.join(opts.glob_path, '%ssummary_table.json' % opts.output_user_tag[0])) + tabledata = load_json(os.path.join(opts.glob_path, '%ssummary_table.json' % opts.output_user_tag[0])) summary += report.Table('Open Box Summary Table').save(tabledata, reportdir) ## Include found/missed as function of decisive characteristic SNR summary += report.Header("Found / Missed Expected SNR") summary += report.Description("""Measuring the found and missed injections as a function of various parameters aids in diagnosing the pipeline as well as providing the expected sensitivity of the pipeline to real signals. The plots in this section show the missed and found injections for the various IFO times for coincident triggers.""") -tabledata = read_format_json(os.path.join(opts.glob_path, '%s_injection_summary.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%s_injection_summary.json' % opts.output_user_tag[0])) footer = "Summary of missed and found injections broken up by detector time for %s" % opts.output_user_tag[0] summary += report.Table('Injection Summary Table', footer=footer).save(tabledata, reportdir) @@ -136,7 +114,7 @@ summary += report.ImageGrid("Expected SNR vs. chirp mass", grid_size=4, footer=i ## Include missed injection table summary += report.Header("Missed Table") -tabledata = read_format_json(os.path.join(opts.glob_path, '%s_missed_summary.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%s_missed_summary.json' % opts.output_user_tag[0])) summary += report.Table('Missed Injection Table for %s' % opts.output_user_tag[0]).save(tabledata, reportdir) @@ -153,7 +131,7 @@ imgfooter = "This is the distance at which you should see a 1.4 1.4 Msun binary summary += report.ImageGrid("Horizon distance", grid_size=1, footer=imgfooter).glob(os.path.join(opts.glob_path, '*HORIZON*.png')) ## Search summary -tabledata = read_format_json(os.path.join(opts.glob_path, '%s_plotsummary.json' % opts.output_user_tag[0])) +tabledata = load_json(os.path.join(opts.glob_path, '%s_plot_summary.json' % opts.output_user_tag[0])) summary += report.Table('Search Summary').save(tabledata, reportdir) ## Include Chi-square distribution plots