From f1681c3774abc489651e2b71ff0e268aa4f191ff Mon Sep 17 00:00:00 2001 From: Branson Stephens <branson.stephens@ligo.org> Date: Mon, 22 Feb 2016 12:55:53 -0600 Subject: [PATCH] Incorporated new version of event rate submission plot and a plot of recent events on the landing page. The latter is only shown if settings.SHOW_RECENT_EVENTS_ON_HOME is True. Setting it to false for now, since we are not in a run. --- .../commands/write_binned_counts.py | 160 +++++++++++ gracedb/reports.py | 25 +- gracedb/view_utils.py | 44 ++- gracedb/views.py | 9 + settings/default.py | 6 + templates/gracedb/histogram.html | 260 ++++++++++++------ templates/gracedb/index.html | 178 +++++++++++- 7 files changed, 594 insertions(+), 88 deletions(-) create mode 100644 gracedb/management/commands/write_binned_counts.py diff --git a/gracedb/management/commands/write_binned_counts.py b/gracedb/management/commands/write_binned_counts.py new file mode 100644 index 000000000..0a88b3cb7 --- /dev/null +++ b/gracedb/management/commands/write_binned_counts.py @@ -0,0 +1,160 @@ +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings + +from gracedb.models import Event, Pipeline, Search, Group + +from datetime import timedelta +from dateutil import parser +from django.utils import timezone +import pytz +import json + +#------------------------------------------------------------------------------------ +# Utilities +#------------------------------------------------------------------------------------ + +# get_counts_for_bin +# Takes as input: +# - the lower bin boundary (a naive datetime object in UTC) +# - the bin_width in hours +# - the pipeline we are interested in +# Returns the number of events in that bin, excluding MDC and Test. +MDC = Search.objects.filter(name='MDC').first() +Test = Group.objects.filter(name='Test').first() + +def get_counts_for_bin(lbb, bin_width, pipeline): + ubb = lbb + timedelta(hours=bin_width) + events = Event.objects.filter(pipeline=pipeline, created__range=(lbb, ubb)) + if MDC: + events = events.exclude(search=MDC) + if Test: + events = events.exclude(group=Test) + return events.count() + +# given a date string, parse it and localize to UTC if necessary +def parse_and_localize(date_string): + if not date_string: + return None + dt = parser.parse(date_string) + if not dt.tzinfo: + dt = pytz.utc.localize(dt) + return dt + +# make a list of pipeline objects +PIPELINES = [] +for n in settings.BINNED_COUNT_PIPELINES: + try: + PIPELINES.append(Pipeline.objects.get(name=n)) + except: + pass + +OTHER_PIPELINES = [] +for p in Pipeline.objects.all(): + if p.name not in PIPELINES: + OTHER_PIPELINES.append(p) + +def get_record(lbb, bin_width): + bc = lbb + timedelta(hours=bin_width/2) + r = { 'time': bc, 'delta_t': bin_width } + total = 0 + for p in PIPELINES: + count = get_counts_for_bin(lbb, bin_width, p) + total += count + r[p.name] = count + other = 0 + for p in OTHER_PIPELINES: + other += get_counts_for_bin(lbb, bin_width, p) + r['Other'] = other + total += other + r['Total'] = total + return r + +#------------------------------------------------------------------------------------ +# Binned counts command +#------------------------------------------------------------------------------------ +class Command(BaseCommand): + help = 'Manage the binned counts file used for plotting rates.' + + def add_arguments(self, parser): + # start and end should be ISO-8601 strings + # they will be interpreted as UTC, of course + # delta is an integer in hours + parser.add_argument('start') + parser.add_argument('end') + parser.add_argument('delta', type=int) + + def handle(self, *args, **options): + + # First of all, that bin width had better be an even number of hours. + bin_width = options['delta'] + if bin_width % 2 != 0: + raise ValueError("Bin width must be divisible by 2. Sorry.") + + # Let's take our desired range and turn it into UTC datetime objects + start = parse_and_localize(options['start']) + end = parse_and_localize(options['end']) + + duration = end - start + # This timedelta has days, seconds, and total seconds. + # What we want to verify is that is an integer number of hours. + # That is, the total seconds should be divisible by 3600. + hours, r_seconds = divmod(duration.total_seconds(), 3600) + if r_seconds != 0.0: + msg = "The start and end times must be separated by an integer number of hours." + raise ValueError(msg) + + # Now we need to verify that the number of hours is divisible by our + # bin width + bins, r_hours = divmod(hours, bin_width) + bins = int(bins) + + if r_hours != 0.0: + msg = "The start and end times must correspond to an integer number of bins." + raise ValueError(msg) + + # read in the file and interpret it as JSON + f = None + try: + f = open(settings.BINNED_COUNT_FILE, 'r') + except: + pass + + records = [] + if f: + try: + records = json.loads(f.read()) + except: + pass + f.close() + + # process the records so that the time is a datetime for all of them + # Note that the times here are at the bin centers + def dt_record(r): + r['time'] = parse_and_localize(r['time']) + return r + records = [dt_record(r) for r in records] + + # accumlate the necessary records + new_records = [] + for i in range(bins): + lbb = start + timedelta(hours = i*bin_width) + bc = lbb + timedelta(hours = bin_width/2) + # look for an existing record with the desired lower bin + # boundary and delta. + found = False + for r in records: + if bc == r['time'] and bin_width == r['delta_t']: + found = True + new_records.append(r) + if not found: + new_records.append(get_record(lbb, bin_width)) + + def strtime_record(r): + r['time'] = r['time'].replace(tzinfo=None).isoformat() + return r + new_records = [strtime_record(r) for r in new_records] + + # write out the file + f = open(settings.BINNED_COUNT_FILE, 'w') + f.write(json.dumps(new_records)) + f.close() diff --git a/gracedb/reports.py b/gracedb/reports.py index 9725cc5e4..5c3676153 100644 --- a/gracedb/reports.py +++ b/gracedb/reports.py @@ -4,7 +4,7 @@ from django.template import RequestContext from django.shortcuts import render_to_response from django.conf import settings -from gracedb.models import Event +from gracedb.models import Event, Group, Search from gracedb.permission_utils import filter_events_for_user from gracedb.permission_utils import internal_user_required from django.db.models import Q @@ -25,9 +25,11 @@ import StringIO import base64 import sys import calendar -from datetime import timedelta +from datetime import timedelta, datetime from utils import posixToGpsTime from django.utils import timezone +import pytz +import json @internal_user_required def histo(request): @@ -57,10 +59,18 @@ def histo(request): # uptime = None # Rate information + #try: + # rate_info = open(settings.RATE_INFO_FILE).read() + #except IOError: + # rate_info = None + + # For the binned counts, read in the contents of the file. try: - rate_info = open(settings.RATE_INFO_FILE).read() - except IOError: - rate_info = None + f = open(settings.BINNED_COUNT_FILE, 'r') + binned_counts = f.read() + f.close() + except: + binned_counts = None return render_to_response( 'gracedb/histogram.html', @@ -68,7 +78,8 @@ def histo(request): #'ifar' : ifar, #'uptime' : uptime, #'rate' : json.dumps(rate_data(request)), - 'rate' : rate_info, + #'rate' : rate_info, + 'binned_counts': binned_counts, 'url_prefix' : settings.REPORT_INFO_URL_PREFIX, }, context_instance=RequestContext(request)) @@ -315,5 +326,3 @@ def cbc_report(request, format=""): { 'form' : form, }, context_instance=RequestContext(request)) - - diff --git a/gracedb/view_utils.py b/gracedb/view_utils.py index 1b0f0c9b0..c19b6bc2b 100644 --- a/gracedb/view_utils.py +++ b/gracedb/view_utils.py @@ -6,7 +6,7 @@ from django.utils.html import escape, urlize #from django.utils.http import urlquote from django.utils.safestring import mark_safe -from gracedb.models import SingleInspiral +from gracedb.models import SingleInspiral, Event, Search, Group from utils.vfile import VersionedFile from permission_utils import is_external @@ -32,6 +32,9 @@ import pytz import time import calendar +from django.utils import timezone +from datetime import datetime, timedelta + SERVER_TZ = pytz.timezone(settings.TIME_ZONE) def timeToUTC(dt): if not dt.tzinfo: @@ -849,3 +852,42 @@ def check_query_far_range(q, floor=settings.VOEVENT_FAR_FLOOR): elif c[0] == 'far__range' and c[1][1] < floor: raise BadFARRange +#--------------------------------------------------------------------------------------- +#--------------------------------------------------------------------------------------- +# Get a serialized list of recent events for use with the d3 visualization of +# recent events. +#--------------------------------------------------------------------------------------- +#--------------------------------------------------------------------------------------- + +def get_recent_events_string(request): + #t_high = datetime(2016, 1, 12, 16, 0) # End of O1 + #t_high = pytz.utc.localize(t_high) + t_high = timezone.now() + dt = timedelta(days=7) + t_low = t_high - dt + # XXX Warning: If you open this up to non-internal users, you need + # to filter these events. + events = Event.objects.filter(created__range=(t_low, t_high)) + + # Explicitly filter out MDC and Test events + try: + mdc = Search.objects.get(name='MDC') + events = events.exclude(search=mdc) + except: + pass + + try: + test = Group.objects.get(name='Test') + events = events.exclude(group=test) + except: + pass + + if events.count() == 0: + return '' + + event_list = [ {'pipeline': e.pipeline.name, + 'graceid': e.graceid(), + 'created': e.created.isoformat() } for e in events ] + + return json.dumps(event_list) + diff --git a/gracedb/views.py b/gracedb/views.py index 71a07c4e2..1c409d9b5 100644 --- a/gracedb/views.py +++ b/gracedb/views.py @@ -25,6 +25,7 @@ from view_logic import create_emobservation from view_logic import create_label from view_utils import assembleLigoLw, get_file from view_utils import flexigridResponse, jqgridResponse +from view_utils import get_recent_events_string import os from django.conf import settings @@ -94,6 +95,14 @@ def index(request): events = Event.objects.filter(labelling__label__name=label_name) context['signoff_graceids'] = [e.graceid() for e in events] + recent_events = '' + if request.user and not is_external(request.user) and settings.SHOW_RECENT_EVENTS_ON_HOME: + try: + recent_events = get_recent_events_string(request) + except Exception, e: + pass + context['recent_events'] = recent_events + return render_to_response('gracedb/index.html', context, context_instance=RequestContext(request)) diff --git a/settings/default.py b/settings/default.py index 4897d5177..9010637b2 100644 --- a/settings/default.py +++ b/settings/default.py @@ -203,6 +203,12 @@ REPORTS_IFAR = [ ), ] +# Stuff for the new rates plot +BINNED_COUNT_PIPELINES = ['gstlal', 'MBTAOnline', 'CWB', 'LIB', 'gstlal-spiir' ] +BINNED_COUNT_FILE = "/home/gracedb/data/binned_counts.json" + +# Whether or not to show the recent events on the landing page +SHOW_RECENT_EVENTS_ON_HOME = False # RSS Feed Defaults FEED_MAX_RESULTS = 50 diff --git a/templates/gracedb/histogram.html b/templates/gracedb/histogram.html index 5068e4069..b50cea1a2 100644 --- a/templates/gracedb/histogram.html +++ b/templates/gracedb/histogram.html @@ -17,87 +17,48 @@ function toggle(id) { } </script> -{# http/https depending on this pages' protocol #} -<script src="http{% if request.is_secure %}s{% endif %}://ajax.googleapis.com/ajax/libs/dojo/1.7.1/dojo/dojo.js" - data-dojo-config="async: true, isDebug: false, parseOnLoad: true"> -</script> - -<script> - {% if rate %} - timeData = {{ rate|safe }}; - {% else %} - timeData = [ - { x: 1, y: 1 }, - { x: 2, y: 3 }, - { x: 3, y: 2 }, - { x: 4, y: 4 }, - { x: 5, y: 5 }, - { x: 6, y: 6 }, - { x: 7, y: 7 } - ]; - {% endif %} - - require([ - "dojo/parser", - "dojox/charting/Chart", - "dojox/charting/themes/Claro", // Shrooms, Tom - "dojox/charting/plot2d/Lines", - "dojox/charting/widget/Legend", - "dojox/charting/axis2d/Default", - "dojo/domReady!" - ], function (parser, Chart, theme, Lines, Legend ) { - - var chart = new Chart("CN"); - - function xlabel(n) { - console.log("Got: " + n); - var dt = new Date(); - dt.setTime(1000*n); - return (dt.getMonth()+1)+"/"+dt.getDate()+"/"+dt.getFullYear(); - }; - - chart.setTheme(theme); - chart.addPlot("default", { - type: Lines, - }); - chart.addAxis("x", {labelFunc:xlabel}); - chart.addAxis("y", { vertical: true }); - - chart.addSeries("Total / Day", timeData['total']); - // chart.addSeries("Low Mass", timeData['LM']); - // chart.addSeries("Omega", timeData['Omega']); - // chart.addSeries("MBTA", timeData['MBTA']); - // chart.addSeries("cWB", timeData['CWB']); - - chart.render(); - - var legend = new Legend({ chart: chart }, "leg1"); - } - ); -</script> - {% endblock %} {% block content %} -<br/> -<a href="{% url "cbc_report" %}"><h3>Dynamic CBC Report</h3></a> -<br/> -<br/> +<!-- some style stuff for the plot --> +<style> -<a name="latency" href="javascript:toggle('latency');"><h3>Latency</h3></a> +.axis text { + font: 16px sans-serif; + font-weight: bold; +} -<div id="latency" style="display:none;"> -{% if table %} - {{ table|safe }} -{% else %} - No Latency data. -{% endif %} -</div> +.axis path, +.axis line { + fill: none; + stroke: #000; + stroke-width: 2px; + shape-rendering: crispEdges; +} + +.x.axis path { + display: none; +} + +.line { + fill: none; + stroke: steelblue; + stroke-width: 3px; +} + +.legend { + font-size: 16px; + font-weight: bold; + text-anchor: start; + cursor: pointer; +} + +</style> <br/> -<br/> + <!-- XXX Commenting out the IFAR stuff <a name="ifar" href="javascript:toggle('ifar');"><h3>CBC IFAR</h3></a> @@ -132,18 +93,163 @@ function toggle(id) { --> <a name="rate" href="javascript:toggle('rate');"><h3>Submission Rates</h3></a> +<div id="rate" style="display: block;"> +{% if binned_counts %} + <div id="binned_counts_plot"></div> +{% else %} + No event count information. +{% endif %} +</div> +<br/> +<br/> + +<a href="{% url "cbc_report" %}"><h3>Dynamic CBC Report</h3></a> +<br/> +<br/> + +<a name="latency" href="javascript:toggle('latency');"><h3>Latency</h3></a> -<div id="rate" style="display: block;"> -{% if rate %} - <div id="CN" style="width: 750px; height: 450px;"></div> - <div id="leg1"></div> +<div id="latency" style="display:none;"> +{% if table %} + {{ table|safe }} {% else %} - No rate charts. + No Latency data. {% endif %} </div> + <br/> <br/> <a href="{% url "performance" %}"><h3>GraceDB 3-day performance summary</h3></a> +{% if binned_counts %} + +<!------ Script and stuff for the rates plot --> +<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script> +<script> + +var margin = {top: 20, right: 150, bottom: 70, left: 50}, + width = 950 - margin.left - margin.right, + height = 500 - margin.top - margin.bottom; + +var x = d3.time.scale() + .range([0, width]); + +var y = d3.scale.linear() + .range([height, 0]); + +var color = d3.scale.category10(); + +var xAxis = d3.svg.axis() + .scale(x) + .orient("bottom"); + +var yAxis = d3.svg.axis() + .scale(y) + .orient("left"); + +var line = d3.svg.line() + .x(function(d) { return x(d.date); }) + .y(function(d) { return y(d.nevents); }); + +var svg = d3.select("#binned_counts_plot").append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + +var pipelines = {}; + +var legend = []; + +var data_string = '{{ binned_counts|safe }}'; + +var data = JSON.parse(data_string); + +// These will be pipeline names +var active_keys = d3.keys(data[0]); + +color.domain(active_keys.filter(function(key) { return key !== "time" && key !== "delta_t"; })); + +// Like a data validation or typing step +data.forEach(function(d) { + d.date = d3.time.format.iso.parse(d.time); +}); + +// For each of the active keys, we'll create an array of arrays +pipelines = color.domain().map(function(name) { + return { + name: name, + active: true, + values: data.map(function(d) { + return {date: d.date, nevents: +d[name]}; + }) + }; +}); + +x.domain(d3.extent(data, function(d) { return d.date; })); + +y.domain([ + d3.min(pipelines, function(p) { return d3.min(p.values, function(v) { return v.nevents; }); }), + d3.max(pipelines, function(p) { return d3.max(p.values, function(v) { return v.nevents; }); }) +]); + +console.log("after x and y"); + +svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(xAxis) + .selectAll("text") + .attr("y", 6) + .attr("x", -12) + .attr("dy", ".5em") + .attr("transform", "rotate(315)") + .style("text-anchor", "end"); + +console.log("after x axis"); + +svg.append("g") + .attr("class", "y axis") + .call(yAxis) +.append("text") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text("N"); + +var pipeline = svg.selectAll(".pipeline") + .data(pipelines) +.enter().append("g") + .attr("class", "pipeline") + +pipeline.append("path") + .attr("class", "line") + .attr("id", function(d) { return d.name; }) + .attr("d", function(d) { return line(d.values); }) // whoa, line is a function.... + .style("stroke", function(d) { return color(d.name); }); + +legend = svg.selectAll(".legend") + .data(pipelines) +.enter().append("text") + .attr("x", width + margin.left ) + .attr("y", function(d, i) { return margin.top + i*20 + 10; }) + .attr("id", function(d) { return d.name + "legend"; }) + .attr("class", "legend") + .style("fill", function(d) { return color(d.name); }) + .on("click", function(d) { + var active = d.active ? false : true; + var newOpacity = active ? 1 : 0; + var newLabelOpacity = active ? 1 : 0.2; + d3.select("#" + d.name).transition().style("opacity", newOpacity); + d3.select("#" + d.name + "legend").transition().style("opacity", newLabelOpacity); + d.active = active; + }) + .text(function(d) { return d.name; }); + +</script> + +{% endif %} + {% endblock %} diff --git a/templates/gracedb/index.html b/templates/gracedb/index.html index 1059b9269..865dbeadd 100644 --- a/templates/gracedb/index.html +++ b/templates/gracedb/index.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {% block title %}Home{% endblock %} -{% block heading %}GraceDB Overview{% endblock %} +{% block heading %}{% endblock %} {% block pageid %}home{% endblock %} {% block jscript %} <style> @@ -26,9 +26,48 @@ {% endblock %} {% block content %} + +<!-- stuff for the recent events plot --> + +<style> +body { + font: 10px sans-serif; +} + +.axis text { + font: 16px sans-serif; + /*font-weight: bold; */ +} + +.axis line { + fill: none; + stroke: #000; + shape-rendering: crispEdges; +} + +.axis path { + display: none; +} + +.legend { + font-size: 16px; + font-weight: bold; + text-anchor: start; + cursor: pointer; +} + +</style> + +{% if recent_events != "" %} +<h2>Recent Events</h2> +<div id="recent_events_plot"></div> +<br/> +{% endif %} + <!-- I find myself needing some more space. --> -<br/> +<!-- <br/> --> +<h2>GraceDB Overview</h2> <div class="text"> @@ -67,4 +106,139 @@ follow-ups. </p> {% endif %} {% endif %} +<!-- lots of script for the recent events plot --> +{% if recent_events %} +<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script> +<script> + +var margin = {top: 0, right: 100, bottom: 70, left: 200}; +var width = 1000 - margin.left - margin.right; +var delta_y = 26; + +// Times look like "2015-10-15T13:08:31+00:00 +var parseDate = d3.time.format("%Y-%m-%dT%X+00:00").parse; + +var x = d3.time.scale() + .range([0, width]); + +var y = d3.scale.ordinal() +//var color = d3.scale.category10(); +var color = d3.scale.ordinal(); + +var xAxis = d3.svg.axis() + .scale(x) + .orient("bottom"); + +var svg = d3.select("#recent_events_plot").append("svg") + .attr("width", width + margin.left + margin.right) + + +var data_string = '{{ recent_events|safe }}'; + +var data = JSON.parse(data_string); + +// Convert the text in the created column to dates, use to set the x domain. +data.forEach(function(d) { + d.created = parseDate(d.created); +}); +x.domain(d3.extent(data, function(d) { return d.created; })); + +// I suspect there is a better and less annoying way of doing all this. +// But I don't really know what it is. We specify the ordering of the +// pipelines and the color for each pipeline that may be present. +var PIPELINE_ORDER = [ 'gstlal', 'MBTAOnline', 'CWB', 'LIB', 'Fermi', 'Swift', 'SNEWS', 'HardwareInjection']; + +// These are the first 8 colors of category 10 +var COLOR_HASH = {}; +COLOR_HASH['gstlal'] = '#1f77b4'; +COLOR_HASH['MBTAOnline'] = '#ff7f0e'; +COLOR_HASH['CWB'] = '#2ca02c'; +COLOR_HASH['LIB'] = '#d62728'; +COLOR_HASH['Fermi'] = '#9467bd'; +COLOR_HASH['Swift'] = '#8c564b'; +COLOR_HASH['SNEWS'] = '#e377c2'; +COLOR_HASH['HardwareInjection'] = '#7f7f7f'; + +// Pull out the unique pipeline names present in the data +var data_pipelines = d3.map(data, function(d) { return d.pipeline; }).keys(); + +// Select pipelines from our ordering according to those present in the data. +var pipelines = []; +PIPELINE_ORDER.forEach(function(d) { + if (data_pipelines.indexOf(d) >= 0) { + pipelines.push(d); + } +}); + +// Create the list of colors +var pipeline_colors = []; +pipelines.forEach(function(d) { + pipeline_colors.push(COLOR_HASH[d]); +}); + +// Now set the domain and range of our color map. +color.domain(pipelines); +color.range(pipeline_colors); + +// Now that you know how many pipelines there are, you can determine the height. +height = pipelines.length * delta_y; + +var container = svg + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + +// Calculate the height coordinates for the different pipelines. +var y_range_values = []; +pipelines.forEach(function(d, i) { + y_range_values.push(height - i*delta_y); +}); +y.domain(pipelines).range(y_range_values); + +// Create the yAxis here, since you need a valid scale. +var yAxis = d3.svg.axis() + .scale(y) + .orient("left"); + +var xAxis_yloc = height + delta_y/2; +container.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + xAxis_yloc + ")") + .call(xAxis) + .selectAll("text") + .attr("y", 6) + .attr("x", -12) + .attr("dy", "0.5em") + .attr("transform", "rotate(315)") + .style("text-anchor", "end"); + +var yAxis_xloc = - delta_y/2; +container.append("g") + .attr("class", "y axis") + .attr("transform", "translate(" + yAxis_xloc + ",0)") + .call(yAxis) + +container.selectAll(".y.axis text") + .attr("fill", function(d) { return color(d); }) + +var line_length = 12; + +var get_url = function(d) { + return {% url 'home-events' %} + d.graceid; +} + +container.append("g").selectAll("line") + .data(data) +.enter().append("svg:a") + .attr("xlink:href", function(d) { return get_url(d); }) +.append("line") + .attr("x1", function(d) { return x(d.created); }) + .attr("x2", function(d) { return x(d.created); }) + .attr("y1", function(d) { return y(d.pipeline) - line_length/2; }) + .attr("y2", function(d) { return y(d.pipeline) + line_length/2; }) + .attr("stroke-width", 5) + .attr("stroke", function(d) { return color(d.pipeline); }) + +</script> +{% endif %} {% endblock %} -- GitLab