Merge branch 'summary' into 'master'

Adding lockloss summary plots to webpages

Creates a new webpage with link (in green) located below the github links. Adds a module (locklost/summary.py) that collects data from EVENT_ROOT and creates/saves histogram plots based on this data. These plots are saved to a directory (events/summary_plots), which is further subdivided into directories for plots derived from locklosses in the prior week, month, and over the entire run (ex: events/summary_plots/Month).

The ability to run the summary module from the command line is added to locklost/__main__.py, and the summary plot template is added as locklost/web/templates/summary.tpl.

See merge request !93

locklost/__main__.py

@@ -11,6 +11,7 @@ from . import analyze
 from . import online
 from . import event
 from . import segments
+from . import summary
 from . import plots
 
 ##########
@@ -56,6 +57,9 @@ p = gen_subparser('online', online.main)
 online._parser_add_arguments(p)
 
 
+p = gen_subparser('summary', summary.main)
+
+
 def list_events(args):
     """List all events"""
     for e in event.find_events():

locklost/config.py

@@ -48,6 +48,8 @@ if IFO == 'H1':
 elif IFO == 'L1':
     CDS_EVENT_ROOT = 'https://llocds.ligo-la.caltech.edu/data/lockloss/events'
 
+O3_GPS_START = 1238112018
+
 SEARCH_STRIDE = 10000
 
 DATA_ACCESS = os.getenv('DATA_ACCESS', 'gwpy')

locklost/summary.py

+import os
+import numpy as np
+import matplotlib.pyplot as plt
+from lal.gpstime import gps_to_utc, gps_time_now
+
+from . import config
+from .event import find_events, LocklossEvent
+from . import plotutils
+
+EPOCHS = {
+    'run': config.O3_GPS_START,
+    'month': gps_time_now() - 30*24*3600,
+    'week': gps_time_now() - 7*24*3600,
+}
+
+def grab_data(gps):
+    """ Returns relevant lockloss summary data within three time ranges.
+
+    Looks through O3 lockloss data and returns counts for the locklosses
+    from each state, the duration in the final state, and the first saturated
+    suspension channel for the three most common locklosses. Does this for the
+    run, the last 30 days, and the last week.
+    """
+
+    shifts = {
+        'owl': {
+            'time': np.arange(0,8),
+            'counts': 0,
+        },
+        'day': {
+            'time': np.arange(8,16),
+            'counts': 0,
+        },
+        'eve': {
+            'time': np.arange(16,24),
+            'counts': 0,
+        },
+    }
+    transitions = []
+    observe_durations = []
+    saturations = {
+        'als': [],
+        'darm': [],
+        'observe': [],
+    }
+    five_sats = []
+    shift_losses = {
+        'owl': 0,
+        'day': 0,
+        'eve': 0,
+    }
+    tags = {
+        'MAINTENANCE': 0,
+        'ADS_EXCURSION': 0,
+        'BOARD_SAT': 0,
+        'BRS_GLITCH': 0,
+        'SEISMIC': 0,
+        'WINDY': 0,
+        'Unknown': 0
+    }
+
+    event_count = 0
+    for event in find_events(after=gps, state='0-600'):
+        transitions.append(event.transition_index[0])
+        #check event/append to relevant lists
+        observe_durations = check_durations(event, observe_durations)
+        saturations, five_sats = check_saturations(event, saturations, five_sats)
+        shifts = check_shift(event, shifts)
+        tags = check_tags(event, tags)
+        event_count += 1
+
+    print ("Events analyzed: %i" % (event_count))
+
+    return transitions, observe_durations, saturations, five_sats, shifts, tags
+
+
+def check_tags(event, tags):
+    ref_sum = sum(tags.values())
+    for tag_key in tags:
+        if event.has_tag(tag_key):
+            tags[tag_key] += 1
+            break
+    if sum(tags.values()) == ref_sum:
+        tags['Unknown'] += 1
+
+    return tags
+
+
+def check_shift(event, shifts):
+    gps = gps_to_utc(event.gps)
+    if event.has_tag('Observe'):
+        for key in shifts.keys():
+            if gps.hour in shifts[key]['time']:
+                shifts[key]['counts'] += 1
+                break
+
+    return shifts
+
+
+def check_durations(event, durations):
+    """ Check if lockloss was from Observe and log lock duration. """
+    if event.has_tag('OBSERVE'):
+        previous = event.previous_state
+        if previous:
+            durations.append((previous['end']-previous['start'])/3600)
+
+    return durations
+
+
+def check_saturations(event, saturations, five_sats):
+    sat_path = event.path('saturations.csv')
+    if os.path.exists(sat_path):
+        sats = get_five_sats(sat_path)
+        five_sats.append(sats)
+    sat_conditions = {
+        'observe': event.has_tag('OBSERVE'),
+        'darm': event.transition_index[0] == 101,
+        'als': event.transition_index[0] == 15,
+    }
+    for key, condition in sat_conditions.items():
+        if condition:
+            if os.path.exists(sat_path):
+                saturations[key].append(sats[0])
+            else:
+                saturations[key].append('No saturations')
+
+    return saturations, five_sats
+
+
+def get_five_sats(sat_path):
+    """ Returns shortened names of first (up to) five saturating suspension
+    channels. """
+    all_sats = np.genfromtxt(
+        sat_path,
+        delimiter=' ',
+        dtype=str,
+        usecols=0,
+    )
+    five_sats = np.array([])
+    all_sats = np.array(all_sats, ndmin=1)
+    sat_lim = min([all_sats.size, 5])
+    for sat in all_sats[:sat_lim]:
+        # create shortened channel name (excluding IFO, quadrant, characters)
+        sat_123 = sat.split('-')
+        sat1 = sat_123[0]
+        sat2 = sat_123[1].split('_')[0]
+        sat3 = sat_123[1].split('_')[1]
+        channel_shorthand = '%s %s %s' % (sat1, sat2, sat3)
+        # Make sure the degenerate channels don't get added
+        if not channel_shorthand in five_sats:
+            five_sats = np.append(five_sats, channel_shorthand)
+
+    return five_sats
+
+
+def plot_summary(epoch, gps):
+    """ Plots lockloss summary data and saves to example_plots.
+
+    Plots histograms for lockloss state transitions, time lengths in Observing,
+    and first saturating suspension channel for the three most common lockloss
+    states. Saves these to the example_plots directory. """
+
+    summary_path = os.path.join(config.WEB_ROOT, 'summary_plots')
+    if not os.path.exists(summary_path):
+        os.mkdir(summary_path)
+    epoch_path = os.path.join(summary_path, epoch)
+    if not os.path.exists(epoch_path):
+        os.mkdir(epoch_path)
+
+    transitions, observe_durations, saturations, five_sats, shifts, tags = grab_data(gps)
+    state_occurence = np.array([])
+    sat_occurence = {'als': [], 'darm': [], 'observe': [],}
+    sat_names = {'observe': 'Observe', 'darm':'ACQUIRE_DRM1_1F', 'als':'LOCKING_ALS'}
+    sat_positions = {'als': 0, 'darm': 0, 'observe': 0,}
+
+    # Collecting str values for the histogram x-axis and counting number
+    # of values in each 'bin'
+    # Transition state bin structuring
+    states = set(transitions)
+    for state in states:
+        state_occurence = np.append(state_occurence, transitions.count(state))
+    state_bar = sorted(zip(states, state_occurence)) # sorting states numerically
+    states, state_occurence = zip(*state_bar)
+    state_position = np.arange(len(states)) # for histogram later
+    str_states = [str(i) for i in states] #for histogram later
+
+    # Saturating suspension channel bin structuring
+    sat_sets = {key:set(val) for key, val in saturations.items()}
+    for key, sat_set in sat_sets.items():
+        for channel in sat_set:
+            sat_occurence[key] = np.append(sat_occurence[key], saturations[key].count(channel))
+        sat_bar = sorted(zip(sat_set, sat_occurence[key]))
+        sat_sets[key], sat_occurence[key] = zip(*sat_bar)
+        sat_positions[key] = np.arange(len(sat_set))
+
+    plotutils.set_rcparams()
+
+    # Transition state plot
+    fig, ax = plt.subplots(1, figsize=(22,16))
+    ax.bar(
+        state_position,
+        state_occurence,
+        align='center',
+    )
+    ax.set_xlabel('State from which lockloss has occurred', labelpad=10)
+    ax.set_ylabel('Number of locklosses')
+    ax.set_title('O3 lockloss occurences by final state: %s' % (epoch))
+    ax.set_xticks(state_position)
+    ax.tick_params(axis='x', which='major', labelsize=18)
+    ax.set_xticklabels(str_states, rotation = 45, ha = 'right')
+    ax.set_xlim([-1,state_position.size])
+    plt.gcf().text(0.02, 0.02, "Created: {}".format(gps_time_now()), fontsize=16)
+    fig.tight_layout()
+
+    plot_name = 'Lockloss_states'
+    outpath_plot = os.path.join(epoch_path, plot_name)
+    fig.savefig(outpath_plot, bbox_inches='tight')
+    plt.close()
+
+    # Lock duration plot
+    fig, ax = plt.subplots(1, figsize=(22,16))
+    if epoch == 'run':
+        bin_num = 30
+    if epoch == 'month':
+        bin_num = 10
+    if epoch == 'week':
+        bin_num = 5
+    fig, ax = plt.subplots(1, figsize=(22,16))
+    ax.hist(
+        observe_durations,
+        bins=bin_num,
+        align='mid',
+    )
+    ax.set_xlabel('Lock duration [hours]', labelpad=10)
+    ax.set_ylabel('Number of locks')
+    ax.set_title('Observe lock durations: %s' % (epoch))
+    plt.gcf().text(0.02, 0.02, "Created: {}".format(gps_time_now()), fontsize=16)
+    fig.tight_layout()
+
+    plot_name = 'Lock_durations'
+    outpath_plot = os.path.join(epoch_path, plot_name)
+    fig.savefig(outpath_plot, bbox_inches='tight')
+    plt.close()
+
+    # Saturating suspension channel plot
+    for key, occurence in sat_occurence.items():
+        fig, ax = plt.subplots(1, figsize=(22,16))
+        ax.bar(
+            sat_positions[key],
+            occurence,
+            align='center',
+        )
+        ax.set_xlabel('First suspension to saturate before lockloss', labelpad=10)
+        ax.set_ylabel('Number of locklosses')
+        ax.set_title('%s locklosses by saturating suspension: %s' % (sat_names[key], epoch))
+        ax.set_xticks(sat_positions[key])
+        ax.set_xticklabels(sat_sets[key], rotation = 45, ha = 'right')
+        ax.set_xlim([-1,sat_positions[key].size])
+        plt.gcf().text(0.02, 0.02, "Created: {}".format(gps_time_now()), fontsize=16)
+        fig.tight_layout()
+
+        plot_name = '%s_lockloss_saturations' % (sat_names[key])
+        outpath_plot = os.path.join(epoch_path, plot_name)
+        fig.savefig(outpath_plot, bbox_inches='tight')
+        plt.close()
+
+    # Lockloss shift plot
+    counts = [x['counts'] for x in shifts.values()]
+    shifts = shifts.keys()
+    fig, ax = plt.subplots(1, figsize=(22,16))
+    shift_x = np.array([0,1,2])
+    ax.bar(
+        shift_x,
+        counts,
+        align='center',
+    )
+    ax.set_xlabel('Operating shift', labelpad=10)
+    ax.set_ylabel('Number of locklosses')
+    ax.set_title('Number of locklosses per shift: %s' % (epoch))
+    ax.set_xticks(shift_x)
+    ax.set_xticklabels(shifts, rotation=45, ha='right')
+    ax.set_xlim([-1, shift_x.size])
+    plt.gcf().text(0.02, 0.02, "Created: {}".format(gps_time_now()), fontsize=16)
+    fig.tight_layout()
+
+    plot_name = 'Lockloss_by_shift'
+    outpath_plot = os.path.join(epoch_path, plot_name)
+    fig.savefig(outpath_plot, bbox_inches='tight')
+    plt.close()
+
+    # Associated tag plot
+    temp_dict = dict(tags)
+    for key, value in tags.items():
+        if value == 0:
+            del temp_dict[key]
+    tags = temp_dict
+    ts = tags.keys()
+    counts = tags.values()
+    fig, ax = plt.subplots(1, figsize=(22,16))
+    shift_x = np.arange(0, len(tags))
+    ax.bar(
+        shift_x,
+        counts,
+        align='center',
+    )
+    ax.set_xlabel('Lockloss tags', labelpad=10)
+    ax.set_ylabel('Number of locklosses')
+    ax.set_title('Number of locklosses with given tag: %s' % (epoch))
+    ax.set_xticks(shift_x)
+    ax.set_xticklabels(ts, rotation=45, ha='right')
+    ax.set_xlim([-1, shift_x.size])
+    plt.gcf().text(0.02, 0.02, "Created: {}".format(gps_time_now()), fontsize=16)
+    fig.tight_layout()
+
+    plot_name = 'Lockloss_by_tag'
+    outpath_plot = os.path.join(epoch_path, plot_name)
+    fig.savefig(outpath_plot, bbox_inches='tight')
+    plt.close()
+
+######################################################
+
+def main(args=None):
+    """
+    Create histogram of lockloss states in O3.
+    """
+    for epoch, gps in EPOCHS.items():
+        print ('Looking at locklosses over the %s' % (epoch))
+        plot_summary(epoch, gps)
+
+if __name__ == '__main__':
+    main()

locklost/web/__main__.py

@@ -140,6 +140,15 @@ def index(tag='all'):
             query=query,
         )
 
+@app.route("/summary")
+def summary_route():
+    return bottle.template(
+        'summary.tpl',
+        IFO=config.IFO,
+        web_script=WEB_SCRIPT,
+        online_status=online_status(),
+        is_home=False,
+    )
 
 @app.route("/event/<gps>")
 def event_route(gps):

locklost/web/templates/base.tpl

@@ -21,6 +21,8 @@
 <a href="https://git.ligo.org/jameson.rollins/locklost">git repo </a>
 <a href="https://git.ligo.org/jameson.rollins/locklost/blob/master/README.md">docs </a>
 <a href="https://git.ligo.org/jameson.rollins/locklost/issues">issue tracker </a>
+<br>
+<a href="{{web_script}}/summary" style="color:green">Summary Plots</a>
 <hr />
 
 % if is_home:

locklost/web/templates/summary.tpl

+% rebase('base.tpl', IFO=IFO, web_script=web_script, online_status=online_status)
+
+% import os
+
+% from locklost import config
+
+<!-- Summary title -->
+<h4>Summary Plots</h4>
+
+<!-- Plot tabs -->
+<button class="tablink" onclick="openPage('run', this, 'orange')">Run</button>
+<button class="tablink" onclick="openPage('month', this, 'green')">Month</button>
+<button class="tablink" onclick="openPage('week', this, 'blue')" id="defaultOpen">Week</button>
+
+% for id in ['run', 'month', 'week']:
+%   plot_dir = os.path.join(config.EVENT_ROOT, 'summary_plots', id)
+%   plotnames = os.listdir(plot_dir)
+    <div id={{id}} class="tabcontent">
+%   for plotname in plotnames:
+%     plot_url = os.path.join(config.WEB_ROOT, 'events/summary_plots', id, plotname)
+      <hr />
+      <div class="container">
+      <div class="row">
+      % include('plots.tpl', plots=[plot_url], size=8)
+      </div>
+      </div>
+%   end
+    </div>
+% end
+
+<script>
+function openPage(pageName, elmnt, color) {
+  // Hide all elements with class="tabcontent" by default */
+  var i, tabcontent, tablinks;
+  tabcontent = document.getElementsByClassName("tabcontent");
+  for (i = 0; i < tabcontent.length; i++) {
+    tabcontent[i].style.display = "none";
+  }
+
+  // Remove the background color of all tablinks/buttons
+  tablinks = document.getElementsByClassName("tablink");
+  for (i = 0; i < tablinks.length; i++) {
+    tablinks[i].style.backgroundColor = "";
+  }
+
+  // Show the specific tab content
+  document.getElementById(pageName).style.display = "block";
+
+  // Add the specific color to the button used to open the tab content
+  elmnt.style.backgroundColor = color;
+}
+
+// Get the element with id="defaultOpen" and click on it
+document.getElementById("defaultOpen").click();
+</script>

test/run

@@ -26,7 +26,7 @@ case $IFO in
         exit 1
         ;;
 esac
-        
+
 export LOG_LEVEL=DEBUG
 export LOCKLOST_EVENT_ROOT=$(mktemp -d)
 export LOCKLOST_WEB_ROOT=https://ldas-jobs.ligo.caltech.edu/~lockloss
@@ -51,6 +51,8 @@ test $nevents -eq $NEVENTS
 
 $PYTHON -m locklost analyze $EVENT
 
+$PYTHON -m locklost summary
+
 $PYTHON -m locklost show $EVENT
 
 $PYTHON -m locklost plot-history $LOCKLOST_EVENT_ROOT/history.svg