Merge branch 'api_overhaul' into 'master'

API overhaul

See merge request !27

.gitlab-ci.yml

@@ -122,6 +122,7 @@ build:debian:stretch:
     - ${PYTHON} -m pip install
           "pytest==3.0.*"
           "pytest-cov"
+          "kafka-python"
   script:
     - export SCALDRC_PATH=etc/example_config.yml
     - ${PYTHON} -m pytest -v --doctest-modules --cov=ligo/scald --ignore build/ --ignore dist/ --ignore doc/ --ignore setup.py
@@ -136,6 +137,7 @@ test:el7-python2:
           python-pytest
           python-pytest-cov
     - yum ${YUM_OPTS} --nogpgcheck localinstall dist/el7/python2-ligo-scald-*.rpm
+    - pip install kafka-python
   cache:
     key: "${CI_JOB_NAME}"
     paths:
@@ -159,6 +161,7 @@ test:debian-python2:stretch:
           python-ligo-common
           python-urllib3
     - dpkg -i dist/python-ligo-scald_*_all.deb
+    - pip install kafka-python
   cache:
     key: "${CI_JOB_NAME}"
     paths:
@@ -184,6 +187,7 @@ test:debian-python3:stretch:
           python3-ligo-common
           python3-urllib3
     - dpkg -i dist/python3-ligo-scald_*_all.deb
+    - pip3 install kafka-python
   cache:
     key: "${CI_JOB_NAME}"
     paths:

etc/example_config.yml

-# configuration file for monitoring
-dashboard:
+### configuration file for monitoring
+
+#--------------------------------------------------------------
+### navbar configuration
+
+navbar:
   title: scald
+  pages:
+    - index
+    - dash2
+
   tabs:
     - name: Docs
       url: https://docs.ligo.org/gstlal-visualisation/ligo-scald
     - name: Git
       url: https://git.ligo.org/gstlal-visualisation/ligo-scald
-  gps: -1
-  duration: 600
 
-plots:
-  - title: Sample SNR Timeseries by IFO
-    type: TimeSeries
-    value: 'checked'
+
+#--------------------------------------------------------------
+### page configuration
+
+pages:
+  index:
+    title: Example 1
+    type: grid
+    mode: online
+    lookback: 300
+    delay: 0
+    plots:
+      - plot: snr_by_ifo
+        grid: {x: 0, y: 0, h: 6, w: 6}
+        visible: true
+      - plot: heartbeat
+        grid: {x: 0, y: 6, h: 6, w: 6}
+        visible: true
+      - plot: segment
+        grid: {x: 6, y: 0, h: 6, w: 6}
+        visible: true
+      - plot: snr_heatmap
+        grid: {x: 6, y: 6, h: 6, w: 6}
+        visible: true
+
+  dash2:
+    title: Example 2
+    type: grid
+    mode: online
+    lookback: 300
+    delay: 0
+    plots:
+      - plot: binned
+        grid: {x: 0, y: 0, h: 6, w: 6}
+        visible: true
+      - plot: multischema
+        grid: {x: 0, y: 6, h: 6, w: 6}
+        visible: true
+      - plot: custom_opts
+        grid: {x: 0, y: 12, h: 6, w: 6}
+        visible: true
+      - plot: twoaxis
+        grid: {x: 6, y: 0, h: 6, w: 6}
+        visible: true
+      - plot: ifar
+        grid: {x: 6, y: 6, h: 12, w: 6}
+        visible: true
+
+
+#--------------------------------------------------------------
+### schema configuration
+
+schemas:
+  snr:
     measurement: inspiral
-    grid:
-      x: 0
-      y: 0
-      w: 4
-      h: 2
-    schema:
-      column: snr
-      tag: ifo
-      tag_key: ifo
-      ifo:
-        - H1
-        - L1
-      aggregate: median
-    layout:
-      yaxis:
-        type: log
-        title:
-          text: SNR
-      margin:
-        l: 40
-        r: 20
-        t: 10
-        b: 20
-
-  - title: Sample Two Axis Chart
-    type: MultiAxis
-    value: 'checked'
+    column: snr
+    tag: ifo
+    tag_key: ifo
+    ifo:
+      - H1
+      - L1
+    aggregate: median
+    default: 0
+
+  H1_snr:
     measurement: inspiral
-    grid:
-      x: 4
-      y: 0
-      w: 4
-      h: 2
-    schema:
-      axis1:
-        schema:
-          column: snr
-          tag: ifo
-          tag_key: ifo
-          ifo:
-            - H1
-          aggregate: median
-      axis2:
-        schema:
-          column: snr
-          tag: ifo
-          tag_key: ifo
-          ifo:
-            - L1
-          aggregate: median
-    layout:
-      yaxis:
-        title: 'y-axis 1, H1'
-      yaxis2:
-        title: 'y-axis 2, L1'
+    column: snr
+    tag: ifo
+    tag_key: ifo
+    ifo:
+      - H1
+    aggregate: median
 
-  - title: Sample Multi Schema Chart
-    type: MultiSchema
-    value: 'checked'
+  L1_snr:
+    measurement: inspiral
+    column: snr
+    tag: ifo
+    tag_key: ifo
+    ifo:
+      - L1
+    aggregate: median
+
+  H1L1_snr:
     measurement: inspiral
-    grid:
-      x: 8
-      y: 0
-      w: 4
-      h: 2
-    schema:
-      schema1:
-        name: H1
+    axis1:
+      schema:
         column: snr
         tag: ifo
         tag_key: ifo
         ifo:
           - H1
         aggregate: median
-      schema2:
-        name: L1
+    axis2:
+      schema:
         column: snr
         tag: ifo
         tag_key: ifo
         ifo:
           - L1
         aggregate: median
-      schema3:
-        name: V1
-        column: snr
-        tag: ifo
-        tag_key: ifo
-        ifo:
-          - V1
-        aggregate: median
 
-  - title: Sample Customizable DataOptions
-    type: TimeSeries
-    value: 'checked'
+  H1L1V1_snr:
     measurement: inspiral
-    grid:
-      x: 0
-      y: 2
-      w: 4
-      h: 2
-    schema:
+    schema1:
+      name: H1
       column: snr
       tag: ifo
       tag_key: ifo
       ifo:
         - H1
+      aggregate: median
+    schema2:
+      name: L1
+      column: snr
+      tag: ifo
+      tag_key: ifo
+      ifo:
         - L1
       aggregate: median
+    schema3:
+      name: V1
+      column: snr
+      tag: ifo
+      tag_key: ifo
+      ifo:
+        - V1
+      aggregate: median
+
+  snr_by_job:
+    measurement: inspiral
+    column: snr
+    tag: job
+    tag_key: job
+    aggregate: median
+
+  heartbeat:
+    backend: custom
+    measurement: latency
+    column: latency
+    tag: job
+    tag_key: job
+    aggregate: median
+    default: 1e4
+    transform: latency
+
+  latency:
+    backend: custom
+    measurement: latency
+    column: latency
+    tag: job
+    tag_key: job
+    aggregate: median
+    default: 1e4
+    transform: none
+
+  segment:
+    segment:
+      measurement: segment
+      column: seg
+      tag: job
+      tag_key: job
+      aggregate: median
+    timeseries:
+      measurement: segment
+      name: "TimeSeries"
+      column: snr
+      tag: ifo
+      tag_key: ifo
+      ifo:
+        - "TimeSeries"
+      aggregate: median
+
+  ifar:
+    measurement: inspiral
+    column: far
+    far: 1e-3
+    lookback: 10000
+
+  binned:
+    measurement: inspiral
+    tbin: 10
+    dt: 10
+    column: snr
+    tag: ifo
+    tag_key: ifo
+    ifo:
+      - H1
+    aggregate: median
+
+
+#--------------------------------------------------------------
+### plot configuration
+
+plots:
+  snr_by_ifo:
+    title: SNR Timeseries by IFO
+    type: TimeSeries
+    schema: snr
+    layout:
+      yaxis:
+        type: log
+        title: {text: SNR}
+
+  twoaxis:
+    title: Two Axis, H1,L1 SNR Timeseries
+    type: MultiAxis
+    schema: H1L1_snr
+    layout:
+      yaxis: {title: 'y-axis 1, H1'}
+      yaxis2: {title: 'y-axis 2, L1'}
+
+  multischema:
+    title: Multi Schema, SNR Timeseries by IFO
+    type: MultiSchema
+    schema: H1L1V1_snr
+
+  custom_opts:
+    title: Customizable DataOptions
+    type: TimeSeries
+    schema: snr
     layout:
       yaxis:
         type: log
-        title:
-          text: SNR
-      margin:
-        l: 40
-        r: 20
-        t: 10
-        b: 20
+        title: {text: SNR}
+      margin: {l: 40, r: 20, t: 10, b: 20}
     data_options:
       H1:
         mode: 'markers'
@@ -147,156 +255,115 @@ plots:
           dash: 'dashdot'
       colorscale: 'Viridis'
 
-  - title: Sample SNR Heatmap by Job
+  snr_heatmap:
+    title: Sample SNR Heatmap by Job
     type: TimeHeatMap
-    value: 'checked'
-    measurement: inspiral
-    schema:
-      column: snr
-      tag: job
-      tag_key: job
-      aggregate: median
-    grid:
-      x: 4
-      y: 2
-      w: 4
-      h: 2
+    schema: snr_by_job
     layout:
-      yaxis:
-        title:
-          text: Job ID
-      margin:
-        l: 60
-        r: 20
-        t: 10
-        b: 20
-    data_options:
-      colorscale: "Viridis"
+      yaxis: {title: {text: Job ID}}
 
-  - title: Sample TimeSeries and Segment Plot
+  segment:
+    title: TimeSeries + Segment Plot
     type: TimeSegment
-    value: 'checked'
-    measurement: inspiral
-    grid:
-     x: 4
-     y: 2
-     w: 4
-     h: 2
-    schema:
-      segment:
-        column: segment
-        tag: job
-        tag_key: job
-        aggregate: median
-      timeseries:
-        name: "TimeSeries"
-        column: snr
-        tag: ifo
-        tag_key: ifo
-        ifo:
-          - "TimeSeries"
-        aggregate: median
+    schema: segment
     layout:
-      yaxis:
-        title:
-          text: "Time Series Axis"
+      yaxis: {title: {text: 'TimeSeries Axis'}}
     data_options:
       segment:
         colorscale: [['0.0', 'rgb(212,22,22)'],['1.0', 'rgb(48,133,5)']]
       timeseries:
-        # type: 'scatter' # UNNEEDED
         colorscale: 'Viridis'
 
-  - title: Sample Latest Heartbeat by Job
+  heartbeat:
+    title: Latest Heartbeat by Job
     type: Bar
-    value: 'checked'
-    measurement: inspiral
-    schema:
-      backend: custom
-      column: latency
-      tag: job
-      tag_key: ifo
-      aggregate: median
-    grid:
-      x: 8
-      y: 2
-      w: 4
-      h: 2
+    schema: heartbeat
     layout:
-      xaxis:
-        title:
-          text: Job ID
-      yaxis:
-        title:
-          text: Latency [s]
-      margin:
-        l: 40
-        r: 20
-        t: 10
-        b: 40
-
-  - title: Sample IFAR Plot
+      xaxis: {title: {text: 'Job ID'}}
+      yaxis: {title: {text: 'Latency [s]'}}
+
+  ifar:
+    title: IFAR Plot + Table
     type: IFAR
-    value: 'checked'
-    measurement: inspiral
-    schema:
-      column: far
-      far: 1e-3
-      lookback: 10000
+    schema: ifar
     layout:
       xaxis:
-        title:
-          text: Inverse False-Alarm Rate [s]
+        title: {text: 'Inverse False-Alarm Rate [s]'}
         type: log
       yaxis:
-        title:
-          text: Number of Events
+        title: {text: 'Number of Events'}
         type: log
-      margin:
-        l: 40
-        r: 20
-        t: 10
-        b: 40
-    grid:
-      x: 0
-      y: 8
-      w: 12
-      h: 8
-
-  - title: Sample SNR Bins Time Histogram
+
+  binned:
+    title: SNR Binned Time Histogram
     type: BinnedTimeHistogram
-    value: 'checked'
-    measurement: inspiral
-    grid:
-      x: 0
-      y: 0
-      w: 4
-      h: 2
-    schema:
-      tbin: 10
-      dt: 10
-      column: snr
-      tag: ifo
-      tag_key: ifo
-      ifo:
-        - H1
-      aggregate: median
+    schema: binned
     layout:
-      yaxis:
-        title:
-          text: SNR
-      margin:
-        l: 40
-        r: 20
-        t: 10
-        b: 20
-
-measurements:
-  inspiral:
-    tag: job
-    num_tags: 50
-    default: 1e4
-    transform: none
+      yaxis: {title: {text: SNR}}
+
+
+#--------------------------------------------------------------
+### default plot settings
+
+plotly:
+  default:
+    layout:
+      font:
+        family: "'Nunito', sans-serif"
+        size: 12
+        color: '#666'
+      plot_bgcolor: 'rgba(0,0,0,0)'
+      paper_bgcolor: 'rgba(0,0,0,0)'
+      showlegend: 'true'
+    data_options:
+      marker:
+        color: 'rgb(255, 0, 0)'
+      colorscale: "Viridis"
+      H1:
+        mode: 'lines'
+        line:
+          width: 0.5
+          color: 'rgb(238,0,0)'
+      L1:
+        mode: 'lines'
+        line:
+          width: 0.5
+          color: 'rgb(75,166,255)'
+      V1:
+        mode: 'lines'
+        line:
+          width: 0.5
+          color: 'rgb(155,89,182)'
+
+  TimeSeries:
+    layout:
+      margin: {l: 50, r: 0, t: 0, b: 20}
+
+  TimeHeatMap:
+    layout:
+      margin: {l: 50, r: 0, t: 10, b: 25}
+
+  IFAR:
+    layout:
+      margin: {l: 60, r: 0, t: 10, b: 50}
+
+  Bar:
+    layout:
+      margin: {l: 50, r: 0, t: 10, b: 40}
+
+
+#--------------------------------------------------------------
+### nagios configuration
+
+nagios:
+  heartbeat:
+    lookback: 180
+    schema: heartbeat
+    alert_type: heartbeat
+
+
+#--------------------------------------------------------------
+### data backend configuration
 
 backends:
   default:
@@ -304,6 +371,7 @@ backends:
     db: mydb
     hostname: localhost
     port: 8086
+