From 3108693bfd63ebcb649e1db972ce35636e21d241 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Wed, 3 Apr 2019 09:13:33 -0500
Subject: [PATCH] Add web view for enabling/disabling pipelines
A web view has been created which allows EM advocates to disable
or enable pipeline submissions. This is to prevent misbehaving
pipelines from submitting bad information. The mechanism works by
preventing events from being submitted to a given pipeline, not
by revoking certificates or removing user account permissions.
---
gracedb/api/v1/events/views.py | 11 +-
gracedb/core/tests/utils.py | 4 +
.../0033_pipelinelog_and_pipeline_enabled.py | 41 +++++++
gracedb/events/models.py | 20 +++
gracedb/events/urls.py | 8 ++
gracedb/events/views.py | 114 +++++++++++++++++-
gracedb/ligoauth/context_processors.py | 5 +-
.../0023_add_manage_pipeline_permissions.py | 66 ++++++++++
.../templates/gracedb/manage_pipelines.html | 104 ++++++++++++++++
gracedb/templates/navbar_frag.html | 3 +
10 files changed, 370 insertions(+), 6 deletions(-)
create mode 100644 gracedb/events/migrations/0033_pipelinelog_and_pipeline_enabled.py
create mode 100644 gracedb/migrations/auth/0023_add_manage_pipeline_permissions.py
create mode 100644 gracedb/templates/gracedb/manage_pipelines.html
diff --git a/gracedb/api/v1/events/views.py b/gracedb/api/v1/events/views.py
index e21c174ef..e47d6e586 100644
--- a/gracedb/api/v1/events/views.py
+++ b/gracedb/api/v1/events/views.py
@@ -13,7 +13,7 @@ from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.db import IntegrityError
from django.http import HttpResponse, HttpResponseForbidden, \
- HttpResponseNotFound, HttpResponseServerError
+ HttpResponseNotFound, HttpResponseServerError, HttpResponseBadRequest
from django.http.request import QueryDict
from django.utils.functional import wraps
@@ -469,6 +469,15 @@ class EventList(InheritPermissionsAPIView):
if not user_has_perm(request.user, "populate", pipeline):
return HttpResponseForbidden("You don't have permission on this pipeline.")
+ # Get search since we won't block MDC event submissions even if the
+ # pipeline is disabled
+ search_name = request.data.get('search', None)
+ if not pipeline.enabled and search_name != 'MDC':
+ err_msg = ('The {0} pipeline has been temporarily disabled by '
+ 'an EM advocate due to suspected misbehavior.').format(
+ pipeline.name)
+ return HttpResponseBadRequest(err_msg)
+
# The following looks a bit funny but it is actually necessary. The
# django form expects a dict containing the POST data as the first
# arg, and a dict containing the FILE data as the second. In the
diff --git a/gracedb/core/tests/utils.py b/gracedb/core/tests/utils.py
index dddbaff35..3705cb397 100644
--- a/gracedb/core/tests/utils.py
+++ b/gracedb/core/tests/utils.py
@@ -253,6 +253,10 @@ class SignoffGroupsAndUsersSetup(TestCase):
content_type__app_label='superevents',
codename='do_adv_signoff')
g.permissions.add(*p)
+ p = Permission.objects.filter(
+ content_type__app_label='events',
+ codename='manage_pipeline')
+ g.permissions.add(*p)
# Also add user to internal group
internal_group.user_set.add(user)
diff --git a/gracedb/events/migrations/0033_pipelinelog_and_pipeline_enabled.py b/gracedb/events/migrations/0033_pipelinelog_and_pipeline_enabled.py
new file mode 100644
index 000000000..4ce9a33f5
--- /dev/null
+++ b/gracedb/events/migrations/0033_pipelinelog_and_pipeline_enabled.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-04-02 20:19
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('events', '0032_create_imbh_search'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='PipelineLog',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('action', models.CharField(choices=[(b'D', b'disable'), (b'E', b'enable')], max_length=10)),
+ ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+ ],
+ ),
+ migrations.AlterModelOptions(
+ name='pipeline',
+ options={'permissions': (('manage_pipeline', 'Can enable or disable pipeline'),)},
+ ),
+ migrations.AddField(
+ model_name='pipeline',
+ name='enabled',
+ field=models.BooleanField(default=True),
+ ),
+ migrations.AddField(
+ model_name='pipelinelog',
+ name='pipeline',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='events.Pipeline'),
+ ),
+ ]
diff --git a/gracedb/events/models.py b/gracedb/events/models.py
index b2dee94d3..f38f86e0e 100644
--- a/gracedb/events/models.py
+++ b/gracedb/events/models.py
@@ -69,10 +69,30 @@ class Group(models.Model):
class Pipeline(models.Model):
name = models.CharField(max_length=100)
+ enabled = models.BooleanField(default=True)
# XXX Need any additional fields? Like a librarian email? Or perhaps even fk?
+ class Meta:
+ permissions = (
+ ('manage_pipeline', 'Can enable or disable pipeline'),
+ )
def __unicode__(self):
return self.name
+
+class PipelineLog(models.Model):
+ PIPELINE_LOG_ACTION_DISABLE = 'D'
+ PIPELINE_LOG_ACTION_ENABLE = 'E'
+ PIPELINE_LOG_ACTION_CHOICES = (
+ (PIPELINE_LOG_ACTION_DISABLE, 'disable'),
+ (PIPELINE_LOG_ACTION_ENABLE, 'enable'),
+ )
+ creator = models.ForeignKey(UserModel)
+ pipeline = models.ForeignKey(Pipeline)
+ created = models.DateTimeField(auto_now_add=True)
+ action = models.CharField(max_length=10,
+ choices=PIPELINE_LOG_ACTION_CHOICES)
+
+
class Search(models.Model):
name = models.CharField(max_length=100)
description = models.TextField(blank=True)
diff --git a/gracedb/events/urls.py b/gracedb/events/urls.py
index e0d425532..6794322fd 100644
--- a/gracedb/events/urls.py
+++ b/gracedb/events/urls.py
@@ -45,6 +45,14 @@ urlpatterns = [
url(r'^(?P<graceid>[GEHMT]\d+)/emobservation/(?P<num>([\d]*|preview))$',
views.emobservation_entry, name="emobservation_entry"),
+ # Manage pipelines
+ url(r'^pipelines/manage/$', views.PipelineManageView.as_view(),
+ name='manage-pipelines'),
+ url(r'^pipelines/(?P<pk>\d+)/enable/$', views.PipelineEnableView.as_view(),
+ name='enable-pipeline'),
+ url(r'^pipelines/(?P<pk>\d+)/disable/$',
+ views.PipelineDisableView.as_view(), name='disable-pipeline'),
+
# Legacy URLs -------------------------------------------------------------
# Event detail
url(r'^view/(?P<graceid>[GEHMT]\d+)', views.view, name="legacyview"),
diff --git a/gracedb/events/views.py b/gracedb/events/views.py
index 472100517..caced391c 100644
--- a/gracedb/events/views.py
+++ b/gracedb/events/views.py
@@ -1,25 +1,31 @@
+from datetime import timedelta
from django.core.exceptions import PermissionDenied
from django.http import HttpResponse
from django.http import HttpResponseRedirect, HttpResponseNotFound, HttpResponseBadRequest, Http404
from django.http import HttpResponseForbidden, HttpResponseServerError
from django.template import RequestContext
-from django.urls import reverse
+from django.urls import reverse, reverse_lazy
from django.shortcuts import render
+from django.utils import timezone
+from django.utils.decorators import method_decorator
+from django.views.generic import ListView
+from django.views.generic.edit import UpdateView
from core.file_utils import get_file_list
from core.http import check_and_serve_file
from .models import Event, Group, EventLog, Label, Tag, Pipeline, Search, GrbEvent
-from .models import EMGroup, Signoff
+from .models import EMGroup, Signoff, PipelineLog
from .forms import CreateEventForm, SignoffForm
+from django.contrib.auth.decorators import permission_required
from django.contrib.auth.models import User, Permission
from django.contrib.auth.models import Group as AuthGroup
from django.contrib.contenttypes.models import ContentType
from .permission_utils import filter_events_for_user, user_has_perm
-from .permission_utils import internal_user_required, is_external, \
- check_external_file_access
+from .permission_utils import is_external, check_external_file_access
from guardian.models import GroupObjectPermission
+from ligoauth.decorators import internal_user_required
from .view_logic import _createEventFromForm
from .view_logic import get_performance_info
@@ -170,6 +176,15 @@ def _create(request):
if not user_has_perm(request.user, "populate", pipeline):
return HttpResponseForbidden("You do not have permission to submit events to this pipeline.")
+ # Get search since we won't block MDC event submissions even if
+ # the pipeline is disabled
+ search_name = request.POST.get('search', None)
+ if not pipeline.enabled and search_name != 'MDC':
+ err_msg = ('The {0} pipeline has been temporarily disabled by '
+ 'an EM advocate due to suspected misbehavior.').format(
+ pipeline.name)
+ return HttpResponseBadRequest(err_msg)
+
form = CreateEventForm(request.POST, request.FILES)
if form.is_valid():
# Alert is issued in this function
@@ -958,3 +973,94 @@ def modify_signoff(request, event):
# Finished. Redirect back to the event.
return HttpResponseRedirect(reverse("view", args=[event.graceid]))
+
+# Managing pipeline submissions -----------------------------------------------
+PIPELINE_LIST = ['gstlal', 'pycbc', 'MBTAOnline', 'CWB', 'oLIB', 'spiir']
+PIPELINE_LOG_ACTION_DICT = dict(PipelineLog.PIPELINE_LOG_ACTION_CHOICES)
+
+@method_decorator(internal_user_required(raise_exception=True),
+ name='dispatch')
+class PipelineManageView(ListView):
+ model = Pipeline
+ template_name = 'gracedb/manage_pipelines.html'
+ log_number = 10
+
+ def get_queryset(self):
+ qs = Pipeline.objects.filter(name__in=PIPELINE_LIST).order_by('name')
+ return qs
+
+ def get_context_data(self, **kwargs):
+ context = super(PipelineManageView, self).get_context_data(**kwargs)
+
+ # Get number of events created in a few different time periods for
+ # each pipeline and last submission time
+ n_events_dict = {}
+ submission_dict = {}
+ now = timezone.now()
+ dts = [now-timedelta(minutes=1), now-timedelta(minutes=10),
+ now-timedelta(minutes=60), now-timedelta(days=1)]
+ for p in self.object_list:
+ n_events_dict[p.name] = [p.event_set.filter(created__gt=dt)
+ .exclude(group__name='Test').exclude(search__name='MDC')
+ .count() for dt in dts]
+ last_event = p.event_set.exclude(group__name='Test').exclude(
+ search__name='MDC').order_by('-pk').first()
+ submission_dict[p.name] = getattr(last_event, 'created', None)
+ context['n_events_dict'] = n_events_dict
+ context['submission_dict'] = submission_dict
+
+ # Get list of pipeline logs
+ context['logs'] = PipelineLog.objects.order_by('-created')[
+ :self.log_number]
+ log_message_template = '{pipeline} {action}d by {user} at {dt}'
+ context['log_messages'] = [log_message_template.format(
+ pipeline=log.pipeline.name, user=log.creator.get_full_name(),
+ dt=log.created.strftime('%H:%M:%S %Z on %B %e, %Y'),
+ action=PIPELINE_LOG_ACTION_DICT[log.action])
+ for log in context['logs']]
+
+ # Determine whether user can enable/disable pipelines
+ context['user_can_manage'] = self.request.user.has_perm(
+ 'events.manage_pipeline')
+
+ return context
+
+
+@method_decorator(permission_required('events.manage_pipeline',
+ raise_exception=True), name='dispatch')
+class PipelineEnableView(UpdateView):
+ """Enable a pipeline"""
+ success_url = reverse_lazy('manage-pipelines')
+
+ def get_queryset(self):
+ qs = Pipeline.objects.filter(name__in=PIPELINE_LIST)
+ return qs
+
+ def get(self, request, *args, **kwargs):
+ return self.post(request, *args, **kwargs)
+
+ def post(self, request, *args, **kwargs):
+ self.object = self.get_object()
+ if not self.object.enabled:
+ self.object.enabled = True
+ self.object.save(update_fields=['enabled'])
+ PipelineLog.objects.create(creator=request.user,
+ pipeline=self.object,
+ action=PipelineLog.PIPELINE_LOG_ACTION_ENABLE)
+ return HttpResponseRedirect(self.get_success_url())
+
+
+@method_decorator(permission_required('events.manage_pipeline',
+ raise_exception=True), name='dispatch')
+class PipelineDisableView(PipelineEnableView):
+ """Disable a pipeline"""
+
+ def post(self, request, *args, **kwargs):
+ self.object = self.get_object()
+ if self.object.enabled:
+ self.object.enabled = False
+ self.object.save(update_fields=['enabled'])
+ PipelineLog.objects.create(creator=request.user,
+ pipeline=self.object,
+ action=PipelineLog.PIPELINE_LOG_ACTION_DISABLE)
+ return HttpResponseRedirect(self.get_success_url())
diff --git a/gracedb/ligoauth/context_processors.py b/gracedb/ligoauth/context_processors.py
index a44dc9cd6..98724f9e6 100644
--- a/gracedb/ligoauth/context_processors.py
+++ b/gracedb/ligoauth/context_processors.py
@@ -4,11 +4,14 @@ def LigoAuthContext(request):
user_is_internal = False
user_is_lvem = False
+ user_is_advocate = False # user is an EM advocate
if request.user:
if request.user.groups.filter(name=settings.LVC_GROUP).exists():
user_is_internal = True
if request.user.groups.filter(name=settings.LVEM_GROUP).exists():
user_is_lvem = True
+ if request.user.groups.filter(name=settings.EM_ADVOCATE_GROUP).exists():
+ user_is_advocate = True
return {'user': request.user, 'user_is_internal': user_is_internal,
- 'user_is_lvem': user_is_lvem}
+ 'user_is_lvem': user_is_lvem, 'user_is_advocate': user_is_advocate}
diff --git a/gracedb/migrations/auth/0023_add_manage_pipeline_permissions.py b/gracedb/migrations/auth/0023_add_manage_pipeline_permissions.py
new file mode 100644
index 000000000..fcdafaafa
--- /dev/null
+++ b/gracedb/migrations/auth/0023_add_manage_pipeline_permissions.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-04-02 21:21
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations
+from django.contrib.auth.management import create_permissions
+
+
+GROUP_NAME = settings.EM_ADVOCATE_GROUP
+PERMISSION_KWARGS = {
+ 'content_type__app_label': 'events',
+ 'codename': 'manage_pipeline',
+}
+
+
+
+# We have to run this to force the permissions to actually be created.
+# Otherwise they are created by a post-migrate signal and it's not possible
+# to run all of the migrations in a single command
+def create_perms(apps, schema_editor):
+ for app_config in apps.get_app_configs():
+ app_config.models_module = True
+ create_permissions(app_config, apps=apps, verbosity=0)
+ app_config.models_module = None
+
+
+def add_perm(apps, schema_editor):
+ Group = apps.get_model('auth', 'Group')
+ Permission = apps.get_model('auth', 'Permission')
+
+ # Get permission
+ perm = Permission.objects.get(**PERMISSION_KWARGS)
+
+ # Get group
+ group = Group.objects.get(name=GROUP_NAME)
+
+ # Add permission to group
+ perm.group_set.add(group)
+
+
+def remove_perm(apps, schema_editor):
+ Group = apps.get_model('auth', 'Group')
+ Permission = apps.get_model('auth', 'Permission')
+
+ # Get permission
+ perm = Permission.objects.get(**PERMISSION_KWARGS)
+
+ # Get group
+ group = Group.objects.get(name=GROUP_NAME)
+
+ # Remove permission from group
+ perm.group_set.remove(group)
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('auth', '0022_populate_raven_users_group'),
+ ('events', '0033_pipelinelog_and_pipeline_enabled'),
+ ]
+
+ operations = [
+ migrations.RunPython(create_perms, migrations.RunPython.noop),
+ migrations.RunPython(add_perm, remove_perm),
+ ]
diff --git a/gracedb/templates/gracedb/manage_pipelines.html b/gracedb/templates/gracedb/manage_pipelines.html
new file mode 100644
index 000000000..078b36ab3
--- /dev/null
+++ b/gracedb/templates/gracedb/manage_pipelines.html
@@ -0,0 +1,104 @@
+{% extends "base.html" %}
+{% load util_tags %}
+
+{% block title %}Manage pipelines{% endblock %}
+{% block heading %}Enable/disable analysis pipelines{% endblock %}
+{% block pageid %}pipelines{% endblock %}
+
+{% load static %}
+{% block headcontents %}
+ <link rel="stylesheet" href="{% static "css/bootstrap_buttons.css" %}"></script>
+ {{ block.super }}
+{% endblock %}
+
+{% block content %}
+
+<div style="padding-bottom: 20px;">
+<p>This page provides a brief summary of a pipeline's recent activity and provides the ability for EM advocates to disable (or enable) a pipeline's ability to submit events.
+When a pipeline is disabled, non-Test, non-MDC events cannot be submitted for it.
+This functionality is intended to mitigate the effects of a pipeline that is misbehaving and submitting erroneous event data.</p>
+<p>All users: if you suspect any problems with a pipeline, you should contact one of the EM advocates who is currently on duty. The EM advocate schedule for O3 is <a href="https://ldas-jobs.ligo.caltech.edu/~emfollow/followup-advocate-guide/roster.html#shift-calendar">here</a>.</p>
+<p>EM advocates: after you disable or enable a pipeline, you should contact the <a href="mailto:emfollow@ligo.org">low-latency group</a> to describe the action you have taken and the reasons for it.</p>
+</div>
+
+{% if user_can_manage %}
+<div align="center" style="padding-bottom: 20px;">
+<div style="background-color: red; color: white; display: inline-block; padding: 10px; border-radius: 5px;">
+<p align="center" style="font-weight: bold; font-size: 1.8rem; margin: 0;">WARNING: this page is live and any actions you take here will have consequences. Please be careful!</p>
+</div>
+</div>
+{% endif %}
+
+<div style="padding-bottom: 40px;">
+<table class="bootstrap-dark" align="center">
+<caption>Pipeline status</caption>
+ <tr>
+ <th></th>
+ <th></th>
+ <th style="text-align: center;" colspan="5">Recent submissions</th>
+ <th></th>
+ </tr>
+ <tr>
+ <th>Pipeline</th>
+ <th class="center">Current status</th>
+ <th class="center">Time of last submission (UTC)</th>
+ <th class="center">Last minute</th>
+ <th class="center">Last 10 minutes</th>
+ <th class="center">Last hour</th>
+ <th class="center">Last day</th>
+ <th class="center">Action</th>
+ </tr>
+ {% for pipeline in pipeline_list %}
+ <tr>
+ <td>{{ pipeline.name }}</td>
+ <td class="center">
+ {% if pipeline.enabled %}
+ <span style="color: green; font-weight: bold;">Enabled</span>
+ {% else %}
+ <span style="color: red; font-weight: bold;">Disabled</span>
+ {% endif %}
+ </td>
+ <td class="center">{{ submission_dict|get_item:pipeline.name }}</td>
+ {% with n_events_dict|get_item:pipeline.name as n_events_list %}
+ <td class="center">{{ n_events_list.0 }}</td>
+ <td class="center">{{ n_events_list.1 }}</td>
+ <td class="center">{{ n_events_list.2 }}</td>
+ <td class="center">{{ n_events_list.3 }}</td>
+ {% endwith %}
+ <td class="center">
+ {% if pipeline.enabled %}
+ {% if user_can_manage %}
+ <a class="btn btn-danger" href="{% url "disable-pipeline" pipeline.pk %}">Disable</a>
+ {% else %}
+ <button class="btn btn-danger" disabled>Disable</button>
+ {% endif %}
+ {% else %}
+ {% if user_can_manage %}
+ <a class="btn btn-success" href="{% url "enable-pipeline" pipeline.pk %}">Enable</a>
+ {% else %}
+ <button class="btn btn-success" disabled>Enable</button>
+ {% endif %}
+ {% endif %}
+ </td>
+ </tr>
+ {% endfor %}
+</table>
+</div>
+
+{% if log_messages %}
+<div style="padding-bottom: 10px;">
+<table class="bootstrap-dark" align="center">
+<caption>Recent activity</caption>
+<tr>
+ <th>#</th>
+ <th>Comment</th>
+{% for log in log_messages %}
+<tr>
+ <td>{{ forloop.revcounter }}
+ <td>{{ log }}</td>
+</tr>
+{% endfor %}
+</table>
+</div>
+{% endif %}
+{% endblock %}
diff --git a/gracedb/templates/navbar_frag.html b/gracedb/templates/navbar_frag.html
index 578a3e61d..562fcfaa3 100644
--- a/gracedb/templates/navbar_frag.html
+++ b/gracedb/templates/navbar_frag.html
@@ -13,6 +13,9 @@
{% elif user_is_lvem %}
<li id="nav-password"><a href="{% url "manage-password" %}">Manage Password</a></li>
{% endif %}
+ {% if user_is_advocate %}
+ <li id="nav-pipelines"><a href="{% url "manage-pipelines" %}">Pipelines</a></li>
+ {% endif %}
<li id="nav-docs"><a href="{% url "home" %}documentation/">Documentation</a></li>
{% if user %}
{% if user.is_superuser %}
--
GitLab