From ba37dbb3a707445791c9f931e981c54c88252edf Mon Sep 17 00:00:00 2001 From: Alexander Pace <alexander.pace@ligo.org> Date: Thu, 30 Nov 2023 20:27:38 +0000 Subject: [PATCH] GWTC models and API creates gwtc catalog/superevent/gevent database tables and foreignkeys. assigns permissions to catalog_managers group, and sets up the creation API backend. TODO: work on querying and catalog event views. --- config/settings/base.py | 1 + gracedb/api/v1/gwtc/__init__.py | 0 gracedb/api/v1/gwtc/permissions.py | 23 ++ gracedb/api/v1/gwtc/serializers.py | 128 +++++++++ gracedb/api/v1/gwtc/tests/__init__.py | 0 gracedb/api/v1/gwtc/tests/test_access.py | 246 ++++++++++++++++++ gracedb/api/v1/gwtc/urls.py | 17 ++ gracedb/api/v1/gwtc/viewsets.py | 61 +++++ gracedb/api/v1/main/views.py | 10 + gracedb/api/v1/urls.py | 4 + gracedb/core/tests/utils.py | 50 ++++ gracedb/events/tests/mixins.py | 38 ++- gracedb/gwtc/__init__.py | 0 gracedb/gwtc/apps.py | 6 + gracedb/gwtc/migrations/0001_initial.py | 102 ++++++++ gracedb/gwtc/migrations/__init__.py | 0 gracedb/gwtc/models.py | 92 +++++++ gracedb/gwtc/tests/mixins.py | 39 +++ gracedb/gwtc/urls.py | 56 ++++ gracedb/gwtc/views.py | 3 + .../migrations/0096_create_cm_authgroup.py | 72 +++++ .../auth/0030_create_gwtc_groups.py | 42 +++ .../migrations/auth/0031_assign_gwtc_perms.py | 67 +++++ 23 files changed, 1056 insertions(+), 1 deletion(-) create mode 100644 gracedb/api/v1/gwtc/__init__.py create mode 100644 gracedb/api/v1/gwtc/permissions.py create mode 100644 gracedb/api/v1/gwtc/serializers.py create mode 100644 gracedb/api/v1/gwtc/tests/__init__.py create mode 100644 gracedb/api/v1/gwtc/tests/test_access.py create mode 100644 gracedb/api/v1/gwtc/urls.py create mode 100644 gracedb/api/v1/gwtc/viewsets.py create mode 100644 gracedb/gwtc/__init__.py create mode 100644 gracedb/gwtc/apps.py create mode 100644 gracedb/gwtc/migrations/0001_initial.py create mode 100644 gracedb/gwtc/migrations/__init__.py create mode 100644 gracedb/gwtc/models.py create mode 100644 gracedb/gwtc/tests/mixins.py create mode 100644 gracedb/gwtc/urls.py create mode 100644 gracedb/gwtc/views.py create mode 100644 gracedb/ligoauth/migrations/0096_create_cm_authgroup.py create mode 100644 gracedb/migrations/auth/0030_create_gwtc_groups.py create mode 100644 gracedb/migrations/auth/0031_assign_gwtc_perms.py diff --git a/config/settings/base.py b/config/settings/base.py index 1b8f282d7..a75b33634 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -398,6 +398,7 @@ INSTALLED_APPS = [ 'ligoauth', 'search', 'superevents', + 'gwtc', 'rest_framework', 'guardian', 'django_twilio', diff --git a/gracedb/api/v1/gwtc/__init__.py b/gracedb/api/v1/gwtc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gracedb/api/v1/gwtc/permissions.py b/gracedb/api/v1/gwtc/permissions.py new file mode 100644 index 000000000..d39bf36d0 --- /dev/null +++ b/gracedb/api/v1/gwtc/permissions.py @@ -0,0 +1,23 @@ +import logging + +from django.conf import settings +from django.urls import resolve + +from rest_framework import permissions + +from superevents.models import Superevent, Signoff +from ..permissions import FunctionalModelPermissions, \ + FunctionalObjectPermissions, FunctionalParentObjectPermissions + +# Set up logger +logger = logging.getLogger(__name__) + + +class gwtc_model_permissions(FunctionalModelPermissions): + authenticated_users_only = True + allowed_methods = ['GET', 'POST'] + + def get_post_permissions(self, request): + # Just return the 'add_gwtc_catalog' perm for now until there + # needs to be finer-grained control or new methods. + return ['gwtc.add_gwtc_catalog'] diff --git a/gracedb/api/v1/gwtc/serializers.py b/gracedb/api/v1/gwtc/serializers.py new file mode 100644 index 000000000..c7be6c5ec --- /dev/null +++ b/gracedb/api/v1/gwtc/serializers.py @@ -0,0 +1,128 @@ +from __future__ import absolute_import +import functools +import logging +import os +import json + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from rest_framework import fields, serializers, validators +from rest_framework.exceptions import ValidationError + +from gwtc.models import gwtc_catalog, gwtc_gevent, gwtc_superevent +from superevents.models import Superevent +from events.models import Event, Pipeline + +# Set up user model +UserModel = get_user_model() + +# Set up logger +logger = logging.getLogger(__name__) + +class gwtc_serializer(serializers.ModelSerializer): + number = serializers.SlugField(required=True, allow_null=False, allow_blank=False) + smap = serializers.JSONField(required=True, write_only=True, allow_null=False) + version = serializers.ReadOnlyField() + # Get user from request automatically + user = serializers.HiddenField(write_only=True, + default=serializers.CurrentUserDefault()) + submitter = serializers.SlugRelatedField(slug_field='username', + read_only=True) + created = serializers.DateTimeField(format=settings.GRACE_STRFTIME_FORMAT, + read_only=True) + + # Custom display fields. These are read-only fields for displaying/GET'ing + # with the API: + gwtc_superevents = serializers.SerializerMethodField(read_only=True) + + + class Meta: + model = gwtc_catalog + fields = ('number', 'version', 'created', 'submitter', + 'smap', 'user', 'gwtc_superevents') + + # Look at the smap json. Perform checks such as verifying that superevents + # are in the database, and that events are part of that superevent. + def validate_smap(self, value): + for sevent, events in value.items(): + # first try and get the superevent: + try: + s = Superevent.get_by_date_id(sevent) + except (ObjectDoesNotExist, Superevent.DateIdError): + raise serializers.ValidationError(f'Superevent {sevent} ' + 'was not found in the database.') + + # Now loop over the events and pipelines that are part of that superevent: + for pipeline, event in events.items(): + # see if the event exists: + try: + e = Event.getByGraceid(event) + + # ensure that we have the base event class: + if hasattr(e, 'event_ptr'): + e = e.event_ptr + + except ObjectDoesNotExist: + raise serializers.ValidationError(f'Catalog Error: Event {event} ' + 'was not found in the database.') + + # now, see if the event actually is part of the superevent: + if not e in s.events.all(): + raise serializers.ValidationError(f'Catalog Error: Event {event} ' + f'is not part of superevent {sevent}.') + + # now see if the event's pipeline matches up with what is provided. + if not e.pipeline.name == pipeline: + raise serializers.ValidationError('Catalog Error: ' + f'Pipeline of {event} ({e.pipeline.name}) ' + f'does not match the supplied pipeline ({pipeline})') + + # I think that's good? so return the validated data + return value + + def validate(self, data): + data = super(gwtc_serializer, self).validate(data) + return data + + def create(self, validated_data): + # This is the routine that creates the catalog object, the catalog superevents, + # and catalog events. + + # First, create the catalog object: + new_gwtc = gwtc_catalog.objects.create(number = validated_data.pop('number'), + submitter = validated_data.pop('user')) + + # Similar to the validation step, loop over the superevents and events and then + # add them to the catalog. + + catalog_map = validated_data.pop('smap') + + for sevent, events in catalog_map.items(): + + # Get the superevent, create the gwtc superevent + s = Superevent.get_by_date_id(sevent) + new_gwtc_superevent = gwtc_superevent.objects.create( + superevent = s, + gwtc_catalog = new_gwtc) + + # Now create the events. It would be more efficient to pre-fetch the pipelines, + # but whatever. + for pipeline, event in events.items(): + new_gwtc_event = gwtc_gevent.objects.create( + gwtc_catalog = new_gwtc, + gwtc_superevent = new_gwtc_superevent, + gevent = Event.getByGraceid(event), + pipeline = Pipeline.objects.get(name=pipeline)) + + return new_gwtc + + # Populate custom fields. + def get_gwtc_superevents(self, obj): + + return {se.superevent.superevent_id: {e.gevent.pipeline.name: e.gevent.graceid + for e in se.gwtc_gevent_set.all()} + for se in obj.gwtc_superevent_set.all()} diff --git a/gracedb/api/v1/gwtc/tests/__init__.py b/gracedb/api/v1/gwtc/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gracedb/api/v1/gwtc/tests/test_access.py b/gracedb/api/v1/gwtc/tests/test_access.py new file mode 100644 index 000000000..cd1674a12 --- /dev/null +++ b/gracedb/api/v1/gwtc/tests/test_access.py @@ -0,0 +1,246 @@ +from __future__ import absolute_import +import datetime +import ipdb +import json +import pytest + +from django.conf import settings +from django.urls import reverse +from django.core.cache import cache + +from guardian.shortcuts import assign_perm, remove_perm + +from api.tests.utils import GraceDbApiTestBase +from core.tests.utils import GraceDbTestBase, \ + catalog_managers_group_and_user_setup +from gwtc.tests.mixins import gwtc_create_mixin +from ...settings import API_VERSION + + +def v_reverse(viewname, *args, **kwargs): + """Easily customizable versioned API reverse for testing""" + viewname = 'api:{version}:'.format(version=API_VERSION) + viewname + return reverse(viewname, *args, **kwargs) + +class test_gwtc_list(GraceDbApiTestBase, gwtc_create_mixin, + catalog_managers_group_and_user_setup): + + @classmethod + def setUpClass(cls): + super(test_gwtc_list, cls).setUpClass() + cls.url = v_reverse('gwtc:gwtc-list') + + @classmethod + def setUpTestData(cls): + super(test_gwtc_list, cls).setUpTestData() + + # an initial catalog object: + cls.gwtc_test_1 = cls.create_gwtc_catalog(user=cls.internal_user, + gwtc_number='test1') + + # a template json string for POST operations: + cls.json_upload = '{{ "{superevent_id}":{{"{pipeline}":"{graceid}"}} }}' + + # two test superevents: + cls.test_superevent1 = cls.create_superevent(cls.internal_user) + cls.test_superevent2 = cls.create_superevent(cls.internal_user) + + # a test coincinspiralevent: + cls.test_coinc = cls.create_coinc_event('CBC', 'gstlal') + + # smap jsons based on those two test superevents: + cls.smap_superevent1 = cls.json_upload.format(superevent_id = cls.test_superevent1.superevent_id, + pipeline = cls.test_superevent1.preferred_event.pipeline.name, + graceid = cls.test_superevent1.preferred_event.graceid) + + cls.smap_superevent2 = cls.json_upload.format(superevent_id = cls.test_superevent2.superevent_id, + pipeline = cls.test_superevent2.preferred_event.pipeline.name, + graceid = cls.test_superevent2.preferred_event.graceid) + + # json with an invalid superevent ID (date not in range): + cls.smap_bad_sid = cls.json_upload.format(superevent_id = "S123456ab", + pipeline = cls.test_superevent1.preferred_event.pipeline.name, + graceid = cls.test_superevent1.preferred_event.graceid) + + # json with valid superevent id, but it isn't in the database: + cls.smap_no_sevent = cls.json_upload.format(superevent_id = "S231129ab", + pipeline = cls.test_superevent1.preferred_event.pipeline.name, + graceid = cls.test_superevent1.preferred_event.graceid) + + # json with graceid not in the database: + cls.smap_no_gevent = cls.json_upload.format(superevent_id = cls.test_superevent2.superevent_id, + pipeline = cls.test_superevent2.preferred_event.pipeline.name, + graceid = "G12345") + + # json with invalid pipeline name in the upload: + cls.smap_bad_pipeline = cls.json_upload.format(superevent_id = cls.test_superevent2.superevent_id, + pipeline = "gstlol", + graceid = cls.test_superevent2.preferred_event.graceid) + + # json with a valid coinc graceid, but it's not part of the associated superevent: + cls.smap_coinc_gevent = cls.json_upload.format(superevent_id = cls.test_superevent2.superevent_id, + pipeline = cls.test_superevent2.preferred_event.pipeline.name, + graceid = cls.test_coinc.graceid) + + + def test_internal_user_get_gwtc_list(self): + """Internal user sees catalog entries""" + response = self.request_as_user(self.url, "GET", self.internal_user) + + # confirm the user can access the catalogs + self.assertEqual(response.status_code, 200) + + # confirm there is only one catalog entry: + data = response.data['results'] + self.assertEqual(len(data), 1) + + # confirm the number is correct: + data = data[0] + gwtc_name = data['number'] + self.assertEqual(gwtc_name, 'test1') + + # confirm that the version is 1: + gwtc_version = int(data['version']) + self.assertEqual(gwtc_version, 1) + + + def test_public_gwtc_access(self): + """At this time, the public cannot see gwtc objects""" + response = self.request_as_user(self.url, "GET") + + # confirm the public cannot see gwtc objects + self.assertEqual(response.status_code, 403) + + + def test_internal_user_not_create_gwtc(self): + """Authenticated users not in catalog_managers cannot create gwtc's""" + request_data = {"number": "test1", "smap": self.smap_superevent1} + response = self.request_as_user(self.url, "POST", self.internal_user, + data=request_data) + + # confirm the status code is correct: + self.assertEqual(response.status_code, 403) + + + def test_cm_user_create_gwtc(self): + """Verify a catalog manager can create new gwtc objects""" + + request_data = {"number": "test1", "smap": self.smap_superevent1} + response = self.request_as_user(self.url, "POST", self.cm_user, + data=request_data) + + # confirm the status code is correct: + self.assertEqual(response.status_code, 201) + + # get the new list and confirm version numbers, etc: + response = self.request_as_user(self.url, "GET", self.cm_user) + + # confirm there are two catalog entries: + data = response.data['results'] + self.assertEqual(len(data), 2) + + + def test_queries_and_paths(self): + """Make another catalog with a different number, and then verify + that the API query paths work.""" + + # create a second number='test1' gwtc. this should be version 2: + request_data = {"number": "test1", "smap": self.smap_superevent1} + response = self.request_as_user(self.url, "POST", self.cm_user, + data=request_data) + self.assertEqual(response.status_code, 201) + + # create a number='test2' gwtc: + request_data = {"number": "test2", "smap": self.smap_superevent2} + response = self.request_as_user(self.url, "POST", self.cm_user, + data=request_data) + self.assertEqual(response.status_code, 201) + + # get all the gwtc's and confirm there are three: + response = self.request_as_user(self.url, "GET", self.internal_user) + data = response.data['results'] + self.assertEqual(len(data), 3) + + # get all 'test1' gwtc's and count there are two: + response = self.request_as_user(self.url+'test1/', + "GET", self.internal_user) + data = response.data['results'] + self.assertEqual(len(data), 2) + + # get the 'latest' test1 gtwc, verify that it's version=2 + response = self.request_as_user(self.url+'test1/latest/', + "GET", self.internal_user) + gwtc_version = int(response.data['version']) + self.assertEqual(gwtc_version, 2) + + + + def test_get_wrong_number(self): + """Querying for the wrong number gives no results""" + + response = self.request_as_user(self.url+'test123/', + "GET", self.internal_user) + + # confirm the status code is 200: + self.assertEqual(response.status_code, 200) + + # confirm there are no catalog entries: + data = response.data['results'] + self.assertEqual(len(data), 0) + + + def test_create_bad_superevent_id(self): + """User attempts to upload smap with invalid superevent_id""" + + request_data = {"number": "test1", "smap": self.smap_bad_sid} + response = self.request_as_user(self.url, "POST", self.cm_user, + data=request_data) + + # confirm the status code is correct: + self.assertEqual(response.status_code, 400) + + + def test_create_no_superevent(self): + """User attempts to upload smap with a superevent_id not in the db""" + + request_data = {"number": "test1", "smap": self.smap_no_gevent} + response = self.request_as_user(self.url, "POST", self.cm_user, + data=request_data) + + # confirm the status code is correct: + self.assertEqual(response.status_code, 400) + + + def test_create_no_gevent(self): + """User attempts to upload smap with a graceid not in the db""" + + request_data = {"number": "test1", "smap": self.smap_no_gevent} + response = self.request_as_user(self.url, "POST", self.cm_user, + data=request_data) + + # confirm the status code is correct: + self.assertEqual(response.status_code, 400) + + + def test_create_pipeline_mismatch(self): + """User attempts to upload smap with a mismatch between pipeline + and the g-event""" + + request_data = {"number": "test1", "smap": self.smap_bad_pipeline} + response = self.request_as_user(self.url, "POST", self.cm_user, + data=request_data) + + # confirm the status code is correct: + self.assertEqual(response.status_code, 400) + + + def test_coinc_not_in_superevent(self): + """ User attempts to upload smap with valid graceic, but not part + of the specified superevent """ + + request_data = {"number": "test1", "smap": self.smap_coinc_gevent} + response = self.request_as_user(self.url, "POST", self.cm_user, + data=request_data) + + # confirm the status code is correct: + self.assertEqual(response.status_code, 400) diff --git a/gracedb/api/v1/gwtc/urls.py b/gracedb/api/v1/gwtc/urls.py new file mode 100644 index 000000000..9a342629e --- /dev/null +++ b/gracedb/api/v1/gwtc/urls.py @@ -0,0 +1,17 @@ +from django.urls import re_path, include, path +from rest_framework import routers + +from .viewsets import * + + +urlpatterns = [ + # Listing of all gwtc's: + re_path(r'^$', gwtc_viewset.as_view({'get': 'list', 'post': 'create'}), name='gwtc-list'), + + # Listing of all versions of a gwtc number: + path('<slug:number>/', gwtc_viewset.as_view({'get': 'list',}), name='gwtc-number-list'), + + # show a single version of a gwtc number: + path('<slug:number>/<slug:version>/', gwtc_viewset.as_view({'get': 'retrieve',}), name='gwtc-version-detail'), + +] diff --git a/gracedb/api/v1/gwtc/viewsets.py b/gracedb/api/v1/gwtc/viewsets.py new file mode 100644 index 000000000..5eb984eae --- /dev/null +++ b/gracedb/api/v1/gwtc/viewsets.py @@ -0,0 +1,61 @@ +import os + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.db.models import Max +from django.http import HttpResponse +from django.shortcuts import get_object_or_404 + +from guardian.shortcuts import get_objects_for_user +from rest_framework import mixins, parsers, permissions, serializers, status, \ + viewsets +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.views import APIView + +from ..mixins import SafeCreateMixin, InheritDefaultPermissionsMixin +from gwtc.models import gwtc_catalog, gwtc_gevent, gwtc_superevent +from .serializers import gwtc_serializer +from .permissions import gwtc_model_permissions + +class gwtc_viewset(SafeCreateMixin, InheritDefaultPermissionsMixin, + viewsets.ModelViewSet): + queryset = gwtc_catalog.objects.all() + serializer_class = gwtc_serializer + permission_classes = (gwtc_model_permissions, ) + + # This function gets called for 'list' methods, and returns a queryset. This + # is called for `GET` requests to /api/gwtc/ and /api/gwtc/{number} + def get_queryset(self): + # filter the kwargs. 'number' might be none, depending on the request + # address, in which case return the original queryset. + selected_number = self.kwargs.get('number') + selected_version = self.kwargs.get('version') + + # if no catalog number is specified, return all the catalogs + if not selected_number: + return self.queryset + else: + # if the user specified a number, but not a version, return all the versions + # of that catalog number + + return self.queryset.filter(number=selected_number) + + # This function gets called for 'list' methods, and returns a queryset. This + # is called for `GET` requests to /api/gwtc/{number}/{version} + def get_object(self): + # Get the kwargs from the request path. + selected_number = self.kwargs.get('number') + selected_version = self.kwargs.get('version') + + # a request to 'latest' should return the latest value, so put in a special + # case for that. + + if selected_version.lower() == 'latest': + selected_version = \ + self.queryset.filter(number=selected_number).aggregate(Max('version'))['version__max'] + filter_kwargs = {'number': selected_number, 'version': selected_version} + + obj = get_object_or_404(self.queryset, **filter_kwargs) + + return obj diff --git a/gracedb/api/v1/main/views.py b/gracedb/api/v1/main/views.py index 8387bc657..ab947a966 100644 --- a/gracedb/api/v1/main/views.py +++ b/gracedb/api/v1/main/views.py @@ -107,6 +107,13 @@ class GracedbRoot(APIView): request=request) update_grbevent = update_grbevent.replace("G1200", "{graceid}") + gwtc_number_list = api_reverse('gwtc:gwtc-number-list', args=['4a-1'], request=request) + gwtc_number_list = gwtc_number_list.replace('4a-1', '{number}') + + gwtc_version_detail = api_reverse('gwtc:gwtc-version-detail', args=['4a-1', '10'], request=request) + gwtc_version_detail = gwtc_version_detail.replace('4a-1', '{number}') + gwtc_version_detail = gwtc_version_detail.replace('10', '{version}') + # XXX Need a template for the tag list? templates = { @@ -124,6 +131,8 @@ class GracedbRoot(APIView): "taglist-template" : taglist, "signoff-list-template": signofflist, "update-grbevent-template": update_grbevent, + "gwtc-number-list": gwtc_number_list, + "gwtc-version-detail": gwtc_version_detail, } # Get superevent templates @@ -135,6 +144,7 @@ class GracedbRoot(APIView): "superevents" : api_reverse("superevents:superevent-list", request=request), "events" : api_reverse("events:event-list", request=request), + "gwtc" : api_reverse("gwtc:gwtc-list", request=request), "self" : api_reverse("root", request=request), "performance" : api_reverse("performance-info", request=request), "user-info": api_reverse("user-info", request=request), diff --git a/gracedb/api/v1/urls.py b/gracedb/api/v1/urls.py index 77612d251..1f8faadd6 100644 --- a/gracedb/api/v1/urls.py +++ b/gracedb/api/v1/urls.py @@ -7,6 +7,7 @@ from .main.views import GracedbRoot, PerformanceInfo, TagList, UserInfoView, \ from .events import urls as event_urls from .superevents import urls as superevent_urls +from .gwtc import urls as gwtc_urls # Turn off api caching: from django.views.decorators.cache import never_cache @@ -36,4 +37,7 @@ urlpatterns = [ # Superevents section of the API ------------------------------------------ re_path(r'^superevents/', include((superevent_urls, 'superevents'))), + + # Catalog section of the API ---------------------------------------------- + re_path(r'^gwtc/', include((gwtc_urls, 'gwtc'))), ] diff --git a/gracedb/core/tests/utils.py b/gracedb/core/tests/utils.py index 2df1c84cf..51c5b63d8 100644 --- a/gracedb/core/tests/utils.py +++ b/gracedb/core/tests/utils.py @@ -215,6 +215,56 @@ class SupereventManagersGroupAndUserSetup(TestCase): codename__in=sm_permissions_codenames) cls.sm_group.permissions.add(*perms) +class catalog_managers_group_and_user_setup(TestCase): + """ + Base class which sets up cata;pg_managers group and user + These are accessible with self.cm_group and self.cm_user. + Also adds appropriate permissions. + """ + @classmethod + def setUpTestData(cls): + + # Run super + super(catalog_managers_group_and_user_setup, cls).setUpTestData() + + # Get or create access managers + cls.cm_group, _ = AuthGroup.objects.get_or_create( + name='catalog_managers') + + # Get or create user + cls.cm_user, _ = UserModel.objects.get_or_create( + username='catalog.manager') + + # Add user to catalog managers group + cls.cm_group.user_set.add(cls.cm_user) + + # Also add user to internal group + internal_group, created = AuthGroup.objects.get_or_create( + name=settings.LVC_GROUP) + if created: + internal_group.ldap_name = 'internal_ldap_group' + internal_group.save(update_fields=['ldap_name']) + + # Create AuthorizedLdapMember, link it to internal group: + authldapmember, created = AuthorizedLdapMember.objects.get_or_create( + name='TestLDAPAuthMember') + if created: + authldapmember.ldap_gname='internal_ldap_group' + authldapmember.ldap_authgroup=internal_group + authldapmember.save() + + internal_group.user_set.add(cls.cm_user) + + # Get permissions + cm_permissions_codenames = [ + 'add_gwtc_catalog', + 'delete_gwtc_catalog', + ] + perms = Permission.objects.filter( + content_type__app_label='gwtc', + codename__in=cm_permissions_codenames) + cls.cm_group.permissions.add(*perms) + class AccessManagersGroupAndUserSetup(TestCase): """ diff --git a/gracedb/events/tests/mixins.py b/gracedb/events/tests/mixins.py index 2a3eb8917..91dbc442c 100644 --- a/gracedb/events/tests/mixins.py +++ b/gracedb/events/tests/mixins.py @@ -7,7 +7,7 @@ from django.contrib.auth.models import Group as AuthGroup, Permission from django.contrib.contenttypes.models import ContentType from core.tests.utils import GraceDbTestBase -from events.models import Event, Group, Pipeline, Search, Tag +from events.models import Event, Group, Pipeline, Search, Tag, CoincInspiralEvent from events.permission_utils import assign_default_event_perms from events.views import update_event_perms_for_group @@ -53,6 +53,42 @@ class EventCreateMixin(object): return event + @staticmethod + def create_coinc_event(group_name, pipeline_name, search_name=None, + user=None): + """ + """ + + # Create group, pipeline, and optionally, user + group, _ = Group.objects.get_or_create(name=group_name) + pipeline, _ = Pipeline.objects.get_or_create(name=pipeline_name) + if user is None: + user, _ = UserModel.objects.get_or_create(username='event.user') + + # Compile event dict + event_dict = { + 'group': group, + 'pipeline': pipeline, + 'submitter': user, + 'gpstime': 123, + } + + # Set up search (if not None) and add to event_dict + if search_name is not None: + search, _ = Search.objects.get_or_create(name=search_name) + event_dict['search'] = search + + # Create event and return + event = CoincInspiralEvent.objects.create(**event_dict) + + # Save event to trigger field computation: + event.save() + + # Make data directory (should get removed at the end by + # GraceDbTestBase tearDown function) + os.makedirs(event.datadir) + + return event class EventSetup(GraceDbTestBase, EventCreateMixin): """ diff --git a/gracedb/gwtc/__init__.py b/gracedb/gwtc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gracedb/gwtc/apps.py b/gracedb/gwtc/apps.py new file mode 100644 index 000000000..a7c908a51 --- /dev/null +++ b/gracedb/gwtc/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class GwtcConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'gwtc' diff --git a/gracedb/gwtc/migrations/0001_initial.py b/gracedb/gwtc/migrations/0001_initial.py new file mode 100644 index 000000000..9b0c1bedf --- /dev/null +++ b/gracedb/gwtc/migrations/0001_initial.py @@ -0,0 +1,102 @@ +# Generated by Django 3.2.20 on 2023-10-05 20:32 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('superevents', '0018_superevent_autoincrement_indexes'), + ('auth', '0030_create_gwtc_groups'), + ('events', '0082_add_PyGRB_pipeline'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='gwtc_catalog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('number', models.SlugField()), + ('version', models.PositiveIntegerField()), + ('created', models.DateTimeField(auto_now_add=True)), + ('submitter', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-id'], + 'default_permissions': ('add', 'view', 'delete'), + }, + ), + migrations.CreateModel( + name='gwtc_superevent', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('gwtc_catalog', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gwtc.gwtc_catalog')), + ('superevent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='superevents.superevent')), + ], + options={ + 'ordering': ['-id'], + 'default_permissions': ('add', 'view', 'delete'), + 'unique_together': {('superevent', 'gwtc_catalog')}, + }, + ), + migrations.CreateModel( + name='gwtc_gevent', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('gevent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='events.event')), + ('gwtc_catalog', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gwtc.gwtc_catalog')), + ('gwtc_superevent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gwtc.gwtc_superevent')), + ('pipeline', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='events.pipeline')), + ], + options={ + 'ordering': ['-id'], + 'default_permissions': ('add', 'view', 'delete'), + 'unique_together': {('gwtc_superevent', 'gevent', 'pipeline')}, + }, + ), + migrations.CreateModel( + name='gwtc_catalog_userobjectpermission', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gwtc.gwtc_catalog')), + ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + 'unique_together': {('user', 'permission', 'content_object')}, + }, + ), + migrations.CreateModel( + name='gwtc_catalog_groupobjectpermission', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('content_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gwtc.gwtc_catalog')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.group')), + ('permission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.permission')), + ], + options={ + 'permissions': (('view_gwtc_cataloggroupobjectpermission', 'Can view gwtc_cataloggroupobjectpermission'),), + 'abstract': False, + 'default_permissions': ('add', 'view', 'delete'), + 'unique_together': {('group', 'permission', 'content_object')}, + }, + ), + migrations.AddIndex( + model_name='gwtc_catalog', + index=models.Index(fields=['number'], name='gwtc_gwtc_c_number_562de5_idx'), + ), + migrations.AddIndex( + model_name='gwtc_catalog', + index=models.Index(fields=['version'], name='gwtc_gwtc_c_version_2797bf_idx'), + ), + migrations.AlterUniqueTogether( + name='gwtc_catalog', + unique_together={('number', 'version')}, + ), + ] diff --git a/gracedb/gwtc/migrations/__init__.py b/gracedb/gwtc/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gracedb/gwtc/models.py b/gracedb/gwtc/models.py new file mode 100644 index 000000000..9234a3ae2 --- /dev/null +++ b/gracedb/gwtc/models.py @@ -0,0 +1,92 @@ +from django.contrib.auth import get_user_model +from django.db import models + +# Generic python stuff: +import logging + +# Import GraceDB stuff: +from core.models import AutoIncrementModel +from events.models import Event, Pipeline +from superevents.models import Superevent + +# permissions stuff: +from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase + +# Other setup +UserModel = get_user_model() +logger = logging.getLogger(__name__) + + + +class gwtc_catalog(AutoIncrementModel): + # Catalog versioning fields: + number = models.SlugField(null=False, max_length=50) + version = models.PositiveIntegerField(null=False) + + AUTO_FIELD = 'version' + AUTO_CONSTRAINTS = ('number',) + + # Book-keeping fields: + submitter = models.ForeignKey(UserModel, on_delete=models.CASCADE) + created = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f'GWTC{self.number}, version {self.version}' + + class Meta: + ordering = ["-id"] + unique_together = ( + ('number', 'version'), + ) + indexes = [models.Index(fields=['number', ]), + models.Index(fields=['version', ]),] + + default_permissions = ('add', 'view', 'delete') + +class gwtc_superevent(models.Model): + superevent = models.ForeignKey(Superevent, on_delete=models.CASCADE) + gwtc_catalog = models.ForeignKey(gwtc_catalog, on_delete=models.CASCADE) + + def __str__(self): + return f'{self.superevent.superevent_id} in ' \ + f'GWTC{self.gwtc_catalog.number}, version {self.gwtc_catalog.version}' + + class Meta: + ordering = ["-id"] + unique_together = ( + ('superevent', 'gwtc_catalog'), + ) + default_permissions = ('add', 'view', 'delete') + +class gwtc_gevent(models.Model): + gwtc_catalog = models.ForeignKey(gwtc_catalog, on_delete=models.CASCADE) + gwtc_superevent = models.ForeignKey(gwtc_superevent, on_delete=models.CASCADE) + gevent = models.ForeignKey(Event, on_delete=models.CASCADE) + pipeline = models.ForeignKey(Pipeline, on_delete=models.CASCADE) + + def __str__(self): + return f'{self.gevent.graceid} in ' \ + f'GWTC{self.gwtc_catalog.number}, version {self.gwtc_catalog.version}' + + class Meta: + ordering = ["-id"] + unique_together = ( + ('gwtc_superevent', 'gevent', 'pipeline'), + ) + default_permissions = ('add', 'view', 'delete') + +class gwtc_catalog_groupobjectpermission(GroupObjectPermissionBase): + content_object = models.ForeignKey(gwtc_catalog, on_delete=models.CASCADE) + + class Meta(GroupObjectPermissionBase.Meta): + + default_permissions = ('add', 'view', 'delete') + + permissions = ( + ('view_gwtc_cataloggroupobjectpermission', + 'Can view gwtc_cataloggroupobjectpermission'), + ) + +class gwtc_catalog_userobjectpermission(UserObjectPermissionBase): + content_object = models.ForeignKey(gwtc_catalog, on_delete=models.CASCADE) + diff --git a/gracedb/gwtc/tests/mixins.py b/gracedb/gwtc/tests/mixins.py new file mode 100644 index 000000000..aeddccfca --- /dev/null +++ b/gracedb/gwtc/tests/mixins.py @@ -0,0 +1,39 @@ +from gwtc.models import gwtc_catalog, gwtc_gevent, gwtc_superevent +from superevents.tests.mixins import SupereventCreateMixin + +# A class to create test gwtc superevents, events, and catalogs: + +class gwtc_create_mixin(SupereventCreateMixin): + """ + mixin to create test gwtc entries + """ + @classmethod + def create_gwtc_catalog(cls, user, gwtc_number='test1'): + + # create one superevent: + new_superevent = cls.create_superevent(user) + + # Now create a catalog object: + test_gwtc = gwtc_catalog.objects.create(number = gwtc_number, + submitter = user) + + # create the associated gwtc event and superevent: + test_gwtc_superevent = gwtc_superevent.objects.create( + superevent = new_superevent, + gwtc_catalog = test_gwtc) + + test_gwtc_event = gwtc_gevent.objects.create( + gwtc_catalog = test_gwtc, + gwtc_superevent = test_gwtc_superevent, + gevent = new_superevent.preferred_event, + pipeline = new_superevent.preferred_event.pipeline) + + return test_gwtc + +# @classmethod +# def setUpTestData(cls): +# super(gwtc_create_mixin, cls).setUpTestData() + + # Create a catalog: + + diff --git a/gracedb/gwtc/urls.py b/gracedb/gwtc/urls.py new file mode 100644 index 000000000..c3196f496 --- /dev/null +++ b/gracedb/gwtc/urls.py @@ -0,0 +1,56 @@ +from django.conf.urls import url, include +from django.urls import path +from superevent.models import Superevent +from superevent import views + +app_name = 'gwtc' + + +# URLs which are nested below a superevent detail +# These are included under a superevent's ID URL prefix (see below) +suburlpatterns = [ + + # Superevent detail view + url(r'^view/$', views.SupereventDetailView.as_view(), name="view"), + + # File list (file detail/download is handled through the API) + url(r'^files/$', views.SupereventFileList.as_view(), name="file-list"), +] + +# Legacy URL patterns - don't really need them, but we use them for the +# convenience of users who may be accustomed to the legacy event URL patterns + +# public page url patterns. This needs to go first so django matches +# the 'public' string before parsing it interprets it as a superevent_id. +# Note to future generations: don't name a GW 'public' or the link breaks. + +public_urlpatterns = [ + # The /superevents/public/ url route gets redirected to the + # latest observation run. + path('public/', views.public_alerts_redirect, + name="public-alerts-redirect"), + + # each run has its own page, but must be in the list of runs that's + # in settings/base.py. Otherwise, 404 (this gets defined in the view). + # the "obsrun" variable controls which events are shown. + path('public/<slug:obsrun>/', views.SupereventPublic.as_view(), + name="public-alerts"), + +] +legacy_urlpatterns = public_urlpatterns + [ + # Legacy URLs for superevent detail view + path('<str:superevent_id>/', + views.SupereventDetailView.as_view(), + name="legacyview1"), + path('view/<str:superevent_id>/', + views.SupereventDetailView.as_view(), + name="legacyview2"), +] + +# Full urlpatterns: legacy urls plus suburlpatterns nested under +# superevent_id +urlpatterns = legacy_urlpatterns + [ + path('<str:superevent_id>/', + include(suburlpatterns)), + +] diff --git a/gracedb/gwtc/views.py b/gracedb/gwtc/views.py new file mode 100644 index 000000000..91ea44a21 --- /dev/null +++ b/gracedb/gwtc/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/gracedb/ligoauth/migrations/0096_create_cm_authgroup.py b/gracedb/ligoauth/migrations/0096_create_cm_authgroup.py new file mode 100644 index 000000000..dade24080 --- /dev/null +++ b/gracedb/ligoauth/migrations/0096_create_cm_authgroup.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-05-28 18:18 +from __future__ import unicode_literals + +from django.db import migrations + +GROUP_DATA = [ + { + 'name': 'catalog_managers', + 'description': ('Catalog Managers who can create and delete GWTC objects.'), + }, +] + + +def create_authgroups(apps, schema_editor): + DjangoGroup = apps.get_model('auth', 'Group') + AuthGroup = apps.get_model('ligoauth', 'AuthGroup') + Tag = apps.get_model('events', 'Tag') + + # Create AuthGroup instances + for group in GROUP_DATA: + g, created = DjangoGroup.objects.get_or_create(name=group['name']) + ag = AuthGroup(group_ptr=g) + ag.description = group['description'] + + # Update from base class + ag.__dict__.update(g.__dict__) + + # Save + ag.save() + + # Add optional extras and save + if 'new_name' in group: + ag.name = group['new_name'] + if 'ldap_name' in group: + ag.ldap_name = group['ldap_name'] + if 'tag_name' in group: + tag, _ = Tag.objects.get_or_create(name=group['tag_name']) + ag.tag = tag + ag.save() + + +def delete_authgroups(apps, schema_editor): + AuthGroup = apps.get_model('ligoauth', 'AuthGroup') + + # Loop over groups and delete AuthGroup + for group in GROUP_DATA: + # Get AuthGroup + group_name = group['name'] + if 'new_name' in group: + group_name = group['new_name'] + ag = AuthGroup.objects.get(name=group_name) + + # Reset name if needed + if 'new_name' in group: + ag.name = group['name'] + ag.save() + + # Delete AuthGroup and keep DjangoGroup base class + ag.delete(keep_parents=True) + + +class Migration(migrations.Migration): + + dependencies = [ + ('ligoauth', '0095_another_gstlalcbc_cert'), + ('auth', '0030_create_gwtc_groups'), + ] + + operations = [ + migrations.RunPython(create_authgroups, delete_authgroups), + ] diff --git a/gracedb/migrations/auth/0030_create_gwtc_groups.py b/gracedb/migrations/auth/0030_create_gwtc_groups.py new file mode 100644 index 000000000..6951a4976 --- /dev/null +++ b/gracedb/migrations/auth/0030_create_gwtc_groups.py @@ -0,0 +1,42 @@ +# migration for creating gwtc catalog managers' group. +# copied and edited by hand from a previous migration. + +from __future__ import unicode_literals + +from django.db import migrations + +GROUPS = { + 'catalog_managers': ['cbcflow', 'chad.hanna@ligo.org', 'rhiannon.udall@ligo.org', + 'chad.hanna@ligo.org', 'rebecca.ewing@ligo.org', + 'prathamesh.joshi@ligo.org', 'divya.singh@ligo.org', + 'alexander.pace@ligo.org'], +} + +def add_groups(apps, schema_editor): + Group = apps.get_model('auth', 'Group') + User = apps.get_model('auth', 'User') + + for group_name, usernames in GROUPS.items(): + g, _ = Group.objects.get_or_create(name=group_name) + users = User.objects.filter(username__in=usernames) + g.user_set.add(*users) + + +def remove_groups(apps, schema_editor): + Group = apps.get_model('auth', 'Group') + User = apps.get_model('auth', 'User') + + for group_name in GROUPS: + g = Group.objects.get(name=group_name) + g.delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0029_alter_user_username'), + ] + + operations = [ + migrations.RunPython(add_groups, remove_groups), + ] diff --git a/gracedb/migrations/auth/0031_assign_gwtc_perms.py b/gracedb/migrations/auth/0031_assign_gwtc_perms.py new file mode 100644 index 000000000..c34cd0625 --- /dev/null +++ b/gracedb/migrations/auth/0031_assign_gwtc_perms.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.14 on 2018-08-02 17:42 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations +from django.contrib.auth.management import create_permissions + + +# Group names +CATALOG = 'catalog_managers' # handle catalog creation +LVC = 'internal_users' + +CATALOG_PERMS = { + # Superevent permissions + 'add_gwtc_catalog': [CATALOG], + 'delete_gwtc_catalog': [CATALOG], + # Viewing permissions + 'view_gwtc_cataloggroupobjectpermission': [LVC], +} + + +# We have to run this to force the permissions to actually be created. +# Otherwise they are created by a post-migrate signal and it's not possible +# to run all of the migrations in a single command +def create_perms(apps, schema_editor): + for app_config in apps.get_app_configs(): + app_config.models_module = True + create_permissions(app_config, apps=apps, verbosity=0) + app_config.models_module = None + + +def add_perms(apps, schema_editor): + Group = apps.get_model('auth', 'Group') + Permission = apps.get_model('auth', 'Permission') + + # Add catalog permissions to groups + for codename, group_names in CATALOG_PERMS.items(): + p = Permission.objects.get(codename=codename, + content_type__app_label='gwtc') + groups = Group.objects.filter(name__in=group_names) + p.group_set.add(*groups) + + +def remove_perms(apps, schema_editor): + Group = apps.get_model('auth', 'Group') + Permission = apps.get_model('auth', 'Permission') + + # Add permissions to groups + for codename, group_names in CATALOG_PERMS.items(): + p = Permission.objects.get(codename=codename, + content_type__app_label='gwtc') + groups = Group.objects.filter(name__in=group_names) + p.group_set.remove(*groups) + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0030_create_gwtc_groups'), + ('gwtc', '0001_initial'), + ] + + operations = [ + migrations.RunPython(create_perms, migrations.RunPython.noop), + migrations.RunPython(add_perms, remove_perms), + ] -- GitLab