diff --git a/config/settings/base.py b/config/settings/base.py index 07538188c7b219578fb4666c1fc799b1cb75598d..85302327ed42ad5d3214c4e72530c843b5091410 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -98,6 +98,8 @@ LVEM_OBSERVERS_GROUP = 'gw-astronomy:LV-EM:Observers' EXEC_GROUP = 'executives' # EM Advocate group name EM_ADVOCATE_GROUP = 'em_advocates' +# Analysis group name for non-GW events +EXTERNAL_ANALYSIS_GROUP = 'External' # Groups directly managed by GraceDB admins ADMIN_MANAGED_GROUPS = [EM_ADVOCATE_GROUP, 'executives'] @@ -288,6 +290,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'maintenance_mode', 'events', + 'superevents', 'userprofile', 'ligoauth', 'rest_framework', @@ -300,6 +303,15 @@ INSTALLED_APPS = [ SHELL_PLUS_MODEL_ALIASES = { # Two 'Group' models - auth.Group and gracedb.Group 'auth': {'Group': 'AuthGroup'}, + # Superevents models which have the same name as + # models in the events app + 'superevents': { + 'EMObservation': 'SupereventEMObservation', + 'Label': 'SupereventLabel', + 'Labelling': 'SupereventLabelling', + 'Log': 'SupereventLog', + 'VOEvent': 'SupereventVOEvent', + } } # Details used by REST API @@ -471,6 +483,11 @@ LOGGING = { 'propagate': True, 'level': LOG_LEVEL, }, + 'superevents': { + 'handlers': ['debug_file','error_file'], + 'propagate': True, + 'level': LOG_LEVEL, + }, 'performance': { 'handlers': ['performance_file'], 'propagate': True, diff --git a/config/urls.py b/config/urls.py index d84da0dd6feeff0b23a114e16656a4ebbeadc941..d1a9049558c7c9b876d75449d7e9a2b6332b91bb 100644 --- a/config/urls.py +++ b/config/urls.py @@ -26,6 +26,7 @@ urlpatterns = [ url(r'^SPPrivacy', events.views.spprivacy, name="spprivacy"), url(r'^DiscoveryService', events.views.discovery, name="discovery"), url(r'^events/', include('events.urls')), + url(r'^superevents/', include('superevents.urls')), url(r'^options/', include('userprofile.urls')), url(r'^feeds/(?P<url>.*)/$', EventFeed()), url(r'^feeds/$', feedview, name="feeds"), diff --git a/gracedb/events/migrations/0019_event_superevent.py b/gracedb/events/migrations/0019_event_superevent.py new file mode 100644 index 0000000000000000000000000000000000000000..22f66bea622c18dea561e1643ce5d6070113bcc0 --- /dev/null +++ b/gracedb/events/migrations/0019_event_superevent.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2018-04-06 19:49 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('superevents', '0001_initial'), + ('events', '0018_update_models'), + ] + + operations = [ + migrations.AddField( + model_name='event', + name='superevent', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='events', to='superevents.Superevent'), + ), + ] diff --git a/gracedb/events/models.py b/gracedb/events/models.py index f9afd3271b9c573e1a52fc137fdf957c7a41f6d3..e9c8c7b9bd706afd0fcd880285860ecd35482157 100644 --- a/gracedb/events/models.py +++ b/gracedb/events/models.py @@ -121,8 +121,8 @@ class Event(models.Model): # Events aren't required to be part of a superevent. If the superevent is # deleted, don't delete the event; just set this FK to null. - #superevent = models.ForeignKey('superevents.Superevent', null=True, - # related_name='events', on_delete=models.SET_NULL) + superevent = models.ForeignKey('superevents.Superevent', null=True, + related_name='events', on_delete=models.SET_NULL) # Note: a default value is needed only during the schema migration # that creates this column. After that, we can safely remove it. diff --git a/gracedb/superevents/__init__.py b/gracedb/superevents/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/gracedb/superevents/migrations/0001_initial.py b/gracedb/superevents/migrations/0001_initial.py new file mode 100644 index 0000000000000000000000000000000000000000..4b9a53ed8b316c180a1f633c4b8aee160656ccc9 --- /dev/null +++ b/gracedb/superevents/migrations/0001_initial.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2018-04-06 19:49 +from __future__ import unicode_literals + +import core.models +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('events', '0018_update_models'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='EMFootprint', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('N', models.IntegerField(editable=False)), + ('ra', models.FloatField()), + ('dec', models.FloatField()), + ('raWidth', models.FloatField()), + ('decWidth', models.FloatField()), + ('start_time', models.DateTimeField()), + ('exposure_time', models.PositiveIntegerField()), + ], + options={ + 'ordering': ['-N'], + 'abstract': False, + }, + ), + migrations.CreateModel( + name='EMObservation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('N', models.IntegerField()), + ('created', models.DateTimeField(auto_now_add=True)), + ('ra', models.FloatField(null=True)), + ('dec', models.FloatField(null=True)), + ('raWidth', models.FloatField(null=True)), + ('decWidth', models.FloatField(null=True)), + ('comment', models.TextField(blank=True)), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='superevents_emobservation_set', to='events.EMGroup')), + ('submitter', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='superevents_emobservation_set', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created', '-N'], + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Labelling', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='superevents_labelling_set', to=settings.AUTH_USER_MODEL)), + ('label', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='superevents_labelling_set', to='events.Label')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Log', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('filename', models.CharField(blank=True, default=b'', max_length=100)), + ('file_version', models.IntegerField(blank=True, default=None, null=True)), + ('comment', models.TextField()), + ('N', models.IntegerField(editable=False)), + ('issuer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created', '-N'], + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Signoff', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('comment', models.TextField(blank=True)), + ('instrument', models.CharField(blank=True, choices=[(b'H1', b'LHO'), (b'L1', b'LLO'), (b'V1', b'Virgo')], max_length=2)), + ('status', models.CharField(choices=[(b'OK', b'OKAY'), (b'NO', b'NOT OKAY')], max_length=2)), + ('signoff_type', models.CharField(choices=[(b'OP', b'operator'), (b'ADV', b'advocate')], max_length=3)), + ('submitter', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='superevents_signoff_set', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Superevent', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('t_start', models.DecimalField(decimal_places=6, max_digits=16)), + ('t_0', models.DecimalField(decimal_places=6, max_digits=16)), + ('t_end', models.DecimalField(decimal_places=6, max_digits=16)), + ('labels', models.ManyToManyField(through='superevents.Labelling', to='events.Label')), + ('preferred_event', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='superevent_preferred_for', to='events.Event')), + ('submitter', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-id'], + 'permissions': (('view_superevent', 'Can view superevent'),), + }, + bases=(models.Model, core.models.ModelToDictMixin), + ), + migrations.CreateModel( + name='VOEvent', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('ivorn', models.CharField(blank=True, default=b'', max_length=200)), + ('filename', models.CharField(blank=True, default=b'', max_length=100)), + ('file_version', models.IntegerField(blank=True, default=None, null=True)), + ('N', models.IntegerField(editable=False)), + ('voevent_type', models.CharField(choices=[(b'PR', b'preliminary'), (b'IN', b'initial'), (b'UP', b'update'), (b'RE', b'retraction')], max_length=2)), + ('issuer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='superevents_voevent_set', to=settings.AUTH_USER_MODEL)), + ('superevent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='superevents.Superevent')), + ], + options={ + 'ordering': ['-created', '-N'], + 'abstract': False, + }, + ), + migrations.AddField( + model_name='signoff', + name='superevent', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='superevents.Superevent'), + ), + migrations.AddField( + model_name='log', + name='superevent', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='superevents.Superevent'), + ), + migrations.AddField( + model_name='log', + name='tags', + field=models.ManyToManyField(related_name='superevent_logs', to='events.Tag'), + ), + migrations.AddField( + model_name='labelling', + name='superevent', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='superevents.Superevent'), + ), + migrations.AddField( + model_name='emobservation', + name='superevent', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='superevents.Superevent'), + ), + migrations.AddField( + model_name='emfootprint', + name='observation', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='superevents.EMObservation'), + ), + migrations.AlterUniqueTogether( + name='voevent', + unique_together=set([('superevent', 'N')]), + ), + migrations.AlterUniqueTogether( + name='signoff', + unique_together=set([('superevent', 'instrument')]), + ), + migrations.AlterUniqueTogether( + name='log', + unique_together=set([('superevent', 'N')]), + ), + migrations.AlterUniqueTogether( + name='emobservation', + unique_together=set([('superevent', 'N')]), + ), + migrations.AlterUniqueTogether( + name='emfootprint', + unique_together=set([('observation', 'N')]), + ), + ] diff --git a/gracedb/superevents/migrations/__init__.py b/gracedb/superevents/migrations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/gracedb/superevents/models.py b/gracedb/superevents/models.py new file mode 100644 index 0000000000000000000000000000000000000000..ce1758b86c3fa419145ef914fa59644fc9deb20a --- /dev/null +++ b/gracedb/superevents/models.py @@ -0,0 +1,305 @@ +from django.db import models, IntegrityError +from django.urls import reverse +from django.contrib.auth import get_user_model +from django.conf import settings +from django.core.exceptions import ValidationError +from django.utils.translation import ugettext_lazy as _ + +from core.models import CleanSaveModel, AutoIncrementModel +from core.models import LogBase, m2mThroughBase +from core.models import ModelToDictMixin +from core.time_utils import posixToGpsTime, gpsToUtc +from events.models import Event, SignoffBase, VOEventBase, EMObservationBase, \ + EMFootprintBase +from core.utils import int_to_letters + +from cStringIO import StringIO +from hashlib import sha1 +import os + +import logging + +# Other setup +UserModel = get_user_model() +logger = logging.getLogger(__name__) + + +class Superevent(CleanSaveModel, ModelToDictMixin): + ID_PREFIX = 'S' + + # Fields ------------------------------------------------------------------ + submitter = models.ForeignKey(UserModel) + date_created = models.DateTimeField(auto_now_add=True) + + # One-to-one relationship with preferred event - an event can only be + # preferred for a single superevent and a superevent can only have + # one preferred event. Ideally, this wouldn't be nullable, but it makes + # the logic a lot easier to handle. We will just have to check whether + # self.preferred_event is None in some cases. + preferred_event = models.OneToOneField(Event, null=True, blank=True, + on_delete=models.SET_NULL, related_name='superevent_preferred_for') + + # Labels + labels = models.ManyToManyField('events.label', through='Labelling') + + # Event time attributes + t_start = models.DecimalField(max_digits=16, decimal_places=6, null=False, + blank=False) + t_0 = models.DecimalField(max_digits=16, decimal_places=6, null=False, + blank=False) + t_end = models.DecimalField(max_digits=16, decimal_places=6, null=False, + blank=False) + + # Meta class -------------------------------------------------------------- + class Meta: + ordering = ["-id"] + + # Extra permissions beyond the standard add, change, delete perms + permissions = (('view_superevent', 'Can view superevent'),) + + # Class method overrides -------------------------------------------------- + def clean(self, *args, **kwargs): + + # If preferred event is not set, just pick the first non-external + # event in the set + # NOTE: do we actually want to do this? + if self.events.exists() and not self.preferred_event: + self.preferred_event = self.events.exclude(group__name= + settings.EXTERNAL_ANALYSIS_GROUP).first() + + if (self.preferred_event and self.preferred_event.group.name == + settings.EXTERNAL_ANALYSIS_GROUP): + raise ValidationError({'preferred_event': + _('External event cannot be set as preferred')}) + + super(Superevent, self).clean(*args, **kwargs) + + def save(self, *args, **kwargs): + """Custom save method for handling date_id and preferred event""" + + # Only modify date_id if pk is not already set (i.e. this is an INSERT) + #pk_set = self._get_pk_val() is not None + #if not pk_set: + # if self.date_id: + # raise Exception('ERROR') + + # # Find an attached event with a gpstime to get the date; + # # try preferred event first, of course. + # if self.preferred_event: + # + # else: + # # Get other superevents from this date to increment id + + # Do base class save + super(Superevent, self).save(*args, **kwargs) + + # Have to do this after save because the superevent needs a pk + # to be used as a foreign key in the event table + if (self.preferred_event and + self.preferred_event not in self.events.all()): + self.events.add(self.preferred_event) + + def get_absolute_url(self): + return self.get_web_url() + + def list_files(self, absolute_paths=True): + if absolute_paths: + file_list = [os.path.join(dir_name, file_name) for (dir_name, _, + file_names) in os.walk(self.datadir) for file_name in + file_names] + else: + file_list = [os.path.relpath(os.path.join(dir_name, file_name), + self.datadir) for (dir_name, _, file_names) in os.walk( + self.datadir) for file_name in file_names] + return file_list + + def default_dict_mapping(self): + # Used by ModelToDictMixin to generate dict from model + mapping = { + 'submitter': self.submitter.username, + 'preferred_event': self.preferred_event.graceid(), + 'gw_events': { + self.QS_KEY: self.get_internal_events(), + self.QS_PROP_KEY: 'graceid', + }, + 'em_events': { + self.QS_KEY: self.get_external_events(), + self.QS_PROP_KEY: 'graceid', + }, + 'labels': { + self.QS_KEY: self.labels.all(), + self.QS_PROP_KEY: 'name', + }, + } + return mapping + + # Properties -------------------------------------------------------------- + @property + def datadir(self): + """ + Mostly taken from events.models.Event.datadir + """ + # Create a file-like object which is the SHA-1 hexdigest of the + # object's primary key. We prepend 'superevent' so as to not + # have collisions with Event files + hash_input = 'superevent' + str(self.id) + hdf = StringIO(sha1(hash_input).hexdigest()) + + # Build up the nodes of the directory structure + nodes = [hdf.read(i) for i in settings.GRACEDB_DIR_DIGITS] + + # Read whatever is left over. This is the 'leaf' directory. + nodes.append(hdf.read()) + return os.path.join(settings.GRACEDB_DATA_DIR, *nodes) + + @property + def superevent_id(self): + return self.superevent_basic_id + # Really, really temporary and not good at all + # Plan: + # Convert event gpstimes to datetime fields + # store gpstime as a property + # Use datetime field to determine number for the day in question + #filter_dict = { + # 'date_created__year': self.date_created.year, + # 'date_created__month': self.date_created.month, + # 'date_created__day': self.date_created.day, + #} + #obj_set = self.__class__.objects.filter(**filter_dict).order_by('date_created') + #day_number = list(obj_set).index(self) + #suffix = int_to_letters(day_number+1) + #event_time_UTC = gpsToUtc(self.preferred_event.gpstime) + #return self.ID_PREFIX + event_time_UTC.strftime('%y%m%d') + suffix + + @property + def superevent_basic_id(self): + return self.ID_PREFIX + '{0:0>4}'.format(self.id) + + # Custom methods ---------------------------------------------------------- + def get_external_events(self): + """Returns a queryset of external events""" + return self.events.filter(group__name=settings.EXTERNAL_ANALYSIS_GROUP) + + def get_internal_events(self): + """Returns a queryset of internal events""" + return self.events.exclude(group__name= + settings.EXTERNAL_ANALYSIS_GROUP) + + #def get_by_superevent_id(self): + # pass + + def get_web_url(self): + return reverse('superevents:view', args=[self.superevent_id]) + + def get_api_url(self): + raise NotImplemented + #return reverse('') + + def __unicode__(self): + return self.superevent_id + + +class Log(CleanSaveModel, LogBase, AutoIncrementModel): + """ + Log message object attached to a Superevent. Uses the AutoIncrementModel + to handle log enumeration on a per-Superevent basis. + """ + AUTO_FIELD = 'N' + AUTO_FK = 'superevent' + superevent = models.ForeignKey(Superevent, null=False, + on_delete=models.CASCADE) + tags = models.ManyToManyField('events.Tag', related_name='superevent_logs') + + class Meta(LogBase.Meta): + unique_together = (('superevent', 'N'),) + + def get_full_file_path(self): + # TODO: add file_version? + return os.path.join(self.superevent.datadir, self.filename) + + def fileurl(self): + # TODO: implement this + super(Log, self).fileurl() + + +class Labelling(m2mThroughBase): + """ + Model which provides the 'through' relationship between Superevents and + Labels. + + We use the Label model from the events app since it's set up already and + provides exactly what we need, so no reason to create a redundant model. + """ + class Meta: + unique_together = (('superevent', 'label'),) + + superevent = models.ForeignKey(Superevent, null=False, + on_delete=models.CASCADE) + + # Labels are connected to Events and Superevents. Currently, + # Label.labelling_set points to the Labelling object which connects the + # Label to an Event. So we need a different name for this Labelling object + # which connects a Label to a Superevent. + label = models.ForeignKey('events.Label', null=False, + related_name='%(app_label)s_%(class)s_set', + on_delete=models.CASCADE) + + +class Signoff(CleanSaveModel, SignoffBase): + """Class for superevent signoffs""" + superevent = models.ForeignKey(Superevent, null=False, + on_delete=models.CASCADE) + + class Meta: + unique_together = (('superevent', 'instrument'),) + + def __unicode__(self): + return "{superevent_id} | {instrument} | {status}".format( + superevent_id=self.superevent.superevent_id, + instrument=self.instrument, status=self.status) + + +class VOEvent(CleanSaveModel, VOEventBase, AutoIncrementModel): + """VOEvent class for superevents""" + AUTO_FIELD = 'N' + AUTO_FK = 'superevent' + superevent = models.ForeignKey(Superevent, null=False, + on_delete=models.CASCADE) + + class Meta(VOEventBase.Meta): + unique_together = (('superevent', 'N'),) + + def fileurl(self): + # TODO: implement this + super(Log, self).fileurl() + + +class EMObservation(CleanSaveModel, EMObservationBase, AutoIncrementModel): + """EMObservation class for superevents""" + AUTO_FIELD = 'N' + AUTO_FK = 'superevent' + superevent = models.ForeignKey(Superevent, null=False, + on_delete=models.CASCADE) + + class Meta(EMObservationBase.Meta): + unique_together = (('superevent', 'N'),) + + def __unicode__(self): + return "{superevent_id} | {group} | {N}".format( + superevent_id=self.superevent.superevent_id, + group=self.group.name, N=self.N) + + def calculateCoveringRegion(self): + footprints = self.emfootprint_set.all() + super(EMObservation, self).calculateCoveringRegion(footprints) + + +class EMFootprint(CleanSaveModel, EMFootprintBase, AutoIncrementModel): + """EMFootprint class for superevent EMObservations""" + AUTO_FIELD = 'N' + AUTO_FK = 'observation' + observation = models.ForeignKey(EMObservation, null=False, + on_delete=models.CASCADE) + + class Meta(EMFootprintBase.Meta): + unique_together = (('observation', 'N'),) diff --git a/gracedb/superevents/urls.py b/gracedb/superevents/urls.py new file mode 100644 index 0000000000000000000000000000000000000000..399a7f0ce6c664573e18c01dedf6eb316491278a --- /dev/null +++ b/gracedb/superevents/urls.py @@ -0,0 +1,15 @@ +from django.conf.urls import url +from .models import Superevent +from . import views + +app_name = 'superevents' + +urlpatterns = [ + #url(r'^$', views.index, name="index"), + #url(r'^create/$', views.create, name="create"), + #url(r'^search/(?P<format>(json|flex))?$', views.search, name="search"), + url(r'^view/(?P<superevent_id>{pref}\d+)$'.format( + pref=Superevent.ID_PREFIX), views.webview, name="view"), + url(r'^create_log/(?P<superevent_id>{pref}\d+)$'.format( + pref=Superevent.ID_PREFIX), views.web_create_log, name="create-log"), +]