Generate flat-resolution FITS files asynchronously

Report the URL of multi-resolution FITS files in GCN Notices.

CHANGES.rst

@@ -19,6 +19,8 @@ Changelog
     Add acceptance tests for the RAVEN alert system. Add options to use
     the superevent search field in coincident searches. Update preferred
     external event based on best available.
+-   Report the URL of multi-resolution FITS files in GCN Notices and create
+    flat-resolution files asynchronously.
 
 1.0.1 (2022-05-09)
 ------------------

gwcelery/tasks/orchestrator.py

@@ -295,6 +295,8 @@ def handle_posterior_samples(alert):
     filename = alert['data']['filename']
     info = '{} {}'.format(alert['data']['comment'], filename)
     prefix, _ = filename.rsplit('.posterior_samples.')
+    skymap_filename = f'{prefix}.multiorder.fits'
+    labels = ['pe', 'sky_loc', 'public']
 
     (
         gracedb.download.si(filename, superevent_id)
@@ -303,22 +305,13 @@ def handle_posterior_samples(alert):
         |
         group(
             skymaps.annotate_fits.s(
-                '{}.fits.gz'.format(prefix),
-                superevent_id, ['pe', 'sky_loc', 'public']
+                skymap_filename, superevent_id, labels
             ),
 
             gracedb.upload.s(
-                '{}.multiorder.fits'.format(prefix), superevent_id,
+                skymap_filename, superevent_id,
                 'Multiresolution fits file generated from "{}"'.format(info),
-                ['pe', 'sky_loc', 'public']
-            ),
-
-            skymaps.flatten.s('{}.fits.gz'.format(prefix))
-            |
-            gracedb.upload.s(
-                '{}.fits.gz'.format(prefix), superevent_id,
-                'Flat-resolution fits file created from "{}"'.format(info),
-                ['pe', 'sky_loc', 'public']
+                labels
             )
         )
     ).delay()
@@ -407,7 +400,8 @@ def _create_voevent(classification, *args, **kwargs):
 
     skymap_filename = kwargs.get('skymap_filename')
     if skymap_filename is not None:
-        skymap_type = re.sub(r'\.fits(\..+)?$', '', skymap_filename)
+        skymap_type = re.sub(
+            r'(\.multiorder)?\.fits(\..+)?(,[0-9]+)?$', '', skymap_filename)
         kwargs.setdefault('skymap_type', skymap_type)
 
     # FIXME: remove ._orig_run when this bug is fixed:
@@ -491,12 +485,6 @@ def preliminary_alert(event, alert, annotation_prefix='',
     else:
         skymap_filename = None
 
-    original_skymap_filename = skymap_filename
-    if skymap_filename.endswith('.multiorder.fits'):
-        skymap_filename = skymap_filename.replace('.multiorder.fits', '.fits')
-    if skymap_filename.endswith('.fits'):
-        skymap_filename += '.gz'
-
     # Determine if the event should be made public.
     is_publishable = (superevents.should_publish(
                       alert['object']['preferred_event_data']) and
@@ -505,16 +493,14 @@ def preliminary_alert(event, alert, annotation_prefix='',
 
     canvas = group(
         (
-            gracedb.download.si(original_skymap_filename, preferred_event_id)
+            gracedb.download.si(skymap_filename, preferred_event_id)
             |
             group(
-                skymaps.flatten.s(annotation_prefix + skymap_filename)
-                |
                 gracedb.upload.s(
                     annotation_prefix + skymap_filename,
                     superevent_id,
-                    message='Flattened from multiresolution file {}'.format(
-                        original_skymap_filename),
+                    message='Localization copied from {}'.format(
+                        preferred_event_id),
                     tags=['sky_loc'] if annotation_prefix else [
                         'sky_loc', 'public']
                 )
@@ -523,15 +509,6 @@ def preliminary_alert(event, alert, annotation_prefix='',
                     'SKYMAP_READY', superevent_id
                 ),
 
-                gracedb.upload.s(
-                    annotation_prefix + original_skymap_filename,
-                    superevent_id,
-                    message='Localization copied from {}'.format(
-                        preferred_event_id),
-                    tags=['sky_loc'] if annotation_prefix else [
-                        'sky_loc', 'public']
-                ),
-
                 skymaps.annotate_fits.s(
                     annotation_prefix + skymap_filename,
                     superevent_id,
@@ -691,7 +668,7 @@ def preliminary_initial_update_alert(filenames, superevent, alert_type):
                 continue
             if skymap_needed \
                     and {'sky_loc', 'public'}.issubset(t) \
-                    and f.endswith('.fits.gz'):
+                    and f.endswith('.multiorder.fits'):
                 skymap_filename = fv
             if em_bright_needed \
                     and 'em_bright' in t \

gwcelery/tasks/skymaps.py

@@ -30,7 +30,18 @@ def annotate_fits(filecontents, versioned_filename, graceid, tags):
     This function downloads a FITS file and then generates and uploads all
     derived images as well as an HTML dump of the FITS header.
     """
-    filebase = versioned_filename.partition('.fits')[0]
+    multiorder_extension = '.multiorder.fits'
+    flat_extension = '.fits'
+
+    if multiorder_extension in versioned_filename:
+        extension = multiorder_extension
+        multiorder = True
+    else:
+        extension = flat_extension
+        multiorder = False
+
+    filebase, _, _ = versioned_filename.partition(extension)
+
     header_msg = (
         'FITS headers for <a href="/api/superevents/{graceid}/files/'
         '{versioned_filename}">{versioned_filename}</a>').format(
@@ -43,18 +54,34 @@ def annotate_fits(filecontents, versioned_filename, graceid, tags):
         'Volume rendering of <a href="/api/superevents/{graceid}/files/'
         '{versioned_filename}">{versioned_filename}</a>').format(
             graceid=graceid, versioned_filename=versioned_filename)
+    flatten_msg = (
+        'Flat-resolution fits file created from '
+        '<a href="/api/superevents/{graceid}/files/'
+        '{versioned_filename}">{versioned_filename}</a>').format(
+            graceid=graceid, versioned_filename=versioned_filename)
 
     group(
-        fits_header.s(versioned_filename) |
+        fits_header.s(versioned_filename)
+        |
         gracedb.upload.s(
             filebase + '.html', graceid, header_msg, tags),
 
-        plot_allsky.s() |
+        plot_allsky.s()
+        |
         gracedb.upload.s(
             filebase + '.png', graceid, allsky_msg, tags),
 
         annotate_fits_volume.s(
-            filebase + '.volume.png', graceid, volume_msg, tags)
+            filebase + '.volume.png', graceid, volume_msg, tags),
+
+        *(
+            [
+                flatten.s(f'{filebase}.fits.gz')
+                |
+                gracedb.upload.s(
+                    f'{filebase}.fits.gz', graceid, flatten_msg, tags)
+            ] if multiorder else []
+        )
     ).delay(filecontents)
 
 

gwcelery/tests/test_tasks_orchestrator.py

@@ -230,7 +230,7 @@ def superevent_initial_alert_download(filename, graceid):
 @patch('gwcelery.tasks.gracedb.expose._orig_run', return_value=None)
 @patch('gwcelery.tasks.gracedb.get_log',
        return_value=[{'tag_names': ['sky_loc', 'public'],
-                      'filename': 'foobar.fits.gz',
+                      'filename': 'foobar.multiorder.fits',
                       'file_version': 0},
                      {'tag_names': ['em_bright'],
                       'filename': 'em_bright.json',
@@ -269,7 +269,7 @@ def test_handle_superevent_initial_alert(mock_create_initial_circular,
     mock_create_voevent.assert_called_once_with(
         'S1234', 'initial', BBH=0.02, BNS=0.94, NSBH=0.03, ProbHasNS=0.0,
         ProbHasRemnant=0.0, Terrestrial=0.01, internal=False, open_alert=True,
-        skymap_filename='foobar.fits.gz,0', skymap_type='foobar',
+        skymap_filename='foobar.multiorder.fits,0', skymap_type='foobar',
         raven_coinc='RAVEN_ALERT' in labels)
     mock_send.assert_called_once_with('contents of S1234-Initial-1.xml')
     if 'RAVEN_ALERT' in labels:
@@ -277,7 +277,7 @@ def test_handle_superevent_initial_alert(mock_create_initial_circular,
     else:
         mock_create_initial_circular.assert_called_once_with('S1234')
     mock_create_tag.assert_has_calls(
-        [call('foobar.fits.gz,0', 'public', 'S1234'),
+        [call('foobar.multiorder.fits,0', 'public', 'S1234'),
          call('em_bright.json,0', 'public', 'S1234'),
          call('p_astro.json,0', 'public', 'S1234'),
          call('S1234-Initial-1.xml', 'public', 'S1234')],

gwcelery/views.py

@@ -161,7 +161,7 @@ def typeahead_skymap_filename():
     return jsonify(_search_by_tag_and_filename(
         request.args.get('superevent_id') or '',
         request.args.get('filename') or '',
-        '.fits.gz', 'sky_loc'
+        '.multiorder.fits', 'sky_loc'
     ))