From e87867559649a7a298af766de2c81811f297cff3 Mon Sep 17 00:00:00 2001 From: Brandon Piotrzkowski Date: Mon, 9 May 2022 15:32:47 -0400 Subject: [PATCH 1/4] Enable skymaps in RAVEN alerts and use logs to remove race condition; fixes #413 --- gwcelery/tasks/external_skymaps.py | 2 +- gwcelery/tasks/orchestrator.py | 15 +++- gwcelery/tests/test_tasks_orchestrator.py | 90 +++++++++++++++++++---- 3 files changed, 89 insertions(+), 18 deletions(-) diff --git a/gwcelery/tasks/external_skymaps.py b/gwcelery/tasks/external_skymaps.py index 92a063b1..a9555252 100644 --- a/gwcelery/tasks/external_skymaps.py +++ b/gwcelery/tasks/external_skymaps.py @@ -47,7 +47,7 @@ def create_combined_skymap(se_id, ext_id): | group( gracedb.upload.s(new_skymap_filename + '-ext.fits.gz', se_id, - message, ['sky_loc', 'public']), + message, ['sky_loc', 'ext_coinc', 'public']), skymaps.plot_allsky.s() | diff --git a/gwcelery/tasks/orchestrator.py b/gwcelery/tasks/orchestrator.py index 2cf0c731..20f9b38c 100644 --- a/gwcelery/tasks/orchestrator.py +++ b/gwcelery/tasks/orchestrator.py @@ -664,6 +664,7 @@ def preliminary_initial_update_alert(filenames, superevent_id, alert_type, return skymap_filename, em_bright_filename, p_astro_filename = filenames + combined_skymap_filename, raven_alert = None, False skymap_needed = (skymap_filename is None) em_bright_needed = (em_bright_filename is None) p_astro_needed = (p_astro_filename is None) @@ -672,12 +673,16 @@ def preliminary_initial_update_alert(filenames, superevent_id, alert_type, t = message['tag_names'] f = message['filename'] v = message['file_version'] + c = message['comment'] fv = '{},{}'.format(f, v) + if 'RAVEN_ALERT' in c: + raven_alert = True if not f: continue if skymap_needed \ and {'sky_loc', 'public'}.issubset(t) \ - and f.endswith('.fits.gz'): + and f.endswith('.fits.gz') \ + and '-ext.fits' not in f: skymap_filename = fv if em_bright_needed \ and 'em_bright' in t \ @@ -687,9 +692,12 @@ def preliminary_initial_update_alert(filenames, superevent_id, alert_type, and 'p_astro' in t \ and f.endswith('.json'): p_astro_filename = fv + if '-ext.fits' in f \ + and {'sky_loc', 'ext_coinc', 'public'}.issubset(t): + combined_skymap_filename = fv if alert_type in {'earlywarning', 'preliminary', 'initial'}: - if 'RAVEN_ALERT' in labels: + if raven_alert: circular_task = circulars.create_emcoinc_circular.si(superevent_id) circular_filename = '{}-emcoinc-circular.txt'.format(alert_type) tags = ['em_follow', 'ext_coinc'] @@ -731,7 +739,8 @@ def preliminary_initial_update_alert(filenames, superevent_id, alert_type, skymap_filename=skymap_filename, internal=False, open_alert=True, - raven_coinc=('RAVEN_ALERT' in labels) + raven_coinc=raven_alert, + combined_skymap_filename=combined_skymap_filename ) | group( diff --git a/gwcelery/tests/test_tasks_orchestrator.py b/gwcelery/tests/test_tasks_orchestrator.py index 450e3955..b3e6b4bd 100644 --- a/gwcelery/tests/test_tasks_orchestrator.py +++ b/gwcelery/tests/test_tasks_orchestrator.py @@ -223,34 +223,99 @@ def superevent_initial_alert_download(filename, graceid): raise ValueError -@pytest.mark.parametrize( # noqa: F811 - 'labels', - [[], ['EM_COINC', 'RAVEN_ALERT']]) @patch('gwcelery.tasks.gracedb.expose._orig_run', return_value=None) @patch('gwcelery.tasks.gracedb.get_log', return_value=[{'tag_names': ['sky_loc', 'public'], 'filename': 'foobar.fits.gz', - 'file_version': 0}, + 'file_version': 0, + 'comment': ''}, {'tag_names': ['em_bright'], 'filename': 'em_bright.json', - 'file_version': 0}, + 'file_version': 0, + 'comment': ''}, {'tag_names': ['p_astro'], 'filename': 'p_astro.json', - 'file_version': 0}]) + 'file_version': 0, + 'comment': ''}]) @patch('gwcelery.tasks.gracedb.create_tag._orig_run', return_value=None) @patch('gwcelery.tasks.gracedb.create_voevent._orig_run', return_value='S1234-Initial-1.xml') @patch('gwcelery.tasks.gracedb.download._orig_run', superevent_initial_alert_download) @patch('gwcelery.tasks.gcn.send.run') -@patch('gwcelery.tasks.circulars.create_emcoinc_circular.run') @patch('gwcelery.tasks.circulars.create_initial_circular.run') def test_handle_superevent_initial_alert(mock_create_initial_circular, - mock_create_emcoinc_circular, mock_send, mock_create_voevent, mock_create_tag, mock_get_log, - mock_expose, labels): + mock_expose): + """Test that the ``ADVOK`` label triggers an initial alert.""" + alert = { + 'alert_type': 'label_added', + 'uid': 'S1234', + 'data': {'name': 'ADVOK'}, + 'object': {'labels': []} + } + + # Run function under test + orchestrator.handle_superevent(alert) + + mock_create_voevent.assert_called_once_with( + 'S1234', 'initial', BBH=0.02, BNS=0.94, NSBH=0.03, ProbHasNS=0.0, + ProbHasRemnant=0.0, Terrestrial=0.01, internal=False, open_alert=True, + skymap_filename='foobar.fits.gz,0', skymap_type='foobar', + raven_coinc=False, combined_skymap_filename=None) + mock_send.assert_called_once_with('contents of S1234-Initial-1.xml') + mock_create_initial_circular.assert_called_once_with('S1234') + mock_create_tag.assert_has_calls( + [call('foobar.fits.gz,0', 'public', 'S1234'), + call('em_bright.json,0', 'public', 'S1234'), + call('p_astro.json,0', 'public', 'S1234'), + call('S1234-Initial-1.xml', 'public', 'S1234')], + any_order=True) + mock_expose.assert_called_once_with('S1234') + + +@pytest.mark.parametrize( # noqa: F811 + 'labels', + [[], ['EM_COINC', 'RAVEN_ALERT']]) +@patch('gwcelery.tasks.gracedb.expose._orig_run', return_value=None) +@patch('gwcelery.tasks.gracedb.get_log', + return_value=[{'tag_names': ['sky_loc', 'public'], + 'filename': 'foobar.fits.gz', + 'file_version': 0, + 'comment': ''}, + {'tag_names': ['em_bright'], + 'filename': 'em_bright.json', + 'file_version': 0, + 'comment': ''}, + {'tag_names': ['p_astro'], + 'filename': 'p_astro.json', + 'file_version': 0, + 'comment': ''}, + {'tag_names': [], + 'filename': '', + 'file_version': None, + 'comment': 'Added label: RAVEN_ALERT'}, + {'tag_names': ['sky_loc', 'ext_coinc', 'public'], + 'filename': 'bayestar-ext.fits.gz', + 'file_version': 0, + 'comment': ''}]) +@patch('gwcelery.tasks.gracedb.create_tag._orig_run', return_value=None) +@patch('gwcelery.tasks.gracedb.create_voevent._orig_run', + return_value='S1234-Initial-1.xml') +@patch('gwcelery.tasks.gracedb.download._orig_run', + superevent_initial_alert_download) +@patch('gwcelery.tasks.gcn.send.run') +@patch('gwcelery.tasks.circulars.create_emcoinc_circular.run') +@patch('gwcelery.tasks.circulars.create_initial_circular.run') +def test_handle_superevent_emcoinc_initial_alert(mock_create_initial_circular, + mock_create_emcoinc_circular, + mock_send, + mock_create_voevent, + mock_create_tag, + mock_get_log, + mock_expose, labels): """Test that the ``ADVOK`` label triggers an initial alert.""" alert = { 'alert_type': 'label_added', @@ -266,12 +331,9 @@ def test_handle_superevent_initial_alert(mock_create_initial_circular, 'S1234', 'initial', BBH=0.02, BNS=0.94, NSBH=0.03, ProbHasNS=0.0, ProbHasRemnant=0.0, Terrestrial=0.01, internal=False, open_alert=True, skymap_filename='foobar.fits.gz,0', skymap_type='foobar', - raven_coinc='RAVEN_ALERT' in labels) + raven_coinc=True, combined_skymap_filename='bayestar-ext.fits.gz,0') mock_send.assert_called_once_with('contents of S1234-Initial-1.xml') - if 'RAVEN_ALERT' in labels: - mock_create_emcoinc_circular.assert_called_once_with('S1234') - else: - mock_create_initial_circular.assert_called_once_with('S1234') + mock_create_emcoinc_circular.assert_called_once_with('S1234') mock_create_tag.assert_has_calls( [call('foobar.fits.gz,0', 'public', 'S1234'), call('em_bright.json,0', 'public', 'S1234'), -- GitLab From 2a18501c88597807e735770ec3079652fa2e931e Mon Sep 17 00:00:00 2001 From: Brandon Piotrzkowski Date: Fri, 8 Jul 2022 14:31:42 -0400 Subject: [PATCH 2/4] sdfkj --- gwcelery/tasks/orchestrator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gwcelery/tasks/orchestrator.py b/gwcelery/tasks/orchestrator.py index 20f9b38c..fdabbc29 100644 --- a/gwcelery/tasks/orchestrator.py +++ b/gwcelery/tasks/orchestrator.py @@ -668,7 +668,9 @@ def preliminary_initial_update_alert(filenames, superevent_id, alert_type, skymap_needed = (skymap_filename is None) em_bright_needed = (em_bright_filename is None) p_astro_needed = (p_astro_filename is None) - if skymap_needed or em_bright_needed or p_astro_needed: + combined_skymap_needed = (combined_skymap_filename is None) + if skymap_needed or em_bright_needed or p_astro_needed \ + or combined_skymap_needed: for message in gracedb.get_log(superevent_id): t = message['tag_names'] f = message['filename'] -- GitLab From 577e60a4777461d2e5031c83df7c3d87cea5f0a6 Mon Sep 17 00:00:00 2001 From: Brandon Piotrzkowski Date: Mon, 25 Jul 2022 17:26:25 -0400 Subject: [PATCH 3/4] Create combined sky map for every coincidence --- gwcelery/tasks/external_triggers.py | 6 ++---- .../tests/test_tasks_external_triggers.py | 21 ++++--------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/gwcelery/tasks/external_triggers.py b/gwcelery/tasks/external_triggers.py index dd588600..4b58fa89 100644 --- a/gwcelery/tasks/external_triggers.py +++ b/gwcelery/tasks/external_triggers.py @@ -265,10 +265,8 @@ def handle_grb_igwn_alert(alert): [alert['object']['search']]) raven.raven_pipeline([alert['object']], se_id, superevent, tl, th, gw_group) - if _skymaps_are_ready(alert['object'], alert['data']['name'], - 'combine'): - # if both sky maps present and a raven alert, create combined - # skymap + + # also if both sky maps present, create combined skymap se_id, ext_id = _get_superevent_ext_ids(graceid, alert['object'], 'combine') external_skymaps.create_combined_skymap(se_id, ext_id) diff --git a/gwcelery/tests/test_tasks_external_triggers.py b/gwcelery/tests/test_tasks_external_triggers.py index 72b4066a..3daf3f41 100644 --- a/gwcelery/tests/test_tasks_external_triggers.py +++ b/gwcelery/tests/test_tasks_external_triggers.py @@ -237,8 +237,10 @@ def test_handle_create_skymap_label_from_superevent(mock_create_label, 'graceid': 'G1234', 'group': 'CBC' }) -def test_handle_skymap_comparison(mock_get_event, mock_get_superevent, - mock_raven_pipeline, mock_get_group): +@patch('gwcelery.tasks.external_skymaps.create_combined_skymap') +def test_handle_skymaps_ready(mock_create_combined_skymap, + mock_get_event, mock_get_superevent, + mock_raven_pipeline, mock_get_group): alert = {"uid": "E1212", "alert_type": "label_added", "data": {"name": "SKYMAP_READY"}, @@ -256,21 +258,6 @@ def test_handle_skymap_comparison(mock_get_event, mock_get_superevent, {'superevent_id': 'S1234', 'preferred_event': 'G1234'}, -5, 1, 'CBC') - - -@patch('gwcelery.tasks.external_skymaps.create_combined_skymap') -def test_handle_skymap_combine(mock_create_combined_skymap): - alert = {"uid": "E1212", - "alert_type": "label_added", - "data": {"name": "RAVEN_ALERT"}, - "object": { - "graceid": "E1212", - "group": "External", - "labels": ["EM_COINC", "EXT_SKYMAP_READY", "SKYMAP_READY", - "RAVEN_ALERT"], - "superevent": "S1234"} - } - external_triggers.handle_grb_igwn_alert(alert) mock_create_combined_skymap.assert_called_once_with('S1234', 'E1212') -- GitLab From 325f0d5891870a0cc003ce90008b3f34deb09388 Mon Sep 17 00:00:00 2001 From: Brandon Piotrzkowski Date: Mon, 25 Jul 2022 17:40:45 -0400 Subject: [PATCH 4/4] Upload combined sky map to external event --- gwcelery/tasks/external_skymaps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gwcelery/tasks/external_skymaps.py b/gwcelery/tasks/external_skymaps.py index a9555252..78243b4f 100644 --- a/gwcelery/tasks/external_skymaps.py +++ b/gwcelery/tasks/external_skymaps.py @@ -46,12 +46,12 @@ def create_combined_skymap(se_id, ext_id): combine_skymaps.si(se_skymap, ext_skymap) | group( - gracedb.upload.s(new_skymap_filename + '-ext.fits.gz', se_id, + gracedb.upload.s(new_skymap_filename + '-ext.fits.gz', ext_id, message, ['sky_loc', 'ext_coinc', 'public']), skymaps.plot_allsky.s() | - gracedb.upload.s(new_skymap_filename + '-ext.png', se_id, + gracedb.upload.s(new_skymap_filename + '-ext.png', ext_id, message_png, ['sky_loc', 'ext_coinc', 'public']) ) ).delay() -- GitLab