CHANGES.rst

@@ -4,6 +4,8 @@ Changelog
 2.5.2 (unreleased)
 ------------------
 
+-   Update bilby_pipe to v1.4.2.
+
 -   Enabled SSM triggers and HasSSM source property
 
 -   Rework determining the preferred external event using a keyfunc method
@@ -26,6 +28,9 @@ Changelog
     external sky map to zero. This at least prevents erroneous results when
     we receive a sky map with these invalid values.
 
+-   Roll back ligo-followup-advocate to 1.2.9 until SSM triggers are planned
+    in production.
+
 2.5.1 "Cactus cat" (2024-08-20)
 -------------------------------
 

doc/quickstart.rst

@@ -6,7 +6,7 @@ Quick start
 These instructions are suitable for installing GWCelery for development and
 testing on any machine.
 
-GWCelery requires Python >= 3.7 and a Linux or UNIX-like operating system. It
+GWCelery requires Python >= 3.10 and a Linux or UNIX-like operating system. It
 does not support Windows.
 
 To install

gwcelery/tasks/condor.py

@@ -13,9 +13,9 @@ References
 import os
 import subprocess
 import tempfile
-from distutils.dir_util import mkpath
 
 import lxml.etree
+import platformdirs
 
 from .. import app
 
@@ -35,8 +35,7 @@ def _escape_args(args):
 
 def _mklog(suffix):
     """Create a unique path for an HTCondor log."""
-    condor_dir = os.path.expanduser('~/.cache/condor')
-    mkpath(condor_dir)
+    condor_dir = platformdirs.user_cache_dir('condor', ensure_exists=True)
     with tempfile.NamedTemporaryFile(dir=condor_dir, suffix=suffix) as f:
         return f.name
 

gwcelery/tasks/em_bright.py

@@ -64,7 +64,7 @@ def plot(contents):
     properties = dict(sorted(properties.items(), reverse=True))
     probs, names = list(properties.values()), list(properties.keys())
 
-    with plt.style.context('seaborn-white'):
+    with plt.style.context('seaborn-v0_8-white'):
         fig, ax = plt.subplots(figsize=(3, 1))
         ax.barh(names, probs)
         ax.barh(names, [1.0 - p for p in probs],

gwcelery/tasks/inference.py

@@ -4,10 +4,10 @@ import json
 import os
 import subprocess
 import urllib
-from distutils.dir_util import mkpath
-from distutils.spawn import find_executable
+from shutil import which
 
 import numpy as np
+import platformdirs
 from bilby_pipe.bilbyargparser import BilbyConfigFileParser
 from bilby_pipe.utils import convert_string_to_dict
 from celery import group
@@ -124,7 +124,7 @@ def prepare_lalinference_ini(event, superevent_id):
         'webdir': os.path.join(
             app.conf['pe_results_path'], superevent_id, 'lalinference'
         ),
-        'paths': [{'name': name, 'path': find_executable(executable)}
+        'paths': [{'name': name, 'path': which(executable)}
                   for name, executable in executables.items()],
         'h1_calibration': _find_appropriate_cal_env(
             trigtime,
@@ -141,7 +141,7 @@ def prepare_lalinference_ini(event, superevent_id):
         'mc': min([sngl['mchirp'] for sngl in singleinspiraltable]),
         'q': min([sngl['mass2'] / sngl['mass1']
                   for sngl in singleinspiraltable]),
-        'mpirun': find_executable('mpirun')
+        'mpirun': which('mpirun')
     }
     return ini_template.render(ini_settings)
 
@@ -933,8 +933,7 @@ def start_pe(event, superevent_id, pe_pipeline):
 
     """
     # make an event directory
-    pipeline_dir = os.path.expanduser('~/.cache/{}'.format(pe_pipeline))
-    mkpath(pipeline_dir)
+    pipeline_dir = platformdirs.user_cache_dir(pe_pipeline, ensure_exists=True)
     event_dir = os.path.join(pipeline_dir, superevent_id)
 
     if pe_pipeline == 'bilby':
@@ -986,9 +985,8 @@ def start_pe(event, superevent_id, pe_pipeline):
         analyses = [pe_pipeline]
         condor_submit_task = condor.submit
 
-    os.mkdir(event_dir)
     for rundir, kwargs, analysis in zip(rundirs, kwargs_list, analyses):
-        mkpath(rundir)
+        os.makedirs(rundir, exist_ok=True)
 
         gracedb.upload.delay(
             filecontents=None, filename=None, graceid=superevent_id,

gwcelery/tasks/orchestrator.py

@@ -413,7 +413,7 @@ def handle_burst_event(alert):
     filename = alert['data']['filename']
 
     # Pipeline is uploading a flat resultion skymap file
-    # Converting to a multiirder ones with proper name.
+    # Converting to a multiorder ones with proper name.
     # FIXME: Remove block when CWB starts to upload skymaps
     # in multiorder format
     if filename.endswith('.fits.gz'):

gwcelery/tasks/p_astro.py

@@ -150,7 +150,7 @@ def plot(contents):
     probs, names = zip(
         *sorted(zip(classification.values(), classification.keys())))
 
-    with plt.style.context('seaborn-white'):
+    with plt.style.context('seaborn-v0_8-white'):
         fig, ax = plt.subplots(figsize=(2.5, 2))
         ax.barh(names, probs)
         for i, prob in enumerate(probs):

gwcelery/tasks/rrt_utils.py

 import json
 
+import numpy as np
 from ligo.skymap.io import read_sky_map
 from ligo.skymap.postprocess.crossmatch import crossmatch
 
@@ -32,12 +33,25 @@ def check_high_profile(skymap, em_bright,
        far_list_sorted[0]["search"] != "BBH":
         gracedb.create_label.si(
             'HIGH_PROFILE', superevent_id).delay()
+        gracedb.upload.delay(None, None, superevent_id,
+                             'Superevent labeled <font color="red">HIGH_PROFILE</font> since '
+                             'event with lowest FAR is a Burst event.',
+                              tags=['em_follow'])
         return "Event with the lowest FAR is a Burst event. Applying label"
 
     # annotation number condition
     preferred_event = superevent['preferred_event_data']
+    if preferred_event["search"] == "SSM":
+        gracedb.create_label.si(
+                'HIGH_PROFILE', superevent_id).delay()
+        gracedb.upload.delay(None, None, superevent_id,
+                             'Superevent labeled <font color="red">HIGH_PROFILE</font> since '
+                             'preferred event is from SSM search.',
+                             tags=['em_follow'])
+        return "Preferred event is from SSM. Applying label"
     if preferred_event["group"] == "CBC":
         em_bright_dict = json.loads(em_bright)
+
         has_remnant = em_bright_dict['HasRemnant']
 
         pastro_dict = json.loads(p_astro)
@@ -50,10 +64,40 @@ def check_high_profile(skymap, em_bright,
             cl = 90
             result = crossmatch(gw_skymap, contours=[cl / 100])
             sky_area = result.contour_areas[0]
-
+            is_far_away = not (gw_skymap.meta.get('distmean', np.nan) < 2000)
         if p_terr < 0.5:
-            if (p_bns > 0.1 or p_nsbh > 0.1 or has_remnant > 0.1 or sky_area < 100):  # noqa: E501
+            if p_bns > 0.1:
+                gracedb.create_label.si(
+                    'HIGH_PROFILE', superevent_id).delay()
+                gracedb.upload.delay(None, None, superevent_id,
+                                     'Superevent labeled <font color="red">HIGH_PROFILE</font> since '
+                                     'because p_BNS > 10%.',
+                                      tags=['em_follow'])
+                return "p_BNS condition satisfied. Applying label"
+            elif p_nsbh > 0.1:
+                gracedb.create_label.si(
+                    'HIGH_PROFILE', superevent_id).delay()
+                gracedb.upload.delay(None, None, superevent_id,
+                                     'Superevent labeled <font color="red">HIGH_PROFILE</font> since '
+                                     'because p_NSBH> 10%.',
+                                      tags=['em_follow'])
+                return "p_NSBH condition satisfied. Applying label"
+            elif has_remnant > 0.1:
+                gracedb.create_label.si(
+                    'HIGH_PROFILE', superevent_id).delay()
+                gracedb.upload.delay(None, None, superevent_id,
+                                     'Superevent labeled <font color="red">HIGH_PROFILE</font> since '
+                                     'because p_HasRemnant> 10%.',
+                                      tags=['em_follow'])
+                return "p_HasRemnant condition satisfied. Applying label"
+            elif (sky_area < 100 and not is_far_away):
                 gracedb.create_label.si(
                     'HIGH_PROFILE', superevent_id).delay()
-                return "Annotations condition satisfied. Applying label"
+                gracedb.upload.delay(None, None, superevent_id,
+                                     'Superevent labeled <font color="red">HIGH_PROFILE</font> since '
+                                     'because area of 90% confidence level '
+                                     'in the skymap is < 100 sq.deg and mean '
+                                     'posterior distance is < 2 Gpc. ',
+                                      tags=['em_follow'])
+                return "Skymap & distance condition satisfied. Applying label"
     return "No conditions satisfied. Skipping"

gwcelery/templates/index.jinja2

@@ -647,7 +647,7 @@
     /* Enable typeahead completion for superevent ID fields. */
     $("[name='superevent_id']").typeahead({
         highlight: true,
-        minLength: 0
+        minLength: 1
     }, {
         source: new Bloodhound({
             queryTokenizer: Bloodhound.tokenizers.whitespace,
@@ -674,7 +674,7 @@
 
         $(this).typeahead({
             highlight: true,
-            minLength: 0
+            minLength: 1
         }, {
             source: bloodhound,
             display: 'graceid',
@@ -726,7 +726,7 @@
 
         $(this).typeahead({
             highlight: true,
-            minLength: 0
+            minLength: 1
         }, {
             source: bloodhound
         });
@@ -759,7 +759,7 @@
 
         $(this).typeahead({
             highlight: true,
-            minLength: 0
+            minLength: 1
         }, {
             source: bloodhound
         });

gwcelery/tests/test_tasks_inference.py

@@ -632,8 +632,8 @@ def test_start_pe(monkeypatch, tmp_path, pipeline):
             submit_rapidpe = Mock(side_effect=mock_condor_submit)
             dag_finished = Mock()
             monkeypatch.setattr('gwcelery.tasks.gracedb.upload.run', Mock())
-            monkeypatch.setattr('os.path.expanduser', Mock(
-                return_value=str(tmp_path)))
+            monkeypatch.setattr('platformdirs.user_cache_dir',
+                                Mock(return_value=str(tmp_path)))
             monkeypatch.setattr('gwcelery.tasks.inference.dag_prepare_task',
                                 dag_prepare_task)
             monkeypatch.setattr(
@@ -660,7 +660,7 @@ def test_start_pe(monkeypatch, tmp_path, pipeline):
                              name="condor_submit")
         dag_finished = Mock(name="dag_finished")
         monkeypatch.setattr('gwcelery.tasks.gracedb.upload.run', Mock())
-        monkeypatch.setattr('distutils.dir_util.mkpath',
+        monkeypatch.setattr('platformdirs.user_cache_dir',
                             Mock(return_value=str(tmp_path)))
         monkeypatch.setattr('gwcelery.tasks.inference.dag_prepare_task',
                             dag_prepare_task)
@@ -679,7 +679,8 @@ def test_start_pe(monkeypatch, tmp_path, pipeline):
     'host', ["gracedb-playground.ligo.org", "gracedb.ligo.org"])
 def test_bbh_rate_limit(monkeypatch, tmp_path, host):
     monkeypatch.setitem(app.conf, 'gracedb_host', host)
-    monkeypatch.setattr('os.path.expanduser', Mock(return_value=str(tmp_path)))
+    monkeypatch.setattr(
+        'platformdirs.user_cache_dir', Mock(return_value=str(tmp_path)))
     event = {'gpstime': 1187008882, 'graceid': 'G1234',
              'extra_attributes': {'CoincInspiral': {'mchirp': 20}}}
     for i in range(6):
@@ -707,7 +708,7 @@ def test_bbh_rate_limit(monkeypatch, tmp_path, host):
     condor_submit = Mock(side_effect=mock_condor_submit)
     dag_finished = Mock()
     monkeypatch.setattr('gwcelery.tasks.gracedb.upload.run', Mock())
-    monkeypatch.setattr('distutils.dir_util.mkpath',
+    monkeypatch.setattr('platformdirs.user_cache_dir',
                         Mock(return_value=str(tmp_path)))
     monkeypatch.setattr('gwcelery.tasks.inference.dag_prepare_task',
                         dag_prepare_task)

gwcelery/tests/test_tasks_rrt_utils.py

@@ -16,11 +16,12 @@ skymap_large = read_binary(data, 'MS220722v_bayestar.multiorder.fits')
 # 3. p_BNS check        ---> True
 # 4. p_NSBH check       ---> True
 # 5. skymap check       ---> True
-# 6. Simple BBH         ---> False
-# 7. Burst but low far  ---> False
-# 8. High p-terrestrial ---> False
-# 9. Burst BBH low far ---> False
-# 10. HIGH_PROFILE_applied-> False
+# 6. SSM preferred      ---> True
+# 7. Simple BBH         ---> False
+# 8. Burst but low far  ---> False
+# 9. High p-terrestrial ---> False
+# 10. Burst BBH low far ---> False
+# 11. HIGH_PROFILE_applied-> False
 
 
 def get_event(graceid):
@@ -60,6 +61,15 @@ def get_event(graceid):
             'far': 1e-10,
             'gpstime': 1234,
         }
+    elif graceid == 'G468':
+        event = {
+            'group': 'CBC',
+            'pipeline': 'gstlal',
+            'search': 'SSM',
+            'graceid': graceid,
+            'far': 1e-10,
+            'gpstime': 1234,
+        }
     return event
 
 
@@ -85,6 +95,10 @@ def get_event(graceid):
           {'HasNS': 0.0, 'HasRemnant': 0.0, 'HasMassGap': 0.0},
           {"BNS": 0.0, "NSBH": 0.0, "BBH": 0.9, "Terrestrial": 0.1},
           skymap_small, True],
+         [{'labels': [], 'gw_events': ['G123', 'G468']},
+          {'HasNS': 0.0, 'HasSSM': 1.0, 'HasMassGap': 0.0},
+          {"BNS": 0.0, "NSBH": 0.0, "BBH": 0.9, "Terrestrial": 0.1},
+          skymap_large, True],
          [{'labels': [], 'gw_events': ['G123']},
           {'HasNS': 0.0, 'HasRemnant': 0.0, 'HasMassGap': 0.0},
           {"BNS": 0.0, "NSBH": 0.0, "BBH": 1.0, "Terrestrial": 0.0},
@@ -110,8 +124,12 @@ def test_high_profile(monkeypatch, superevent, embright,
                       pastro, skymap, result):
     embright = json.dumps(embright)
     pastro = json.dumps(pastro)
-    superevent.update({'superevent_id': 'S123',
-                       'preferred_event_data': get_event('G123')})
+    if 'G468' in superevent['gw_events']:
+        superevent.update({'superevent_id': 'S123',
+                          'preferred_event_data': get_event('G468')})
+    else:
+        superevent.update({'superevent_id': 'S123',
+                           'preferred_event_data': get_event('G123')})
     mock_create_label = Mock()
     monkeypatch.setattr('gwcelery.tasks.gracedb.create_label.run',
                         mock_create_label)

gwcelery/tests/test_tools_nagios.py

-from distutils.spawn import find_executable
 from pathlib import Path
+from shutil import which
 from unittest.mock import Mock
 
 import lal
@@ -87,7 +87,7 @@ def test_nagios(capsys, monkeypatch, request, socket_enabled, starter,
 
     # broker, no worker
 
-    redis_server = find_executable('redis-server')
+    redis_server = which('redis-server')
     if redis_server is None:
         pytest.skip('redis-server is not installed')
     starter.exec_process(

pyproject.toml

@@ -54,7 +54,7 @@ python = "^3.10,<3.12"
 adc-streaming = ">=2.3.0"  # https://github.com/astronomy-commons/adc-streaming/pull/62
 astropy = ">=4.3.1,!=5.3"  # https://github.com/astropy/astropy/issues/11879, https://github.com/gwpy/gwpy/issues/1629
 bilby = ">=2.2.2"
-bilby_pipe = ">=1.3.2"  # https://git.ligo.org/lscsoft/bilby_pipe/-/merge_requests/602
+bilby_pipe = ">=1.4.2"  # https://git.ligo.org/lscsoft/bilby_pipe/-/merge_requests/615
 celery = {version = ">=5.1, <5.4", extras = ["redis"]}  # https://git.ligo.org/emfollow/gwcelery/-/issues/804
 ciecplib = {version = ">=0.8.1", extras = ["kerberos"]}  # for renew-cert.sh, https://git.ligo.org/computing/software/ciecplib/-/issues/145
 click = ">=7"
@@ -62,6 +62,7 @@ comet = "*"
 confluent-kafka = ">=1.9.2"
 flask = ">=2.2,<3.0"  # https://github.com/pytest-dev/pytest-flask/issues/167
 flask-caching = "*"
+flower = ">=2.0.1"  # https://git.ligo.org/emfollow/gwcelery/-/issues/688
 gracedb-sdk = ">=0.2.0"  # https://git.ligo.org/emfollow/gracedb-sdk/-/merge_requests/7
 gwdatafind = ">=1.1.1"
 gwpy = ">=3.0.8"  # https://git.ligo.org/emfollow/gwcelery/-/issues/183
@@ -72,21 +73,22 @@ igwn-alert = ">=0.2.2"
 igwn-gwalert-schema = "^1.0.0"
 imapclient = "*"
 importlib-metadata = { version = "*"}
+ipython = "*"  # Include IPython for a nicer ``gwcelery shell`` experience.
 jinja2 = ">=2.11.2"  # https://github.com/pallets/jinja/issues/1168
 lalsuite = ">=7.16"  # https://git.ligo.org/lscsoft/lalsuite/-/merge_requests/2120
-ligo-followup-advocate = ">=1.2.10"  # https://git.ligo.org/emfollow/gwcelery/-/issues/827
+ligo-followup-advocate = "1.2.9"  # https://git.ligo.org/emfollow/gwcelery/-/issues/827
 ligo-gracedb = ">=2.7.5"  # https://git.ligo.org/lscsoft/gracedb-client/-/issues/28
 ligo-raven = ">=3.2,<4.0"
 ligo-segments = "*"
 "ligo.em-bright" = ">=1.2.1"  # https://git.ligo.org/emfollow/gwcelery/-/issues/816
-"ligo.skymap" = ">=2.0.1"  # https://git.ligo.org/lscsoft/ligo.skymap/-/merge_requests/395
+"ligo.skymap" = "==2.0.1"  # https://git.ligo.org/emfollow/gwcelery/-/issues/870
 lscsoft-glue = "*"
 lxml = "*"
-matplotlib = "<3.7"  # Matplotlib changed an axes behaviour which breaks some of our plotting scripts. When gwpy has a new release, we can unpin this.
 numba = ">=0.56"  # Poetry update chooses an old version of numba and its deps that break the python3.9 and 3.10 build tests if this is not specified; dependence on numba comes from rift. Version chosen because it adds python 3.10 support. This requirement can be dropped here if RIFT adds it https://git.ligo.org/rapidpe-rift/rift/-/issues/24
-numpy = "*"
+numpy = "<2"  # FIXME: TensorFlow (a dependency of GWSkyNet) does not support Numpy 2.x yet. Remove once https://git.ligo.org/computing/sccb/-/issues/1601#note_1120002 is resolved.
 p_astro = ">=1.0.1"  # https://git.ligo.org/lscsoft/p-astro/-/merge_requests/40
 pesummary = ">=1.0.2"  # https://git.ligo.org/computing/sccb/-/issues/1350
+platformdirs = "*"
 pygcn = ">=1.0.1"
 python-ligo-lw = "^1.8.3"
 pyxdg = "*"
@@ -121,9 +123,7 @@ doc = ["pep517", "sphinx"]
 test = ["fastavro", "pytest-celery", "pytest-cov", "pytest-flask", "pytest-socket"]
 
 [tool.poetry.group.dev.dependencies]
-ipython = "*"  # Include IPython for a nicer ``gwcelery shell`` experience.
 ipykernel = "*"  # Include IPython kernel for Jupyter Lab support.
-flower = ">=2.0.1"  # https://git.ligo.org/emfollow/gwcelery/-/issues/688
 flake8 = "^7.0.0"
 isort = "^5.13.2"