CI: Run calibration pipelines in parallel using multiple threads

tests/test_common.py

@@ -29,8 +29,7 @@ import sys
 
 import gi
 gi.require_version('Gst', '1.0')
-from gi.repository import GObject
-from gi.repository import Gst
+from gi.repository import GLib, Gst
 
 
 from gstlal import pipeparts
@@ -38,11 +37,8 @@ from gstlal import pipeio
 from gstlal import simplehandler
 from gstlal import datasource
 
-
-GObject.threads_init()
 Gst.init(None)
 
-
 if sys.byteorder == "little":
 	BYTE_ORDER = "LE"
 else:
@@ -130,7 +126,7 @@ def gapped_complex_test_src(pipeline, buffer_length = 1.0, rate = 2048, width =
 
 def build_and_run(pipelinefunc, name, segment = None, **pipelinefunc_kwargs):
 	print("=== Running Test %s ===" % name)
-	mainloop = GObject.MainLoop()
+	mainloop = GLib.MainLoop.new(None, True)
 	pipeline = pipelinefunc(Gst.Pipeline(name = name), name, **pipelinefunc_kwargs)
 	handler = simplehandler.Handler(mainloop, pipeline)
 	if segment is not None:

tests/tests_pytest/latency.py

@@ -16,9 +16,9 @@ def Latency():
     
     i_lst = [list(i_gps).index(x) for x in list(np.intersect1d(i_gps,o_gps))]   #sub a and b for left and right collumn of output, returns a list of the indexs of a to keep 
     lat = o_real - i_real[i_lst]
-    #e_file.write(str(lat))
+    e_file.write("Median latency = %f" % np.median(lat))
     e_file.close()
 
-    assert np.mean(lat) < 6
+    assert np.median(lat) < 6
     #assert np.percentile(lat, 99) < 10
     #assert np.max(lat) < 10

tests/tests_pytest/run_calib_pipeline.py

@@ -16,25 +16,87 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 import os
+from threading import Thread
 
-def Run_calib():
-    e_file = open('tests/tests_pytest/error_results.txt', 'a')
-    e_file.write('running calib pipeline \n')
+"""
+from tests.tests_pytest.State_Vector import State_Vector
+from tests.tests_pytest.ASD import ASD
+from tests.tests_pytest.act2darm_timeseries import Act2darm
+from tests.tests_pytest.pcal2darm_timeseries import Pcal2darm
+from tests.tests_pytest.diff import Diff
+from tests.tests_pytest.latency import Latency
+"""
 
-    os.system("ls tests/tests_pytest/frames/raw/H-H1_R-*.gwf | lal_path2cache > tests/tests_pytest/raw_frames.cache")
+
+def calib_C00_thread_function():
     os.system("gstlal_compute_strain --gps-start-time 1370673024 --gps-end-time 1370678464 --frame-cache tests/tests_pytest/raw_frames.cache --output-path tests/tests_pytest/frames/GDS --frame-duration=64 --frames-per-file=1 --wings=0 --config-file tests/tests_pytest/filters/gstlal_compute_strain_C00_offlineTest_H1.ini --filters-file-name tests/tests_pytest/filters/gstlal_compute_strain_C00_filters_H1_20230613_ci.npz")
     os.system("ls tests/tests_pytest/frames/GDS/H-H1GDS-*.gwf | lal_path2cache > tests/tests_pytest/GDS_frames.cache")
+
+def calib_Approx_thread_function():
     os.system("gstlal_compute_strain --gps-start-time 1370673024 --gps-end-time 1370678464 --frame-cache tests/tests_pytest/raw_frames.cache --output-path tests/tests_pytest/frames/GDS --frame-duration=64 --frames-per-file=1 --wings=0 --config-file tests/tests_pytest/filters/gstlal_compute_strain_Approx_H1.ini --filters-file-name tests/tests_pytest/filters/gstlal_compute_strain_C00_filters_H1_20230613_ci.npz")
     os.system("ls tests/tests_pytest/frames/GDS/H-H1GDS_Approx-*.gwf | lal_path2cache > tests/tests_pytest/GDS_Approx_frames.cache")
+
+def calib_Exact_thread_function():
     os.system("gstlal_compute_strain --gps-start-time 1370673024 --gps-end-time 1370678464 --frame-cache tests/tests_pytest/raw_frames.cache --output-path tests/tests_pytest/frames/GDS --frame-duration=64 --frames-per-file=1 --wings=0 --config-file tests/tests_pytest/filters/gstlal_compute_strain_Exact_H1.ini --filters-file-name tests/tests_pytest/filters/gstlal_compute_strain_C00_filters_H1_20230613_ci.npz")
     os.system("ls tests/tests_pytest/frames/GDS/H-H1GDS_Exact-*.gwf | lal_path2cache > tests/tests_pytest/GDS_Exact_frames.cache")
+
+def calib_LaterStart_thread_function():
     os.system("gstlal_compute_strain --gps-start-time 1370673091 --gps-end-time 1370678464 --frame-cache tests/tests_pytest/raw_frames.cache --output-path tests/tests_pytest/frames/GDS --frame-duration=64 --frames-per-file=1 --wings=0 --config-file tests/tests_pytest/filters/gstlal_compute_strain_LaterStart_H1.ini --filters-file-name tests/tests_pytest/filters/gstlal_compute_strain_C00_filters_H1_20230613_ci.npz")
     os.system("ls tests/tests_pytest/frames/GDS/H-H1GDS_LaterStart-*.gwf | lal_path2cache > tests/tests_pytest/GDS_LaterStart_frames.cache")
+
+def calib_Latency_thread_function():
     os.system("gstlal_compute_strain --gps-start-time 1373199776 --gps-end-time 1373200767 --frame-cache tests/tests_pytest/raw_frames.cache --output-path tests/tests_pytest/frames/GDS --frame-duration=1 --frames-per-file=1 --wings=0 --config-file tests/tests_pytest/filters/gstlal_compute_strain_C00_LatencyTest_H1.ini --filters-file-name tests/tests_pytest/filters/gstlal_compute_strain_C00_filters_H1_20230613_ci.npz")
 
+def Run_calib():
+    e_file = open('tests/tests_pytest/error_results.txt', 'a')
+    e_file.write('running calib pipeline \n')
+
+    os.system("ls tests/tests_pytest/frames/raw/H-H1_R-*.gwf | lal_path2cache > tests/tests_pytest/raw_frames.cache")
+
+    calib_C00_thread = Thread(target = calib_C00_thread_function)
+    calib_Approx_thread = Thread(target = calib_Approx_thread_function)
+    calib_Exact_thread = Thread(target = calib_Exact_thread_function)
+    calib_LaterStart_thread = Thread(target = calib_LaterStart_thread_function)
+    calib_Latency_thread = Thread(target = calib_Latency_thread_function)
+
+    calib_C00_thread.start()
+    calib_Approx_thread.start()
+    calib_Exact_thread.start()
+    calib_LaterStart_thread.start()
+    calib_Latency_thread.start()
+
+    calib_C00_thread.join()
+    calib_Approx_thread.join()
+    calib_Exact_thread.join()
+    calib_LaterStart_thread.join()
+    calib_Latency_thread.join()
+
     f_num = len(os.listdir('tests/tests_pytest/frames/GDS'))
     assert f_num == 1328
 
     e_file.write('ran calib pipeline \n')
     e_file.close()
 
+    """
+    State_Vector_thread = Thread(target = State_Vector)
+    ASD_thread = Thread(target = ASD)
+    Act2darm_thread = Thread(target = Act2darm)
+    Pcal2darm_thread = Thread(target = Pcal2darm)
+    Diff_thread = Thread(target = Diff)
+    Latency_thread = Thread(target = Latency)
+
+    State_Vector_thread.start()
+    ASD_thread.start()
+    Act2darm_thread.start()
+    Pcal2darm_thread.start()
+    Diff_thread.start()
+    Latency_thread.start()
+
+    State_Vector_thread.join()
+    ASD_thread.join()
+    Act2darm_thread.join()
+    Pcal2darm_thread.join()
+    Diff_thread.join()
+    Latency_thread.join()
+    """
+