diff --git a/.gitignore b/.gitignore index 63c6a81217ed91205e4ce082c0df31b1ed7ad949..a2b1480bfc98c51c9004010eeadbd804c7f6ec93 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,10 @@ libtool .idea gstlal-calibration.spec lib/gstlal-calibration.pc +tests/check_calibration/*.png +tests/check_calibration/*.pdf +tests/check_calibration/*.txt +tests/check_calibration/*.cache tests/check_calibration/Filters/** tests/check_calibration/Frames/** tests/check_calibration/BB_injections/** @@ -75,3 +79,11 @@ tests/*/*/Filters/** tests/*/*/Frames/** tests/*/*/*.swp tests/DCS_test/H1/out.txt +tests/tests_pytest/*.txt +tests/tests_pytest/*.cache +tests/tests_pytest/*.png +tests/tests_pytest/**/*.txt +tests/tests_pytest/**/*.png +tests/tests_pytest/frames/GDS/ +tests/*.dot +python/__init__.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0dc1c96b4e9180ad03965c915855e68214fa3365..b41a5d12a58a339a5e99404377b8deb9a5f5ceb4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -138,6 +138,7 @@ test: # run the tests - python3 -m pytest artifacts: + when: always paths: - tests/tests_pytest/*.txt - tests/tests_pytest/*.png diff --git a/gst/lal/gstlal_smoothkappas.c b/gst/lal/gstlal_smoothkappas.c index 1619bdde43380ba5d4ad15620bf1949f35eefe2e..a62a59b8a4eea5e24a4742f6d40b8c20ed091d50 100644 --- a/gst/lal/gstlal_smoothkappas.c +++ b/gst/lal/gstlal_smoothkappas.c @@ -312,7 +312,7 @@ static GstFlowReturn smooth_complex_buffer_ ## DTYPE(const DTYPE complex *src, g new_element_re = creal(doublesrc); \ *num_bad_in_avg_re = 0; \ } \ - if(gap || cimag(doublesrc) > default_kappa_im + maximum_offset_im || cimag(doublesrc) < default_kappa_im - maximum_offset_im || isnan(cimag(doublesrc)) || isinf(cimag(doublesrc)) || cimag(doublesrc) == 0) { \ + if(gap || cimag(doublesrc) > default_kappa_im + maximum_offset_im || cimag(doublesrc) < default_kappa_im - maximum_offset_im || isnan(cimag(doublesrc)) || isinf(cimag(doublesrc)) || (reject_zeros && cimag(doublesrc) == 0)) { \ if(default_to_median) \ new_element_im = *current_median_im; \ else \ diff --git a/python/calibration_parts.py b/python/calibration_parts.py index 910721c90a14be1dc68edd8dc4d072babca66054..602501e6b54c92768eceb6b2e87e595477f7e84c 100644 --- a/python/calibration_parts.py +++ b/python/calibration_parts.py @@ -508,10 +508,12 @@ def remove_lines_with_witnesses(pipeline, signal, witnesses, freqs, freq_vars, f freqs[i] = [freqs[i]] filter_param = 0.0625 + zero_latency = filter_latency == 0.0 downsample_quality = 4 upsample_quality = 4 - resample_shift = 16.0 + 16.5 - zero_latency = filter_latency == 0.0 + downsample_shift = 0.0 if not zero_latency else 16.0 if downsample_quality == 4 else 96.0 # if downsample_quality == 5 + upsample_shift = 0.0 if not zero_latency else 16.5 if upsample_quality == 4 else 96.5 # if upsample_quality == 5 + resample_shift = downsample_shift + upsample_shift for i in range(len(witnesses)): for j in range(len(witnesses[i])): @@ -554,6 +556,11 @@ def remove_lines_with_witnesses(pipeline, signal, witnesses, freqs, freq_vars, f phase_shift = mktypecast(pipeline, f0_beat_frequency, "Z128LE") phase_shift = multiply_constant(pipeline, phase_shift, 0.0, two_n_pi_delta_t) phase_factor = pipeparts.mkgeneric(pipeline, phase_shift, "cexp") + # We need to drop samples at the start since this later is multiplied by + # tfs_at_f, which is filtered and has a later first timestamp. If we don't + # do this, the multiplier will fill in tfs_at_f with ones at start, so the + # product will be enormous and put noise in the cleaned strain at startup. + phase_factor = mkinsertgap(pipeline, phase_factor, insert_gap = False, chop_length = int(1e9 / compute_rate * ((1.0 - filter_latency) * (filter_samples - 1) + downsample_shift + 7.0 * compute_rate * (not zero_latency)))) phase_factor = pipeparts.mktee(pipeline, phase_factor) # Find amplitude and phase of line in signal @@ -660,7 +667,7 @@ def remove_lines_with_witnesses(pipeline, signal, witnesses, freqs, freq_vars, f reconstructed_line_in_signal = mkmultiplier(pipeline, list_srcs(pipeline, tfs_at_f[i], line_in_witnesses[i]), queue_length = [queue_length]) # The next 2 lines handle the case when the witness channel is missing reconstructed_line_in_signal = mkgate(pipeline, reconstructed_line_in_signal, witness_exists[i], 1, attack_length = -((1.0 - filter_latency) * (filter_samples + resample_shift)), hold_length = -1, name = "linesub_reconstructed_witness_gate_%d_%d_%d_%d" % (m, n, i, function_inst_num), queue_length = queue_length) - reconstructed_line_in_signal = pipeparts.mkgeneric(pipeline, reconstructed_line_in_signal, "lal_smoothkappas", default_kappa_re = 0.0, array_size = 1, avg_array_size = 1, default_to_median = True, filter_latency = filter_latency) + reconstructed_line_in_signal = pipeparts.mkgeneric(pipeline, reconstructed_line_in_signal, "lal_smoothkappas", default_kappa_re = 0.0, array_size = 1, avg_array_size = 1, default_to_median = True, reject_zeros = False, filter_latency = filter_latency) reconstructed_line_in_signal = mkresample(pipeline, reconstructed_line_in_signal, upsample_quality, zero_latency, rate_out) reconstructed_line_in_signal = pipeparts.mkgeneric(pipeline, reconstructed_line_in_signal, "lal_demodulate", line_frequency = -1.0 * freqs[m][n], prefactor_real = -2.0) # Connect to line frequency updater if given diff --git a/tests/tests_pytest/ASD.py b/tests/tests_pytest/ASD.py index 81cdd48fa4d5b41043b13f9794050a497781344d..60534e8b02f67f88051af201d1ca8021f00b5102 100755 --- a/tests/tests_pytest/ASD.py +++ b/tests/tests_pytest/ASD.py @@ -27,8 +27,8 @@ from lal import LIGOTimeGPS from ligo import segments -gps_start_time = 1404305400 -gps_end_time = 1404308200 +gps_start_time = 1404304248 +gps_end_time = 1404308216 ifo = 'H1' def compute_ASD_from_Approx_cache(pipeline, name): diff --git a/tests/tests_pytest/ASD_data/Approx_clean_asd_standard.txt b/tests/tests_pytest/ASD_data/Approx_clean_asd_standard.txt index 0b6a8b1e175b51b070793dd97419525f477fc32c..60e574dd8687c5dba96ce7aa78a1f8887bb3ae70 100644 --- a/tests/tests_pytest/ASD_data/Approx_clean_asd_standard.txt +++ b/tests/tests_pytest/ASD_data/Approx_clean_asd_standard.txt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:113cfd473c6c51dff700ef2b7a7aef3297538c4a842c47d717a1dc4b45b730d4 +oid sha256:181cdcaf03eba0c0ffc6c8ac592e08a06037343558b55049cf766608c5ce437c size 1179687 diff --git a/tests/tests_pytest/ASD_data/Approx_hoft_asd_standard.txt b/tests/tests_pytest/ASD_data/Approx_hoft_asd_standard.txt index 8ed6101682e52ab2841caa3bc25b02a0e18fa124..89c93f957025a0301732cea6032eb58c9f5cb3be 100644 --- a/tests/tests_pytest/ASD_data/Approx_hoft_asd_standard.txt +++ b/tests/tests_pytest/ASD_data/Approx_hoft_asd_standard.txt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d216837e6adbc1173e2dee7961b7c6311f36ce7294edc761e328961b50fe2fda +oid sha256:a8a8104622a40efab84f63737e29ca5b19839f5c2cd35302e13f80d8c70b3a3a size 1179687 diff --git a/tests/tests_pytest/ASD_data/Exact_clean_asd_standard.txt b/tests/tests_pytest/ASD_data/Exact_clean_asd_standard.txt index ee04cac1f1e023fe9f38d45e3265abca8fbd31cb..b048e675fe32f04a86a2174c32955a3f601a3fba 100644 --- a/tests/tests_pytest/ASD_data/Exact_clean_asd_standard.txt +++ b/tests/tests_pytest/ASD_data/Exact_clean_asd_standard.txt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:34050997b2dc411e017eabd6c8436d168ae1f0e938ca5b0943bd219afd4c2fd1 +oid sha256:be1e0ec13bfef7fe893f833e3969bd79d2ec02e684f4b1615dbb0378dbc936e8 size 1179687 diff --git a/tests/tests_pytest/ASD_data/Exact_hoft_asd_standard.txt b/tests/tests_pytest/ASD_data/Exact_hoft_asd_standard.txt index 180efc4c9693f1beff2f9ec6279806912300d1c1..dad08fdc91fe8025ec30303d13533492f9d7667a 100644 --- a/tests/tests_pytest/ASD_data/Exact_hoft_asd_standard.txt +++ b/tests/tests_pytest/ASD_data/Exact_hoft_asd_standard.txt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:da105234a5f00c804ffe8eead891426bcd673d7252b31bdc7250a5afe719ea56 +oid sha256:b3394bdce5b23e11cad1deb3119d501c17b1287267c42dcfd82888e19206e04f size 1179687 diff --git a/tests/tests_pytest/STV_error.py b/tests/tests_pytest/STV_error.py index 72395fbc822d171e8d6f737a6e4c351c4fc94428..55c728b59204cfb4d1b0f2b64b9c47e771075805 100644 --- a/tests/tests_pytest/STV_error.py +++ b/tests/tests_pytest/STV_error.py @@ -51,8 +51,8 @@ def check_bits(filename): data = np.loadtxt('tests/tests_pytest/State_Vector_data/%s.txt' % filename) # Make sure the data overlaps a reasonable amount - assert abs(standard[0][0] - data[0][0]) < 100 - assert abs(standard[-1][0] - data[-1][0]) < 100 + assert abs(standard[0][0] - data[0][0]) < 100, "got unexpected start time for state vector data" + assert abs(standard[-1][0] - data[-1][0]) < 100, "got unexpected end time for state vector data" # Trim if needed missing_initial_data_samples = int(16 * (data[0][0] - standard[0][0])) @@ -76,7 +76,7 @@ def check_bits(filename): for i in [1, 2, 3, 4, 5, 6, 7, 8, 9, 19]: # Require agreement at each sample e_file = open('tests/tests_pytest/error_results.txt', 'a') - e_file.write("Testing %s\n" % bitnames[i]) + e_file.write("Testing %s %s\n" % (filename, bitnames[i])) for j in range(len(data)): assert int(data[j]) & bits[i] == int(standard[j]) & bits[i], "%s failure!" % bitnames[i] e_file.write("%s passed\n" % bitnames[i]) @@ -84,7 +84,7 @@ def check_bits(filename): for i in [0, 10, 11, 12, 13, 14, 15, 16]: # Allow small differences in on/off transition times e_file = open('tests/tests_pytest/error_results.txt', 'a') - e_file.write("Testing %s\n" % bitnames[i]) + e_file.write("Testing %s %s\n" % (filename, bitnames[i])) consecutive_failures = 0 for j in range(len(data)): if int(data[j]) & bits[i] != int(standard[j]) & bits[i]: @@ -97,7 +97,7 @@ def check_bits(filename): for i in [17, 18, 20]: # Require agreement at a majority of samples e_file = open('tests/tests_pytest/error_results.txt', 'a') - e_file.write("Testing %s\n" % bitnames[i]) + e_file.write("Testing %s %s\n" % (filename, bitnames[i])) failures = 0 for j in range(len(data)): if int(data[j]) & bits[i] != int(standard[j]) & bits[i]: diff --git a/tests/tests_pytest/State_Vector_data/State_Vector_Approx_standard.txt b/tests/tests_pytest/State_Vector_data/State_Vector_Approx_standard.txt index 1566f13f976e46909fa1fb940fa7b6992e1fe8ce..1cedcaf41e2c19c9d919f84709405f2c6acbeb82 100644 --- a/tests/tests_pytest/State_Vector_data/State_Vector_Approx_standard.txt +++ b/tests/tests_pytest/State_Vector_data/State_Vector_Approx_standard.txt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:945e58c01dd7cfbb478aff6678b98fb1ed3a18a5b8f454b3e0c4c5b0450f556a -size 1893687 +oid sha256:a3c41ce35045ff551391016d6be4f93f1c91751668cbc541b68436b1fc8b3acf +size 1898241 diff --git a/tests/tests_pytest/State_Vector_data/State_Vector_Exact_standard.txt b/tests/tests_pytest/State_Vector_data/State_Vector_Exact_standard.txt index 1566f13f976e46909fa1fb940fa7b6992e1fe8ce..1cedcaf41e2c19c9d919f84709405f2c6acbeb82 100644 --- a/tests/tests_pytest/State_Vector_data/State_Vector_Exact_standard.txt +++ b/tests/tests_pytest/State_Vector_data/State_Vector_Exact_standard.txt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:945e58c01dd7cfbb478aff6678b98fb1ed3a18a5b8f454b3e0c4c5b0450f556a -size 1893687 +oid sha256:a3c41ce35045ff551391016d6be4f93f1c91751668cbc541b68436b1fc8b3acf +size 1898241 diff --git a/tests/tests_pytest/act2darm_timeseries.py b/tests/tests_pytest/act2darm_timeseries.py index 65c54f2da70ac150d1da70c76b83470098097bd2..72d767796a7c5e3e194720a3f2c76f8679a93d9f 100755 --- a/tests/tests_pytest/act2darm_timeseries.py +++ b/tests/tests_pytest/act2darm_timeseries.py @@ -229,7 +229,6 @@ def act2darm(pipeline, name): def Act2darm(): test_common.build_and_run(act2darm, "act2darm", segment = segments.segment((LIGOTimeGPS(0, 1000000000 * gps_start_time), LIGOTimeGPS(0, 1000000000 * gps_end_time)))) plot_deltal_over_inj_timeseries('act') - assert False rms('A') rms('A', 'E') diff --git a/tests/tests_pytest/error.py b/tests/tests_pytest/error.py index 1fd1e95d0c7322c2bfc1f1744eab6bee59592c74..4536763f01f96720e8bff1a7c8eb4b0cb2af8d5d 100644 --- a/tests/tests_pytest/error.py +++ b/tests/tests_pytest/error.py @@ -82,7 +82,7 @@ def rms(d_type, typ='A'): st_f_list.append('tests/tests_pytest/act_data/%s_GDS-CALIB_STRAIN_over_%s_standard_1_at_%0.1fHz.txt' % (ifo, act_names[i], act_freq_list[i])) - assert len(st_f_list) == len(f_list) + assert len(st_f_list) == len(f_list), "Mismatch between number of standard files and number of test files: %d != %d" % (len(st_f_list), len(f_list)) for i in range(len(f_list)): standard = np.loadtxt(st_f_list[i]) data = np.loadtxt(f_list[i]) @@ -122,8 +122,8 @@ def rms(d_type, typ='A'): break # Make sure the data sets overlapped as expected - assert j > 0.8 * len(data) - assert std_idx == standard_n + assert j > 0.8 * len(data), "%s: failed to use enough testing data" % name[i] + assert std_idx == standard_n, "%s: failed to use all of the standard data" % name[i] mean_sq = sum_sq / std_idx rms = np.sqrt(mean_sq) @@ -133,5 +133,5 @@ def rms(d_type, typ='A'): e_file.close() print(s2, file = sys.stderr) - assert rms < 0.001 + assert rms < 0.001, "%s: error too large!" % name[i] diff --git a/tests/tests_pytest/run_calib_pipeline.py b/tests/tests_pytest/run_calib_pipeline.py index 2e418c92f475be3a9778cab2991c1b060907f8d3..3f87aea2c3799fb57edf745d1fdcc15d120b72ef 100644 --- a/tests/tests_pytest/run_calib_pipeline.py +++ b/tests/tests_pytest/run_calib_pipeline.py @@ -52,4 +52,3 @@ def Run_calib(): f_num = len(os.listdir('tests/tests_pytest/frames/GDS')) assert f_num == 2468 -