From 4a549b4460c519e2589a6f2472d0a361973af706 Mon Sep 17 00:00:00 2001
From: Aaron Viets <aaron.viets@ligo.org>
Date: Tue, 20 Nov 2018 12:21:39 -0800
Subject: [PATCH] gstlal-calibration:  Many small chenges, including
 conditional reading in of a config file parameter in gstlal_compute_strain
 and DCS config file updates.

---
 gstlal-calibration/bin/gstlal_compute_strain  |  2 +-
 .../tests/H1DCS_AllCorrections_Cleaning.ini   | 24 +++++++++----------
 .../H1DCS_AllCorrections_Cleaning_TEST.ini    |  6 ++---
 .../check_calibration/ASD_comparison_plots    |  2 +-
 .../tests/check_calibration/Makefile          | 19 +++++++++------
 .../check_calibration/frame_manipulator.py    |  4 ++--
 6 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/gstlal-calibration/bin/gstlal_compute_strain b/gstlal-calibration/bin/gstlal_compute_strain
index 3db757843f..d2c0a9c14d 100755
--- a/gstlal-calibration/bin/gstlal_compute_strain
+++ b/gstlal-calibration/bin/gstlal_compute_strain
@@ -268,7 +268,7 @@ witness_frequency_resolution = float(DataCleaningConfigs["witnessfrequencyresolu
 witness_tf_update_time = float(DataCleaningConfigs["witnesstfupdatetime"])
 critical_lock_loss_time = float(DataCleaningConfigs["criticallocklosstime"])
 witness_filter_taper_time = float(DataCleaningConfigs["witnessfiltertapertime"])
-witness_tf_time_shift = float(DataCleaningConfigs["witnesstftimeshift"])
+witness_tf_time_shift = float(DataCleaningConfigs["witnesstftimeshift"]) if "witnesstftimeshift" in DataCleaningConfigs else 0.0
 witness_tf_filename = DataCleaningConfigs["witnesstffilename"] if DataCleaningConfigs["witnesstffilename"] != "None" else None
 
 expected_kappatst_real = float(TDCFConfigs["expectedkappatstreal"])
diff --git a/gstlal-calibration/config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning.ini b/gstlal-calibration/config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning.ini
index adfa49bdc5..21e36a9e93 100644
--- a/gstlal-calibration/config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning.ini
+++ b/gstlal-calibration/config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning.ini
@@ -210,7 +210,7 @@ CohUncDARMLine1Channel: CAL-CS_TDEP_DARM_LINE1_UNCERTAINTY
 PowerLinesChannel: PEM-EY_MAINSMON_EBAY_1_DQ
 # Comma-separated list of witness channels to use to subtract noise from h(t)
 # Set to None if no witness channels are to be used
-WitnessChannelList: IMC-WFS_A_DC_PIT_OUT_DQ,IMC-WFS_B_DC_PIT_OUT_DQ,IMC-WFS_A_DC_YAW_OUT_DQ,IMC-WFS_B_DC_YAW_OUT_DQ;PSL-DIAG_BULLSEYE_YAW_OUT_DQ,PSL-DIAG_BULLSEYE_WID_OUT_DQ,PSL-DIAG_BULLSEYE_PIT_OUT_DQ
+WitnessChannelList: IMC-WFS_A_DC_PIT_OUT_DQ,IMC-WFS_B_DC_PIT_OUT_DQ,IMC-WFS_A_DC_YAW_OUT_DQ,IMC-WFS_B_DC_YAW_OUT_DQ,PSL-DIAG_BULLSEYE_YAW_OUT_DQ,PSL-DIAG_BULLSEYE_WID_OUT_DQ,PSL-DIAG_BULLSEYE_PIT_OUT_DQ
 ###############################
 # EPICS Records Channel Names #
 ###############################
@@ -286,7 +286,7 @@ EPICSRefSR: 16
 # Sample rate for power lines channel
 PowerLinesChannelSR: 1024
 # Sample rates at which transfer functions will be computed and witness channels will be filtered, given as a semicolon-separated list, e.g., 2048;2048;512;2048. This must be given if WitnessChannelList is not None, and it must be the same length.
-WitnessChannelSR: 2048;2048
+WitnessChannelSR: 2048
 # Sample rates at which to compute and record TDCFs
 ComputeFactorsSR: 16
 RecordFactorsSR: 16
@@ -305,7 +305,7 @@ BufferLength: 1.0
 FrequencyDomainFiltering: No
 Dewhitening: No
 # Latency of all filtering/averaging/median processes (other than calibration model filters) as a fraction of filter length. Value should be set between 0.0 and 1.0.
-FilterLatency: 0.0
+FilterLatency: 1.0
 
 [DataCleaningConfigurations]
 ####################################################
@@ -330,29 +330,29 @@ PowerLinesTFAveragingTime: 128
 # The length in seconds of the fast Fourier transforms used to compute transfer functions between witness channels and h(t). The fft's are windowed with Hann windows and overlapped.
 WitnessChannelFFTTime: 4.0
 # The number of ffts to take before averaging the witness -> h(t) transfer functions calculation. The average is taken after the ratio h(f) / witness(f).
-NumWitnessFFTs: 1800
+NumWitnessFFTs: 510
 # Sets the minimum number of FFTs necessary to produce the first transfer functions and clean data after data flow starts.
-MinWitnessFFTs: 400
+MinWitnessFFTs: 510
 # The length in seconds of the filters applied to the witness channels before subtracting from h(t)
 WitnessFIRLength: 0.5
 # The frequency resolution of the filters applied to the witness channels before subtracting from h(t). It can be advantageous to lower the frequency resolution in order to average over excess noise.
 WitnessFrequencyResolution: 1.0
 # List of minima and maxima of frequency ranges where the Fourier transform of h(t) will be replaced by a straight line in the calculation of transfer functions between witness channels and h(t) for noise subtraction. Semicolons separate lists for different sets of witness channels. If no notches are desired, use zeros, e.g., \'0;0;0\'. Here is an example using the expected format: \'495.0,515.0,985.0,1015.0;59,60,119,121;0\' This can be useful, e.g., if there are loud lines in the signal that are not present in the witness channels.
-WitnessNotchFrequencies: 495.0,515.0,985.0,1015.0;495.0,515.0,985.0,1015.0
+WitnessNotchFrequencies: 495.0,515.0,985.0,1015.0
 # The amount of time after transfer functions between witness channels and h(t) are finished to begin the calculation of the next set of transfer functions
-WitnessTFUpdateTime: 3600
+WitnessTFUpdateTime: 2
 # If lock-loss lasts at least this many seconds, transfer functions will revert to those computed at the beginning of a lock stretch. Set to zero to disbale.
-CriticalLockLossTime: 1800
+CriticalLockLossTime: 0
 # The amount of time to use to taper in newly computed FIR filters for witness channels being used for noise subtraction.
 WitnessFilterTaperTime: 10
 # If writing transfer functions to file, this sets the name. If transfer functions should not be written to file, this should be set to None
-WitnessTFFilename: transfer_functions
+WitnessTFFilename: transfer_functions_DCS
 # Should the transfer function calculation use a median? If not, an average (mean) is used.
-WitnessTFUseMedian: Yes
+WitnessTFUseMedian: No
 # Should transfer functions be computed on a fixed schedule, so that the output does not depend on start time?  This is useful for running jobs in parallel.  Otherwise, they are computed asap.
-WitnessTFParallelMode: No
+WitnessTFParallelMode: Yes
 # When using parallel mode, how many seconds later should we shift the time when transfer functions start being computed from a multiple of the cycle period?
-WitnessTFTimeShift: 400
+WitnessTFTimeShift: 350
 ###############################
 # Options for HOFT_CLEAN bits #
 ###############################
diff --git a/gstlal-calibration/config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning_TEST.ini b/gstlal-calibration/config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning_TEST.ini
index 28265209db..c369ec1364 100644
--- a/gstlal-calibration/config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning_TEST.ini
+++ b/gstlal-calibration/config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning_TEST.ini
@@ -346,13 +346,13 @@ CriticalLockLossTime: 0
 # The amount of time to use to taper in newly computed FIR filters for witness channels being used for noise subtraction.
 WitnessFilterTaperTime: 10
 # If writing transfer functions to file, this sets the name. If transfer functions should not be written to file, this should be set to None
-WitnessTFFilename: transfer_functions
+WitnessTFFilename: transfer_functions_DCS_TEST
 # Should the transfer function calculation use a median? If not, an average (mean) is used.
-WitnessTFUseMedian: Yes
+WitnessTFUseMedian: No
 # Should transfer functions be computed on a fixed schedule, so that the output does not depend on start time?  This is useful for running jobs in parallel.  Otherwise, they are computed asap.
 WitnessTFParallelMode: Yes
 # When using parallel mode, how many seconds later should we shift the time when transfer functions start being computed from a multiple of the cycle period?
-WitnessTFTimeShift: 400
+WitnessTFTimeShift: 350
 ###############################
 # Options for HOFT_CLEAN bits #
 ###############################
diff --git a/gstlal-calibration/tests/check_calibration/ASD_comparison_plots b/gstlal-calibration/tests/check_calibration/ASD_comparison_plots
index 635de0fffd..4607de6432 100755
--- a/gstlal-calibration/tests/check_calibration/ASD_comparison_plots
+++ b/gstlal-calibration/tests/check_calibration/ASD_comparison_plots
@@ -57,7 +57,7 @@ ax = plot.gca()
 plot.set_ylabel('Strain [Hz $^{-1/2}$]', fontname = 'Times', fontsize = 18)
 plot.set_xlabel('Frequency [Hz]', fontname = 'Times', fontsize = 18)
 plot.add_legend([r'CALCS h(t) ASD', r'GDS h(t) ASD', r'DCS h(t) ASD'], loc='upper right', fontsize='small')
-ax.set_xlim(10,8192)
+ax.set_xlim(0.5,8192)
 ax.set_ylim(1e-24,1e-16)
 ax.legend()
 plot.save('%s_%s_%s_spectrum_comparison.png' % (options.ifo, start_time, end_time))
diff --git a/gstlal-calibration/tests/check_calibration/Makefile b/gstlal-calibration/tests/check_calibration/Makefile
index d73c3ee540..f1ff659fec 100644
--- a/gstlal-calibration/tests/check_calibration/Makefile
+++ b/gstlal-calibration/tests/check_calibration/Makefile
@@ -6,23 +6,25 @@
 # which interferometer (H or L)
 IFO = H
 # determines where to look for filters files (e.g., O1, O2, O3, ER10, ER13, ER14, PreER10, PreER13, PreER14)
-OBSRUN = O2
+OBSRUN = PreER13
 
-START = 1185767808
-END = 1185775616
+START = 1225967424
+#$(shell echo 1185763328 - 700 | bc)
+END = 1225968448
+#$(shell echo 1185767424 + 700 | bc)
 SHMRUNTIME = 400
 # How much time does the calibration need to settle at the start and end?
-PLOT_WARMUP_TIME = 4000
-PLOT_COOLDOWN_TIME = 1024
+PLOT_WARMUP_TIME = 256
+PLOT_COOLDOWN_TIME = 64
 
-GDSCONFIGS = ../../config_files/PreER13/H1/C00/H1GDS_FRAMES_1225558818.ini
+GDSCONFIGS = ../../config_files/PreER13/H1/tests/H1GDS_1226769644.ini
 DCSCONFIGS = ../../config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning.ini
 DCSFCCCONFIGS = ../../config_files/O2/H1/tests/H1DCS_FreqIndepAndFccCorrections_Cleaning.ini
 GDSTESTCONFIGS = ../../config_files/PreER13/H1/H1GDS_TEST_1225558818.ini
 DCSTESTCONFIGS = ../../config_files/O2/H1/tests/H1DCS_AllCorrections_Cleaning_TEST.ini
 GDSSHMCONFIGS = ../../config_files/PreER13/H1/tests/H1GDS_1222058826_shm2frames.ini
 
-all: noise_subtraction_ASD_DCS_TEST
+all: CALCS_GDS_ASD GDS_over_CALCS
 
 ###############################################
 ### These commands should change less often ###
@@ -100,6 +102,9 @@ lines_ratio_DCS: $(IFO)1_hoft_DCS_frames.cache
 latency_test: $(IFO)1_hoft_GDS_SHM_frames.cache
 	python latency_plot.py --intime-file gstlal_compute_strain_timestamps_in.txt --outtime-file gstlal_compute_strain_timestamps_out.txt --plot-filename-prefix $(IFO)1GDS_latency --plot-title '$(IFO)1 Calibration Latency vs Time'
 
+CALCS_GDS_ASD: $(IFO)1_hoft_GDS_frames.cache $(IFO)1_easy_raw_frames.cache
+	./ASD_comparison_plots --ifo $(IFO)1 --gps-start-time $(PLOT_START) --gps-end-time $(PLOT_END) --raw-frame-cache $(IFO)1_easy_raw_frames.cache --hoft-frame-cache $(IFO)1_hoft_GDS_frames.cache
+
 GDS_over_CALCS: $(IFO)1_hoft_GDS_frames.cache $(IFO)1_easy_raw_frames.cache
 	python plot_transfer_function.py --gps-start-time $(PLOT_START) --gps-end-time $(PLOT_END) --ifo $(IFO)1 --denominator-frame-cache $(IFO)1_easy_raw_frames.cache --denominator-channel-name CAL-DELTAL_EXTERNAL_DQ --denominator-name 'CALCS' --numerator-frame-cache-list $(IFO)1_hoft_GDS_frames.cache --numerator-channel-list GDS-CALIB_STRAIN --numerator-name 'GDS' --use-median --magnitude-min 0.7 --magnitude-max 1.3 --phase-min -180.0 --phase-max 180.0 --labels 'GDS / CALCS' --poles '30,0,30,0,30,0,30,0,30,0,30,0,-3.009075115760242e3,3.993177550236464e3,-3.009075115760242e3,-3.993177550236464e3,-5.839434764093102e2,6.674504477214695e3,-5.839434764093102e2,-6.674504477214695e3' --zeros '0.3,0,0.3,0,0.3,0,0.3,0,0.3,0,0.3,0,1.431097327857237e2,8.198751100282409e3,1.431097327857237e2,-8.198751100282409e3,8.574723070843939e2,1.636154629741894e4,8.574723070843939e2,-1.636154629741894e4' --gain 3994.5
 
diff --git a/gstlal-calibration/tests/check_calibration/frame_manipulator.py b/gstlal-calibration/tests/check_calibration/frame_manipulator.py
index 7df4bdad12..0394238d24 100644
--- a/gstlal-calibration/tests/check_calibration/frame_manipulator.py
+++ b/gstlal-calibration/tests/check_calibration/frame_manipulator.py
@@ -102,10 +102,10 @@ try:
 	f = open('frchannels.txt', 'r')
 	available_channels = f.read()
 	f.close()
+	print "Making channel list using available channels"
 except:
-	raise ValueError('Cannot find file frchannels.txt. Run FrChannels')
+	print "Cannot find file frchannels.txt. Run FrChannels to check for available channels"
 
-print "Making channel list using available channels"
 # These are (or may be) in the calibrated frames
 channel_list.append(chan_prefix + "CALIB_STRAIN" + chan_suffix)
 channel_list.append("ODC-MASTER_CHANNEL_OUT_DQ")
-- 
GitLab