From 967b7b2c9a51ecf87fa6470af56cb9cbb1e62794 Mon Sep 17 00:00:00 2001
From: Aaron Viets <aaron.viets@ligo.org>
Date: Mon, 10 Sep 2018 14:49:19 -0700
Subject: [PATCH] gstlal-calibration: Several updates to testing: -
test_commpn.py: Seeking pipeline seems to work now. - pcal-to-darm plotting
script: now works in new setup. - gstlal_compute_strain: removed stray
dq_epics variables
---
gstlal-calibration/bin/gstlal_compute_strain | 21 ------
.../H1DCS_AllCorrections_Cleaning.ini | 2 +-
.../H1DCS_SplitPU_AllCorrections_Cleaning.ini | 2 +-
...GDS_LowLatency_AllCorrections_Cleaning.ini | 2 +-
.../tests/check_calibration/Makefile | 68 ++++++++++++-------
.../check_calibration/pcal2darm_timeseries.py | 28 +++++++-
gstlal-calibration/tests/test_common.py | 7 +-
7 files changed, 75 insertions(+), 55 deletions(-)
diff --git a/gstlal-calibration/bin/gstlal_compute_strain b/gstlal-calibration/bin/gstlal_compute_strain
index 464f4bca69..3dd2a6f2e6 100755
--- a/gstlal-calibration/bin/gstlal_compute_strain
+++ b/gstlal-calibration/bin/gstlal_compute_strain
@@ -365,19 +365,6 @@ else:
remove_pum_act_line = False
remove_uim_act_line = False
-# How many EPICS will we for the CALIB_STATE_VECTOR calculation? It depends on the IFO and the time we are calibrating
-if not compute_calib_statevector:
- num_dq_epics = 0
-elif InputConfigs["datasource"] == "lvshm":
- num_dq_epics = 21
-elif gps_start_time > 1220000000:
- num_dq_epics = 21
-elif instrument == "H1" and gps_start_time > 1175976256:
- num_dq_epics = 14
-elif instrument == "L1" and gps_start_time > 1179588864:
- num_dq_epics = 10
-else:
- num_dq_epics = 9
if (not factors_from_filters_file) and (compute_fs or compute_srcq) and ((InputConfigs["datasource"] is "frames" and gps_start_time < 1175954418)):
raise ValueError("Cannot compute SRC detuning parameters as the needed EPICS channels are not in the frames until GPS time 1175954418. Set ComputeFs: No and ComputeSRCQ: No.")
@@ -434,8 +421,6 @@ if factors_from_filters_file or compute_calib_statevector:
except:
if factors_from_filters_file:
raise ValueError("Cannot compute time-dependent correction factors, as the needed EPICS are not contained in the specified filters file.")
- if num_dq_epics > 0:
- num_dq_epics = 0
try:
EP10_real = float(filters["EP10_real"])
EP10_imag = float(filters["EP10_imag"])
@@ -444,8 +429,6 @@ if factors_from_filters_file or compute_calib_statevector:
except:
if factors_from_filters_file and remove_cal_lines:
remove_esd_act_line = False
- if num_dq_epics > 9:
- num_dq_epics = 9
try:
EP11_real = float(filters["EP11_real"])
EP11_imag = float(filters["EP11_imag"])
@@ -458,8 +441,6 @@ if factors_from_filters_file or compute_calib_statevector:
except:
if factors_from_filters_file and (compute_srcq or compute_fs):
raise ValueError("Cannot compute SRC spring frequency or Q, as the needed EPICS are not contained in the specified filters file.")
- if num_dq_epics > 10:
- num_dq_epics = 10
try:
EP15_real = float(filters["EP15_real"])
@@ -491,8 +472,6 @@ if factors_from_filters_file or compute_calib_statevector:
remove_uim_act_line = False
if factors_from_filters_file and (compute_kappapum or compute_kappauim):
raise ValueError("Cannot compute kappa_P or kappa_U, as the needed EPICS are not contained in the specified filters file.")
- if num_dq_epics > 14:
- num_dq_epics = 14
# Load all of the kappa dewhitening and correction factors
act_pcal_line_freq = float(filters["ka_pcal_line_freq"])
diff --git a/gstlal-calibration/config_files/H1DCS_AllCorrections_Cleaning.ini b/gstlal-calibration/config_files/H1DCS_AllCorrections_Cleaning.ini
index ee14906ffb..d2207278af 100644
--- a/gstlal-calibration/config_files/H1DCS_AllCorrections_Cleaning.ini
+++ b/gstlal-calibration/config_files/H1DCS_AllCorrections_Cleaning.ini
@@ -39,7 +39,7 @@ NumBuffers: 10
# If writing to frame files use these options #
###############################################
FrameType: H1DCS_TEST
-OutputPath: .
+OutputPath: Frames/O2/H1/DCS
[CalibrationConfigurations]
IFO: H1
diff --git a/gstlal-calibration/config_files/H1DCS_SplitPU_AllCorrections_Cleaning.ini b/gstlal-calibration/config_files/H1DCS_SplitPU_AllCorrections_Cleaning.ini
index ec8808e163..c890118bdb 100644
--- a/gstlal-calibration/config_files/H1DCS_SplitPU_AllCorrections_Cleaning.ini
+++ b/gstlal-calibration/config_files/H1DCS_SplitPU_AllCorrections_Cleaning.ini
@@ -39,7 +39,7 @@ NumBuffers: 10
# If writing to frame files use these options #
###############################################
FrameType: H1DCS_TEST
-OutputPath: .
+OutputPath: Frames/O2/H1/DCS
[CalibrationConfigurations]
IFO: H1
diff --git a/gstlal-calibration/config_files/H1GDS_LowLatency_AllCorrections_Cleaning.ini b/gstlal-calibration/config_files/H1GDS_LowLatency_AllCorrections_Cleaning.ini
index 9d27b74a54..7ef017e933 100644
--- a/gstlal-calibration/config_files/H1GDS_LowLatency_AllCorrections_Cleaning.ini
+++ b/gstlal-calibration/config_files/H1GDS_LowLatency_AllCorrections_Cleaning.ini
@@ -39,7 +39,7 @@ NumBuffers: 10
# If writing to frame files use these options #
###############################################
FrameType: H1GDS_TEST
-OutputPath: .
+OutputPath: Frames/O2/H1/GDS
[CalibrationConfigurations]
IFO: H1
diff --git a/gstlal-calibration/tests/check_calibration/Makefile b/gstlal-calibration/tests/check_calibration/Makefile
index 6128b86810..37e1602a34 100644
--- a/gstlal-calibration/tests/check_calibration/Makefile
+++ b/gstlal-calibration/tests/check_calibration/Makefile
@@ -3,47 +3,65 @@
### Inputs for user to modify ###
#################################
-# which interferometer
-IFO = H1
+# which interferometer (H or L)
+IFO = H
# determines where to look for filters files (e.g., O1, O2, O3, ER10, ER13, ER14, PreER10, PreER13, PreER14)
OBSRUN = O2
-START = 1186159035
-END = 1186159557
+START = 1186159040
+END = 1186160064
+# How much time does the calibration need to settle at the start and end?
+PLOT_WARMUP_TIME = 220
+PLOT_COOLDOWN_TIME = 64
+
GDSCONFIGS = ../../config_files/H1GDS_LowLatency_AllCorrections_Cleaning.ini
DCSCONFIGS = ../../config_files/H1DCS_AllCorrections_Cleaning.ini
+GDSTESTCONFIGS = ../../config_files/H1GDS_LowLatency_AllCorrections_Cleaning.ini
+DCSTESTCONFIGS = ../../config_files/H1DCS_AllCorrections_Cleaning.ini
all: DCS_pcal2darm_plots
################################################
-### These commands should changes less often ###
+### These commands should change less often ###
################################################
-$(IFO)_raw_frames.cache:
- gw_data_find -o H -t H1_R -s $(START) -e $(END) -l --url-type file > $@
+PLOT_START = $(shell echo $(START) + $(PLOT_WARMUP_TIME) | bc)
+PLOT_END = $(shell echo $(END) - $(PLOT_COOLDOWN_TIME) | bc)
+
+$(IFO)1_raw_frames.cache:
+ gw_data_find -o $(IFO) -t $(IFO)1_R -s $(START) -e $(END) -l --url-type file > $@
-$(IFO)_C00_frames.cache:
- gw_data_find -o H -t H1_HOFT_C00 -s $(START) -e $(END) -l --url-type file > $@
+$(IFO)1_C00_frames.cache:
+ gw_data_find -o $(IFO) -t $(IFO)1_HOFT_C00 -s $(START) -e $(END) -l --url-type file > $@
-$(IFO)_C01_frames.cache:
- gw_data_find -o H -t H1_HOFT_C01 -s $(START) -e $(END) -l --url-type file > $@
+$(IFO)1_C01_frames.cache:
+ gw_data_find -o $(IFO) -t $(IFO)1_HOFT_C01 -s $(START) -e $(END) -l --url-type file > $@
-$(IFO)_C02_frames.cache:
- gw_data_find -o H -t H1_HOFT_C02 -s $(START) -e $(END) -l --url-type file > $@
+$(IFO)1_C02_frames.cache:
+ gw_data_find -o $(IFO) -t $(IFO)1_HOFT_C02 -s $(START) -e $(END) -l --url-type file > $@
-$(IFO)_hoft_GDS_frames.cache: $(IFO)_raw_frames.cache filters framesdir
+$(IFO)1_hoft_GDS_frames.cache: $(IFO)1_raw_frames.cache filters framesdir
GST_DEBUG=3 gstlal_compute_strain --gps-start-time $(START) --gps-end-time $(END) --frame-duration=64 --frames-per-file=1 --wings=0 --config-file $(GDSCONFIGS)
- ls H-H1GDS_TEST*.gwf | lalapps_path2cache > $@
+ ls Frames/$(OBSRUN)/$(IFO)1/GDS/$(IFO)-$(IFO)1GDS_TEST*.gwf | lalapps_path2cache > $@
+
+$(IFO)1_hoft_DCS_frames.cache: $(IFO)1_raw_frames.cache filters framesdir
+ #GST_DEBUG=3 gstlal_compute_strain --gps-start-time $(START) --gps-end-time $(END) --frame-duration=64 --frames-per-file=1 --wings=0 --config-file $(DCSCONFIGS)
+ ls Frames/$(OBSRUN)/$(IFO)1/DCS/$(IFO)-$(IFO)1DCS_TEST*.gwf | lalapps_path2cache > $@
+
+# In case we want to compare one calibration to another...
+$(IFO)1_hoft_GDS_TEST_frames.cache: $(IFO)1_raw_frames.cache filters framesdir
+ GST_DEBUG=3 gstlal_compute_strain --gps-start-time $(START) --gps-end-time $(END) --frame-duration=64 --frames-per-file=1 --wings=0 --config-file $(GDSTESTCONFIGS)
+ ls Frames/$(OBSRUN)/$(IFO)1/GDS/$(IFO)-$(IFO)1GDS_TEST*.gwf | lalapps_path2cache > $@
-$(IFO)_hoft_DCS_frames.cache: $(IFO)_raw_frames.cache filters framesdir
- GST_DEBUG=3 gstlal_compute_strain --gps-start-time $(START) --gps-end-time $(END) --frame-duration=64 --frames-per-file=1 --wings=0 --config-file $(DCSCONFIGS)
- ls H-H1DCS_TEST*.gwf | lalapps_path2cache > $@
+$(IFO)1_hoft_DCS_TEST_frames.cache: $(IFO)1_raw_frames.cache filters framesdir
+ GST_DEBUG=3 gstlal_compute_strain --gps-start-time $(START) --gps-end-time $(END) --frame-duration=64 --frames-per-file=1 --wings=0 --config-file $(DCSTESTCONFIGS)
+ ls Frames/$(OBSRUN)/$(IFO)1/DCS$(IFO)-$(IFO)1DCS_TEST*.gwf | lalapps_path2cache > $@
-GDS_pcal2darm_plots: $(IFO)_raw_frames.cache $(IFO)_hoft_GDS_frames.cache
- python pcal2darm_timeseries.py --ifo=$(IFO) --raw-frame-cache $(IFO)_raw_frames.cache --calibrated-frame-cache $(IFO)_hoft_GDS_frames.cache --config-file $(GDSCONFIGS) --calibrated-channel-list='GDS-CALIB_STRAIN,GDS-CALIB_STRAIN_CLEAN'
+GDS_pcal2darm_plots: $(IFO)1_raw_frames.cache $(IFO)1_hoft_GDS_frames.cache
+ python pcal2darm_timeseries.py --gps-start-time=$(PLOT_START) --gps-end-time=$(PLOT_END) --ifo=$(IFO)1 --raw-frame-cache $(IFO)1_raw_frames.cache --calibrated-frame-cache $(IFO)_hoft_GDS_frames.cache --config-file $(GDSCONFIGS) --calibrated-channel-list='GDS-CALIB_STRAIN,GDS-CALIB_STRAIN_CLEAN'
-DCS_pcal2darm_plots: $(IFO)_raw_frames.cache $(IFO)_hoft_DCS_frames.cache
- python pcal2darm_timeseries.py --ifo=$(IFO) --raw-frame-cache $(IFO)_raw_frames.cache --calibrated-frame-cache $(IFO)_hoft_DCS_frames.cache --config-file $(DCSCONFIGS) --calibrated-channel-list='DCS-CALIB_STRAIN,DCS-CALIB_STRAIN_CLEAN'
+DCS_pcal2darm_plots: $(IFO)1_raw_frames.cache $(IFO)1_hoft_DCS_frames.cache
+ python pcal2darm_timeseries.py --gps-start-time=$(PLOT_START) --gps-end-time=$(PLOT_END) --ifo=$(IFO)1 --raw-frame-cache $(IFO)1_raw_frames.cache --calibrated-frame-cache $(IFO)1_hoft_DCS_frames.cache --config-file $(DCSCONFIGS) --calibrated-channel-list='DCS-CALIB_STRAIN,DCS-CALIB_STRAIN_CLEAN'
filters:
if [ -d Filters/$(OBSRUN)/GDSFilters ]; then \
@@ -55,9 +73,9 @@ filters:
fi
framesdir:
- mkdir -p Frames/$(OBSRUN)/$(IFO)/easy_raw
- mkdir -p Frames/$(OBSRUN)/$(IFO)/GDS
- mkdir -p Frames/$(OBSRUN)/$(IFO)/DCS
+ mkdir -p Frames/$(OBSRUN)/$(IFO)1/easy_raw
+ mkdir -p Frames/$(OBSRUN)/$(IFO)1/GDS
+ mkdir -p Frames/$(OBSRUN)/$(IFO)1/DCS
clean:
rm *.gwf *.cache *.png
diff --git a/gstlal-calibration/tests/check_calibration/pcal2darm_timeseries.py b/gstlal-calibration/tests/check_calibration/pcal2darm_timeseries.py
index 1cf0572b9f..f45a68a992 100644
--- a/gstlal-calibration/tests/check_calibration/pcal2darm_timeseries.py
+++ b/gstlal-calibration/tests/check_calibration/pcal2darm_timeseries.py
@@ -40,6 +40,7 @@ GObject.threads_init()
Gst.init(None)
import lal
+from lal import LIGOTimeGPS
from gstlal import pipeparts
from gstlal import calibration_parts
@@ -54,9 +55,11 @@ from glue.ligolw.utils import segments as ligolw_segments
array.use_in(ligolw.LIGOLWContentHandler)
param.use_in(ligolw.LIGOLWContentHandler)
from glue.ligolw import utils
-from glue import segments
+from ligo import segments
parser = OptionParser()
+parser.add_option("--gps-start-time", metavar = "seconds", type = int, help = "GPS time at which to start processing data")
+parser.add_option("--gps-end-time", metavar = "seconds", type = int, help = "GPS time at which to stop processing data")
parser.add_option("--ifo", metavar = "name", help = "Name of the interferometer (IFO), e.g., H1, L1")
parser.add_option("--raw-frame-cache", metavar = "name", help = "Raw frame cache file")
parser.add_option("--calibrated-frame-cache", metavar = "name", help = "Calibrated frame cache file")
@@ -85,7 +88,26 @@ Config = ConfigParser.ConfigParser()
Config.read(options.config_file)
InputConfigs = ConfigSectionMap("InputConfigurations")
-filters = numpy.load(InputConfigs["filtersfilename"])
+
+#
+# Load in the filters file that contains filter coefficients, etc.
+#
+
+# Search the directory tree for files with names matching the one we want.
+filters_name = InputConfigs["filtersfilename"]
+filters_paths = []
+# Check the user's home directory
+for dirpath, dirs, files in os.walk(os.environ['HOME']):
+ if filters_name in files:
+ # We prefer filters that came directly from a GDSFilters directory of the calibration SVN
+ if dirpath.count("GDSFilters") > 0:
+ filters_paths.insert(0, os.path.join(dirpath, filters_name))
+ else:
+ filters_paths.append(os.path.join(dirpath, filters_name))
+if not len(filters_paths):
+ raise ValueError("Cannot find filters file %s in home directory %s or in /ligo/svncommon/CalSVN/aligocalibration/trunk/Runs/*/GDSFilters", (filters_name, os.environ['HOME']))
+print "\nLoading calibration filters from %s\n" % filters_paths[0]
+filters = numpy.load(filters_paths[0])
ifo = options.ifo
@@ -194,6 +216,6 @@ def pcal2darm(pipeline, name):
#
-test_common.build_and_run(pcal2darm, "pcal2darm")
+test_common.build_and_run(pcal2darm, "pcal2darm", segment = segments.segment((LIGOTimeGPS(0, 1000000000 * options.gps_start_time), LIGOTimeGPS(0, 1000000000 * options.gps_end_time))))
diff --git a/gstlal-calibration/tests/test_common.py b/gstlal-calibration/tests/test_common.py
index 2ef5b7d9ca..26de03cd54 100644
--- a/gstlal-calibration/tests/test_common.py
+++ b/gstlal-calibration/tests/test_common.py
@@ -36,6 +36,7 @@ from gi.repository import Gst
from gstlal import pipeparts
from gstlal import pipeio
from gstlal import simplehandler
+from gstlal import datasource
GObject.threads_init()
@@ -133,9 +134,9 @@ def build_and_run(pipelinefunc, name, segment = None, **pipelinefunc_kwargs):
pipeline = pipelinefunc(Gst.Pipeline(name = name), name, **pipelinefunc_kwargs)
handler = simplehandler.Handler(mainloop, pipeline)
if segment is not None:
- if pipeline.set_state(Gst.State.PAUSED) == Gst.StateChangeReturn.FAILURE:
- raise RuntimeError("pipeline failed to enter PLAYING state")
- pipeline.seek(1.0, Gst.Format(Gst.Format.TIME), Gst.SeekFlags.FLUSH, Gst.SeekType.SET, segment[0].ns(), Gst.SeekType.SET, segment[1].ns())
+ if pipeline.set_state(Gst.State.READY) != Gst.StateChangeReturn.SUCCESS:
+ raise RuntimeError("pipeline failed to enter READY state")
+ datasource.pipeline_seek_for_gps(pipeline, segment[0].ns() / 1000000000, segment[1].ns() / 1000000000)
if pipeline.set_state(Gst.State.PLAYING) == Gst.StateChangeReturn.FAILURE:
raise RuntimeError("pipeline failed to enter PLAYING state")
pipeparts.write_dump_dot(pipeline, "test_%s" % name, verbose = True)
--
GitLab