Skip to content
Snippets Groups Projects
Commit b7d87c57 authored by Patrick Godwin's avatar Patrick Godwin
Browse files

update fake frames tutorial docs + example files for OSG dag workflow

parent e89f2c1b
No related branches found
No related tags found
1 merge request!41DAG Workflow Overhaul + OSG DAG support
Hide whitespace changes
Inline Side-by-side
doc/source/tutorials/gstlal_fake_data_overview.rst
+ 9
− 9
View file @ b7d87c57
......@@ -50,12 +50,11 @@ This procedure assumes you are on an LDG cluster which has the data you wish to
recolor. Note that some of the tools required on not gstlal based. Please
consult the documentation for the external tools should you have questions.
1. First obtain segments for the data using ``ligolw_segment_query_dqsegdb``
2. Next obtain the frame file cache from ``gw_data_find``
3. Then create the PSD you wish to recolor to (perhaps using ``gstlal_psd_xml_from_asd_txt``)
4. compute a reference spectrum from the frame data that you wish to recolor using ``gstlal_reference_psd``
5. You might choose to optionally "smooth" the reference spectrum in order to leave lines in the underlying data. You can try using ``gstlal_psd_polyfit``
6. Now with segments, a frame cache, a PSD (possibly smoothed) measured from the frame cache, and a PSD that is the target for the recolored spectrum, you are free to use ``gstlal_fake_frames`` according to the documentation.
1. First obtain segments for the data using ``ligolw_query_gwosc_segments``
2. Then create the PSD you wish to recolor to (perhaps using ``gstlal_psd_xml_from_asd_txt``)
3. compute a reference spectrum from the strain data that you wish to recolor using ``gstlal_reference_psd``
4. You might choose to optionally "smooth" the reference spectrum in order to leave lines in the underlying data. You can try using ``gstlal_psd_polyfit``
5. Now with segments, a PSD (possibly smoothed) measured from the strain data, and a PSD that is the target for the recolored spectrum, you are free to use ``gstlal_fake_frames`` according to the documentation.
Recoloring existing data with a HTCondor dag
--------------------------------------------
......@@ -68,13 +67,14 @@ input to the condor dag script has itself been automated in makefiles such as
As an example try this::
$ wget https://git.ligo.org/lscsoft/gstlal/raw/master/gstlal/share/Makefile.2015recolored
$ wget https://git.ligo.org/lscsoft/gstlal/raw/master/gstlal/share/recolored_config.yml
$ make -f Makefile.2015recolored
$ condor_submit_dag gstlal_fake_frames_pipe.dag
$ condor_submit_dag fake_frames_dag.dag
You can monitor the dag progress with::
$ tail -f gstlal_fake_frames_pipe.dag.dagman.out
$ tail -f fake_frames_dag.dag.dagman.out
You should have directories called LIGO and Virgo that contain the recolored frame data. Experiment
You should have a ``/frames`` directory that contains the recolored frame data. Experiment
with changing parameters in the Makefile to generate different PSDs, create frames over different stretches
of data, etc.
gstlal/share/Makefile.2015recolored
+ 14
− 73
View file @ b7d87c57
## @file Makefile.2015recolored
# Makefile for recoloring data to the 2015 noise curves
## Accounting info
GROUP_USER = albert.einstein
ACCOUNTING_TAG = ligo.dev.o2.cbc.explore.test
# Configuration file name
CONFIG = recolor_config.yml
## General info
MIN_SEG_LENGTH = 1000
START = 1187000000
STOP = 1187004000
TRIM = 16
## Noise curves
LIGODESIGNPSD = early_aligo_asd.txt
VIRGODESIGNPSD = v1_early_asd.txt
# controls input data
LIGOSCIENCE = DCH-CLEAN_SCIENCE_C02:1
VIRGOSCIENCE = ITF_SCIENCE
LIGOFRAMETYPEIN = 1_CLEANED_HOFT_C02
VIRGOFRAMETYPEIN = 1O2Repro2A
LIGOINCHANNEL = DCH-CLEAN_STRAIN_C02
VIRGOINCHANNEL = Hrec_hoft_V1O2Repro2A_16384Hz
# controls output data
LIGOFRAMETYPE = T1200307_V4_EARLY_RECOLORED_V2
VIRGOFRAMETYPE = T1300121_V1_EARLY_RECOLORED_V2
LIGOOUTPATH = LIGO
VIRGOOUTPATH = Virgo
LIGOOUTCHANNEL = LDAS-STRAIN
VIRGOOUTCHANNEL = h_16384Hz
# function to extract configuration parameters
yaml = $(shell gstlal_inspiral_config $(CONFIG) $(1))
all : dag
%segmentspadded.xml:
ligolw_segment_query_dqsegdb --segment-url=https://segments.ligo.org -q --gps-start-time ${START} --gps-end-time ${STOP} --include-segments=$*:$(LIGOSCIENCE) --result-name=datasegments > $@
V1segmentspadded.xml:
ligolw_segment_query_dqsegdb --segment-url=https://segments.ligo.org -q --gps-start-time ${START} --gps-end-time ${STOP} --include-segments=V1:$(VIRGOSCIENCE) --result-name=datasegments > $@
frame.%1.cache:
gw_data_find -o $* -t $*$(LIGOFRAMETYPEIN) -l -s $(START) -e $(STOP) --url-type file > $@
frame.V1.cache:
gw_data_find -o V -t V$(VIRGOFRAMETYPEIN) -l -s $(START) -e $(STOP) --url-type file > $@
frame.cache: frame.H1.cache frame.L1.cache frame.V1.cache
cat frame.H1.cache frame.L1.cache frame.V1.cache > $@
segments.xml: H1segmentspadded.xml L1segmentspadded.xml V1segmentspadded.xml
ligolw_add --ilwdchar-compat --output $@ H1segmentspadded.xml L1segmentspadded.xml V1segmentspadded.xml
ligolw_no_ilwdchar $@
segments.xml.gz:
gstlal_query_gwosc_segments $(call yaml,start) $(call yaml,stop) $(call yaml,ifos)
gstlal_segments_trim --trim 0 --gps-start-time $(call yaml,start) --gps-end-time $(call yaml,stop) --min-length 512 --output $@ $@
$(LIGODESIGNPSD) :
wget https://git.ligo.org/lscsoft/gstlal/raw/master/gstlal/share/$@
......@@ -66,36 +32,11 @@ V1recolorpsd.xml.gz : $(VIRGODESIGNPSD)
recolor_psd.xml.gz: H1recolorpsd.xml.gz L1recolorpsd.xml.gz V1recolorpsd.xml.gz
ligolw_add H1recolorpsd.xml.gz L1recolorpsd.xml.gz V1recolorpsd.xml.gz | gzip > $@
dag : segments.xml frame.cache recolor_psd.xml.gz
gstlal_fake_frames_pipe \
--color-psd recolor_psd.xml.gz \
--frame-segments-file segments.xml \
--frame-segments-name datasegments \
--min-segment-length $(MIN_SEG_LENGTH) \
--frame-cache frame.cache \
--channel-name=H1=$(LIGOINCHANNEL) \
--channel-name=L1=$(LIGOINCHANNEL) \
--channel-name=V1=$(VIRGOINCHANNEL) \
--output-channel-name=H1=$(LIGOOUTCHANNEL) \
--output-channel-name=L1=$(LIGOOUTCHANNEL) \
--output-channel-name=V1=$(VIRGOOUTCHANNEL) \
--output-path=H1=$(LIGOOUTPATH) \
--output-path=L1=$(LIGOOUTPATH) \
--output-path=V1=$(VIRGOOUTPATH) \
--frame-type=H1=$(LIGOFRAMETYPE) \
--frame-type=L1=$(LIGOFRAMETYPE) \
--frame-type=V1=$(VIRGOFRAMETYPE) \
--whiten-type medianofpsdperseg \
--frame-duration 16 \
--frames-per-file 256 \
--accounting-group-user $(GROUP_USER) \
--accounting-group $(ACCOUNTING_TAG) \
finish :
ls $(LIGOOUTPATH)/*/*.gwf | lalapps_path2cache > recolor_frame.cache
ls $(VIRGOOUTPATH)/*/*.gwf | lalapps_path2cache >> recolor_frame.cache
gstlal_cache_to_segments recolor_frame.cache recolor_segments.xml.gz
gstlal_segments_trim --trim 4 --min-length $(MIN_SEG_LENGTH) --output recolor_segments_trimmed.xml.gz recolor_segments.xml.gz
.PHONY: dag
dag : segments.xml.gz recolor_psd.xml.gz
gstlal_fake_frames_workflow -c $(CONFIG)
clean :
rm -rf *.sub *.dag* *.cache *.sh *reference_psd.xml.gz *segments.xml.gz logs *.xml *.xml.gz recolor_frame.cache recolor_frame.cache recolor_segments.xml.gz $(LIGODESIGNPSD) $(VIRGODESIGNPSD)
rm -rf *.sub *.dag* *.sh *reference_psd.xml.gz *segments.xml.gz logs *.xml *.xml.gz
rm -rf $(LIGODESIGNPSD) $(VIRGODESIGNPSD)
rm -rf frames reference_psd median_psd
gstlal/share/Makefile.am
+ 1
− 0
View file @ b7d87c57
......@@ -2,6 +2,7 @@ dist_pkgdata_DATA = \
AdvVirgoDesign.txt \
Makefile.2015recolored \
early_aligo_asd.txt \
recolor_config.yml \
v1_early_asd.txt \
ZERO_DET_high_P.txt \
ZERO_DET_low_P.txt
gstlal/share/recolor_config.yml 0 → 100644
+ 42
− 0
View file @ b7d87c57
# general options
start: 1187000000
stop: 1187100000
ifos: H1L1V1
# data discovery options
source:
data-find-server: datafind.gw-openscience.org
frame-type:
H1: H1_GWOSC_O2_16KHZ_R1
L1: L1_GWOSC_O2_16KHZ_R1
V1: V1_GWOSC_O2_16KHZ_R1
channel-name:
H1: GWOSC-16KHZ_R1_STRAIN
L1: GWOSC-16KHZ_R1_STRAIN
V1: GWOSC-16KHZ_R1_STRAIN
sample-rate: 4096
frame-segments-file: segments.xml.gz
frame-segments-name: datasegments
# frame generation options
frames:
output-channel-name:
H1: RECOLORED_STRAIN
L1: RECOLORED_STRAIN
V1: RECOLORED_STRAIN
output-frame-type:
H1: T1200307_V4_EARLY_RECOLORED_V2
L1: T1200307_V4_EARLY_RECOLORED_V2
V1: T1300121_V1_EARLY_RECOLORED_V2
whiten-type: median_psd
track-psd: true
# psd options
psd:
fft-length: 8
sample-rate: 4096
# condor options
condor:
accounting-group: ligo.dev.o2.cbc.explore.test
profile: ldas
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment