From 40099491430cca21640e18a4e3a680563b1ac95d Mon Sep 17 00:00:00 2001
From: Patrick Godwin <patrick.godwin@ligo.org>
Date: Mon, 29 Oct 2018 12:19:31 -0700
Subject: [PATCH] update ll feature extraction makefile + exec with new
options, parameters for online jobs
---
.../bin/gstlal_ll_feature_extractor_pipe | 3 +
.../Makefile.gstlal_feature_extractor_online | 64 ++++++++++++-------
2 files changed, 44 insertions(+), 23 deletions(-)
diff --git a/gstlal-burst/bin/gstlal_ll_feature_extractor_pipe b/gstlal-burst/bin/gstlal_ll_feature_extractor_pipe
index e9eac24169..a0cfbad789 100755
--- a/gstlal-burst/bin/gstlal_ll_feature_extractor_pipe
+++ b/gstlal-burst/bin/gstlal_ll_feature_extractor_pipe
@@ -248,6 +248,7 @@ def parse_command_line():
# Synchronizer/File Sink commands
group = optparse.OptionGroup(parser, "Synchronizer/File Sink Options", "Adjust parameters used for synchronization and dumping of features to disk.")
parser.add_option("--tag", metavar = "string", default = "test", help = "Sets the name of the tag used. Default = 'test'")
+ parser.add_option("--no-drop", default=False, action="store_true", help = "If set, do not drop incoming features based on the latency timeout. Default = False.")
parser.add_option("--processing-cadence", type = "float", default = 0.1, help = "Rate at which the streaming jobs acquire and processes data. Default = 0.1 seconds.")
parser.add_option("--request-timeout", type = "float", default = 0.2, help = "Timeout for requesting messages from a topic. Default = 0.2 seconds.")
parser.add_option("--latency-timeout", type = "float", default = 5, help = "Maximum time before incoming data is dropped for a given timestamp. Default = 5 seconds.")
@@ -329,6 +330,8 @@ if options.save_format == 'kafka':
"input-topic-basename": options.kafka_topic,
"output-topic-basename": '_'.join(['synchronizer', options.tag])
}
+ if options.no_drop:
+ synchronizer_options.update({"no-drop": options.no_drop})
hdf5_sink_options = {
"instrument": ifo,
diff --git a/gstlal-burst/share/feature_extractor/Makefile.gstlal_feature_extractor_online b/gstlal-burst/share/feature_extractor/Makefile.gstlal_feature_extractor_online
index bbe631fbb9..144cec26e3 100644
--- a/gstlal-burst/share/feature_extractor/Makefile.gstlal_feature_extractor_online
+++ b/gstlal-burst/share/feature_extractor/Makefile.gstlal_feature_extractor_online
@@ -94,19 +94,22 @@ ACCOUNTING_TAG=ligo.dev.o3.detchar.onlinedq.idq
GROUP_USER=albert.einstein
CONDOR_COMMANDS:=--condor-command=accounting_group=$(ACCOUNTING_TAG) --condor-command=accounting_group_user=$(GROUP_USER)
+CONDOR_UNIVERSE=vanilla
+#CONDOR_UNIVERSE=local
+
#########################
# Online DAG Parameters #
#########################
TAG = online_test
-DATA_SOURCE = lvshm
-MAX_STREAMS = 200
-SAMPLE_RATE = 1
+DATA_SOURCE = framexmit
+MAX_STREAMS = 50
+SAMPLE_RATE = 16
# Parameter space config of waveform
-WAVEFORM = sine_gaussian
-MISMATCH = 0.02
+WAVEFORM = tapered_sine_gaussian
+MISMATCH = 0.03
QHIGH = 40
# data transfer options
@@ -116,7 +119,7 @@ SAVE_FORMAT = kafka
# save options
SAVE_CADENCE = 20
-PERSIST_CADENCE = 200
+PERSIST_CADENCE = 20
# kafka options
KAFKA_TOPIC = gstlal_features
@@ -124,8 +127,8 @@ KAFKA_SERVER = localhost:9182
KAFKA_GROUP = group_1
# synchronizer/file sink options (kafka only)
-PROCESSING_CADENCE = 0.1
-REQUEST_TIMEOUT = 0.2
+PROCESSING_CADENCE = 0.01
+REQUEST_TIMEOUT = 0.05
LATENCY_TIMEOUT = 12
# cluster where analysis is run
@@ -138,7 +141,7 @@ CLUSTER:=$(shell hostname -d)
IFO = H1
#IFO = L1
-EPOCH = O2
+EPOCH = O3
LEVEL = standard
# if not using standard .ini file, comment and supply custom channel list instead
@@ -204,9 +207,10 @@ dag : plots $(CHANNEL_LIST)
$(FIDELITY_EXCLUDE_COMMANDS) \
$(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \
--disable-kafka-jobs \
- --condor-universe local \
- --request-cpu 2 \
- --request-memory 15GB \
+ --no-drop \
+ --condor-universe $(CONDOR_UNIVERSE) \
+ --request-cpu 6 \
+ --request-memory 8GB \
--verbose \
--disable-web-service ; \
elif [[ $(DATA_SOURCE) == "framexmit" ]] && [[ $(SAVE_FORMAT) == "hdf5" ]] ; then \
@@ -228,9 +232,10 @@ dag : plots $(CHANNEL_LIST)
$(FIDELITY_EXCLUDE_COMMANDS) \
$(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \
--disable-kafka-jobs \
- --condor-universe local \
- --request-cpu 2 \
- --request-memory 15GB \
+ --no-drop \
+ --condor-universe $(CONDOR_UNIVERSE) \
+ --request-cpu 6 \
+ --request-memory 8GB \
--verbose \
--disable-web-service ; \
elif [[ $(DATA_SOURCE) == "lvshm" ]] && [[ $(SAVE_FORMAT) == "kafka" ]] ; then \
@@ -239,6 +244,8 @@ dag : plots $(CHANNEL_LIST)
--shared-memory-partition H1=LHO_RedDtchr \
--shared-memory-assumed-duration 1 \
--sample-rate $(SAMPLE_RATE) \
+ --cadence $(SAVE_CADENCE) \
+ --persist-cadence $(PERSIST_CADENCE) \
--save-format $(SAVE_FORMAT) \
--kafka-topic $(KAFKA_TOPIC) \
--kafka-server $(KAFKA_SERVER) \
@@ -259,9 +266,10 @@ dag : plots $(CHANNEL_LIST)
$(FIDELITY_EXCLUDE_COMMANDS) \
$(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \
--disable-kafka-jobs \
- --condor-universe local \
- --request-cpu 2 \
- --request-memory 15GB \
+ --no-drop \
+ --condor-universe $(CONDOR_UNIVERSE) \
+ --request-cpu 6 \
+ --request-memory 8GB \
--verbose \
--disable-web-service ; \
elif [[ $(DATA_SOURCE) == "framexmit" ]] && [[ $(SAVE_FORMAT) == "kafka" ]] ; then \
@@ -269,6 +277,8 @@ dag : plots $(CHANNEL_LIST)
--data-source $(DATA_SOURCE) \
--save-format $(SAVE_FORMAT) \
--sample-rate $(SAMPLE_RATE) \
+ --cadence $(SAVE_CADENCE) \
+ --persist-cadence $(PERSIST_CADENCE) \
--kafka-topic $(KAFKA_TOPIC) \
--kafka-server $(KAFKA_SERVER) \
--kafka-partition $(KAFKA_GROUP) \
@@ -288,16 +298,18 @@ dag : plots $(CHANNEL_LIST)
$(FIDELITY_EXCLUDE_COMMANDS) \
$(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \
--disable-kafka-jobs \
- --condor-universe local \
- --request-cpu 2 \
- --request-memory 15GB \
+ --no-drop \
+ --condor-universe $(CONDOR_UNIVERSE) \
+ --request-cpu 6 \
+ --request-memory 8GB \
--verbose \
--disable-web-service ; \
fi ;
# Pull latest channel list
$(CHANNEL_LIST) : plots
- wget https://git.ligo.org/detchar/ligo-channel-lists/raw/master/$(EPOCH)/$(CHANNEL_LIST)
+ #wget https://git.ligo.org/detchar/ligo-channel-lists/raw/master/$(EPOCH)/$(CHANNEL_LIST)
+ wget https://git.ligo.org/patrick.godwin/ligo-channel-lists/raw/lho_O3_updatelist/$(EPOCH)/$(CHANNEL_LIST)
# FIXME Add webpages once we have output
# Make webpage directory and copy files across
@@ -309,5 +321,11 @@ $(CHANNEL_LIST) : plots
plots :
mkdir plots
+clean-lite :
+ -rm -rvf *.sub *.dag* *.cache *.sh logs *.sqlite plots *.html Images *.css *.js *.txt
+
clean :
- -rm -rvf *.sub *.dag* *.cache *.sh logs *.sqlite plots *.html Images *.css *.js *.ini
+ -rm -rvf *.sub *.dag* *.cache *.sh logs *.sqlite plots *.html Images *.css *.js *.ini *.txt
+
+clean-all :
+ -rm -rvf *.sub *.dag* *.cache *.sh logs *.sqlite plots *.html Images *.css *.js *.ini *.txt gstlal_feature_*
--
GitLab