Skip to content
Snippets Groups Projects
Commit 40099491 authored by Patrick Godwin's avatar Patrick Godwin
Browse files

update ll feature extraction makefile + exec with new options, parameters for online jobs

parent e5511709
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
gstlal-burst/bin/gstlal_ll_feature_extractor_pipe
+ 3
− 0
View file @ 40099491
...@@ -248,6 +248,7 @@ def parse_command_line(): ...@@ -248,6 +248,7 @@ def parse_command_line():
# Synchronizer/File Sink commands # Synchronizer/File Sink commands
group = optparse.OptionGroup(parser, "Synchronizer/File Sink Options", "Adjust parameters used for synchronization and dumping of features to disk.") group = optparse.OptionGroup(parser, "Synchronizer/File Sink Options", "Adjust parameters used for synchronization and dumping of features to disk.")
parser.add_option("--tag", metavar = "string", default = "test", help = "Sets the name of the tag used. Default = 'test'") parser.add_option("--tag", metavar = "string", default = "test", help = "Sets the name of the tag used. Default = 'test'")
parser.add_option("--no-drop", default=False, action="store_true", help = "If set, do not drop incoming features based on the latency timeout. Default = False.")
parser.add_option("--processing-cadence", type = "float", default = 0.1, help = "Rate at which the streaming jobs acquire and processes data. Default = 0.1 seconds.") parser.add_option("--processing-cadence", type = "float", default = 0.1, help = "Rate at which the streaming jobs acquire and processes data. Default = 0.1 seconds.")
parser.add_option("--request-timeout", type = "float", default = 0.2, help = "Timeout for requesting messages from a topic. Default = 0.2 seconds.") parser.add_option("--request-timeout", type = "float", default = 0.2, help = "Timeout for requesting messages from a topic. Default = 0.2 seconds.")
parser.add_option("--latency-timeout", type = "float", default = 5, help = "Maximum time before incoming data is dropped for a given timestamp. Default = 5 seconds.") parser.add_option("--latency-timeout", type = "float", default = 5, help = "Maximum time before incoming data is dropped for a given timestamp. Default = 5 seconds.")
...@@ -329,6 +330,8 @@ if options.save_format == 'kafka': ...@@ -329,6 +330,8 @@ if options.save_format == 'kafka':
"input-topic-basename": options.kafka_topic, "input-topic-basename": options.kafka_topic,
"output-topic-basename": '_'.join(['synchronizer', options.tag]) "output-topic-basename": '_'.join(['synchronizer', options.tag])
} }
if options.no_drop:
synchronizer_options.update({"no-drop": options.no_drop})
hdf5_sink_options = { hdf5_sink_options = {
"instrument": ifo, "instrument": ifo,
......
gstlal-burst/share/feature_extractor/Makefile.gstlal_feature_extractor_online
+ 41
− 23
View file @ 40099491
...@@ -94,19 +94,22 @@ ACCOUNTING_TAG=ligo.dev.o3.detchar.onlinedq.idq ...@@ -94,19 +94,22 @@ ACCOUNTING_TAG=ligo.dev.o3.detchar.onlinedq.idq
GROUP_USER=albert.einstein GROUP_USER=albert.einstein
CONDOR_COMMANDS:=--condor-command=accounting_group=$(ACCOUNTING_TAG) --condor-command=accounting_group_user=$(GROUP_USER) CONDOR_COMMANDS:=--condor-command=accounting_group=$(ACCOUNTING_TAG) --condor-command=accounting_group_user=$(GROUP_USER)
CONDOR_UNIVERSE=vanilla
#CONDOR_UNIVERSE=local
######################### #########################
# Online DAG Parameters # # Online DAG Parameters #
######################### #########################
TAG = online_test TAG = online_test
DATA_SOURCE = lvshm DATA_SOURCE = framexmit
MAX_STREAMS = 200 MAX_STREAMS = 50
SAMPLE_RATE = 1 SAMPLE_RATE = 16
# Parameter space config of waveform # Parameter space config of waveform
WAVEFORM = sine_gaussian WAVEFORM = tapered_sine_gaussian
MISMATCH = 0.02 MISMATCH = 0.03
QHIGH = 40 QHIGH = 40
# data transfer options # data transfer options
...@@ -116,7 +119,7 @@ SAVE_FORMAT = kafka ...@@ -116,7 +119,7 @@ SAVE_FORMAT = kafka
# save options # save options
SAVE_CADENCE = 20 SAVE_CADENCE = 20
PERSIST_CADENCE = 200 PERSIST_CADENCE = 20
# kafka options # kafka options
KAFKA_TOPIC = gstlal_features KAFKA_TOPIC = gstlal_features
...@@ -124,8 +127,8 @@ KAFKA_SERVER = localhost:9182 ...@@ -124,8 +127,8 @@ KAFKA_SERVER = localhost:9182
KAFKA_GROUP = group_1 KAFKA_GROUP = group_1
# synchronizer/file sink options (kafka only) # synchronizer/file sink options (kafka only)
PROCESSING_CADENCE = 0.1 PROCESSING_CADENCE = 0.01
REQUEST_TIMEOUT = 0.2 REQUEST_TIMEOUT = 0.05
LATENCY_TIMEOUT = 12 LATENCY_TIMEOUT = 12
# cluster where analysis is run # cluster where analysis is run
...@@ -138,7 +141,7 @@ CLUSTER:=$(shell hostname -d) ...@@ -138,7 +141,7 @@ CLUSTER:=$(shell hostname -d)
IFO = H1 IFO = H1
#IFO = L1 #IFO = L1
EPOCH = O2 EPOCH = O3
LEVEL = standard LEVEL = standard
# if not using standard .ini file, comment and supply custom channel list instead # if not using standard .ini file, comment and supply custom channel list instead
...@@ -204,9 +207,10 @@ dag : plots $(CHANNEL_LIST) ...@@ -204,9 +207,10 @@ dag : plots $(CHANNEL_LIST)
$(FIDELITY_EXCLUDE_COMMANDS) \ $(FIDELITY_EXCLUDE_COMMANDS) \
$(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \ $(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \
--disable-kafka-jobs \ --disable-kafka-jobs \
--condor-universe local \ --no-drop \
--request-cpu 2 \ --condor-universe $(CONDOR_UNIVERSE) \
--request-memory 15GB \ --request-cpu 6 \
--request-memory 8GB \
--verbose \ --verbose \
--disable-web-service ; \ --disable-web-service ; \
elif [[ $(DATA_SOURCE) == "framexmit" ]] && [[ $(SAVE_FORMAT) == "hdf5" ]] ; then \ elif [[ $(DATA_SOURCE) == "framexmit" ]] && [[ $(SAVE_FORMAT) == "hdf5" ]] ; then \
...@@ -228,9 +232,10 @@ dag : plots $(CHANNEL_LIST) ...@@ -228,9 +232,10 @@ dag : plots $(CHANNEL_LIST)
$(FIDELITY_EXCLUDE_COMMANDS) \ $(FIDELITY_EXCLUDE_COMMANDS) \
$(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \ $(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \
--disable-kafka-jobs \ --disable-kafka-jobs \
--condor-universe local \ --no-drop \
--request-cpu 2 \ --condor-universe $(CONDOR_UNIVERSE) \
--request-memory 15GB \ --request-cpu 6 \
--request-memory 8GB \
--verbose \ --verbose \
--disable-web-service ; \ --disable-web-service ; \
elif [[ $(DATA_SOURCE) == "lvshm" ]] && [[ $(SAVE_FORMAT) == "kafka" ]] ; then \ elif [[ $(DATA_SOURCE) == "lvshm" ]] && [[ $(SAVE_FORMAT) == "kafka" ]] ; then \
...@@ -239,6 +244,8 @@ dag : plots $(CHANNEL_LIST) ...@@ -239,6 +244,8 @@ dag : plots $(CHANNEL_LIST)
--shared-memory-partition H1=LHO_RedDtchr \ --shared-memory-partition H1=LHO_RedDtchr \
--shared-memory-assumed-duration 1 \ --shared-memory-assumed-duration 1 \
--sample-rate $(SAMPLE_RATE) \ --sample-rate $(SAMPLE_RATE) \
--cadence $(SAVE_CADENCE) \
--persist-cadence $(PERSIST_CADENCE) \
--save-format $(SAVE_FORMAT) \ --save-format $(SAVE_FORMAT) \
--kafka-topic $(KAFKA_TOPIC) \ --kafka-topic $(KAFKA_TOPIC) \
--kafka-server $(KAFKA_SERVER) \ --kafka-server $(KAFKA_SERVER) \
...@@ -259,9 +266,10 @@ dag : plots $(CHANNEL_LIST) ...@@ -259,9 +266,10 @@ dag : plots $(CHANNEL_LIST)
$(FIDELITY_EXCLUDE_COMMANDS) \ $(FIDELITY_EXCLUDE_COMMANDS) \
$(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \ $(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \
--disable-kafka-jobs \ --disable-kafka-jobs \
--condor-universe local \ --no-drop \
--request-cpu 2 \ --condor-universe $(CONDOR_UNIVERSE) \
--request-memory 15GB \ --request-cpu 6 \
--request-memory 8GB \
--verbose \ --verbose \
--disable-web-service ; \ --disable-web-service ; \
elif [[ $(DATA_SOURCE) == "framexmit" ]] && [[ $(SAVE_FORMAT) == "kafka" ]] ; then \ elif [[ $(DATA_SOURCE) == "framexmit" ]] && [[ $(SAVE_FORMAT) == "kafka" ]] ; then \
...@@ -269,6 +277,8 @@ dag : plots $(CHANNEL_LIST) ...@@ -269,6 +277,8 @@ dag : plots $(CHANNEL_LIST)
--data-source $(DATA_SOURCE) \ --data-source $(DATA_SOURCE) \
--save-format $(SAVE_FORMAT) \ --save-format $(SAVE_FORMAT) \
--sample-rate $(SAMPLE_RATE) \ --sample-rate $(SAMPLE_RATE) \
--cadence $(SAVE_CADENCE) \
--persist-cadence $(PERSIST_CADENCE) \
--kafka-topic $(KAFKA_TOPIC) \ --kafka-topic $(KAFKA_TOPIC) \
--kafka-server $(KAFKA_SERVER) \ --kafka-server $(KAFKA_SERVER) \
--kafka-partition $(KAFKA_GROUP) \ --kafka-partition $(KAFKA_GROUP) \
...@@ -288,16 +298,18 @@ dag : plots $(CHANNEL_LIST) ...@@ -288,16 +298,18 @@ dag : plots $(CHANNEL_LIST)
$(FIDELITY_EXCLUDE_COMMANDS) \ $(FIDELITY_EXCLUDE_COMMANDS) \
$(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \ $(UNSAFE_CHANNEL_INCLUDE_COMMANDS) \
--disable-kafka-jobs \ --disable-kafka-jobs \
--condor-universe local \ --no-drop \
--request-cpu 2 \ --condor-universe $(CONDOR_UNIVERSE) \
--request-memory 15GB \ --request-cpu 6 \
--request-memory 8GB \
--verbose \ --verbose \
--disable-web-service ; \ --disable-web-service ; \
fi ; fi ;
# Pull latest channel list # Pull latest channel list
$(CHANNEL_LIST) : plots $(CHANNEL_LIST) : plots
wget https://git.ligo.org/detchar/ligo-channel-lists/raw/master/$(EPOCH)/$(CHANNEL_LIST) #wget https://git.ligo.org/detchar/ligo-channel-lists/raw/master/$(EPOCH)/$(CHANNEL_LIST)
wget https://git.ligo.org/patrick.godwin/ligo-channel-lists/raw/lho_O3_updatelist/$(EPOCH)/$(CHANNEL_LIST)
# FIXME Add webpages once we have output # FIXME Add webpages once we have output
# Make webpage directory and copy files across # Make webpage directory and copy files across
...@@ -309,5 +321,11 @@ $(CHANNEL_LIST) : plots ...@@ -309,5 +321,11 @@ $(CHANNEL_LIST) : plots
plots : plots :
mkdir plots mkdir plots
clean-lite :
-rm -rvf *.sub *.dag* *.cache *.sh logs *.sqlite plots *.html Images *.css *.js *.txt
clean : clean :
-rm -rvf *.sub *.dag* *.cache *.sh logs *.sqlite plots *.html Images *.css *.js *.ini -rm -rvf *.sub *.dag* *.cache *.sh logs *.sqlite plots *.html Images *.css *.js *.ini *.txt
clean-all :
-rm -rvf *.sub *.dag* *.cache *.sh logs *.sqlite plots *.html Images *.css *.js *.ini *.txt gstlal_feature_*
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment