diff --git a/gstlal-ugly/bin/gstlal_feature_extractor_pipe b/gstlal-ugly/bin/gstlal_feature_extractor_pipe index 5dfa3c9b3c1e5cf598a2f6a515995279048c9edb..fb75a5d866a7c300f415ee1100cd18084e33f967 100755 --- a/gstlal-ugly/bin/gstlal_feature_extractor_pipe +++ b/gstlal-ugly/bin/gstlal_feature_extractor_pipe @@ -65,12 +65,8 @@ def breakupseg(seg, maxextent, overlap): if abs(seg) < maxextent: return segments.segmentlist([seg]) - # adjust maxextent so that segments are divided roughly equally - maxextent = max(int(abs(seg) / (int(abs(seg)) // int(maxextent) + 1)), overlap) - maxextent = int(math.ceil(abs(seg) / math.ceil(abs(seg) / maxextent))) - end = seg[1] - seglist = segments.segmentlist() + end = seg[1] while abs(seg): if (seg[0] + maxextent + overlap) < end: @@ -151,6 +147,7 @@ def feature_extractor_node_gen(gstlalFeatureExtractorJob, dag, parent_nodes, seg "channel-name":' '.join(channels), "job-id":str(ii + 1).zfill(4), "cadence":options.cadence, + "persist-cadence":options.persist_cadence, "max-streams":options.max_serial_streams, "disable-web-service":options.disable_web_service, "local-frame-caching":options.local_frame_caching, @@ -183,7 +180,8 @@ def parse_command_line(): parser.add_option("--local-frame-caching", action = "store_true", help = "Pre-reads frame data and stores to local filespace.") parser.add_option("--description", metavar = "string", default = "GSTLAL_IDQ_TRIGGERS", help = "Set the filename description in which to save the output.") parser.add_option("--save-format", action = "store_true", default = "hdf5", help = "Specifies the save format (ascii or hdf5) of features written to disk. Default = hdf5") - parser.add_option("--cadence", type = "int", default = 32, help = "Rate at which to write trigger files to disk. Default = 32 seconds.") + parser.add_option("--cadence", type = "int", default = 20, help = "Rate at which to write trigger files to disk. Default = 20 seconds.") + parser.add_option("--persist-cadence", type = "int", default = 200, help = "Rate at which to persist trigger files to disk. Default = 200 seconds.") parser.add_option("-m", "--mismatch", type = "float", default = 0.05, help = "Mismatch between templates, mismatch = 1 - minimal match. Default = 0.05.") parser.add_option("-q", "--qhigh", type = "float", default = 100, help = "Q high value for half sine-gaussian waveforms. Default = 100.") parser.add_option("--max-parallel-streams", type = "int", default = 50, help = "Number of streams (sum(channel_i * num_rates_i)) to process in parallel. This gives the maximum number of channels to process for a given job. Default = 50.") @@ -192,7 +190,6 @@ def parse_command_line(): parser.add_option("--segment-length", type = "int", default = 6000, help = "Maximum segment length to process per job. Default = 6000 seconds.") parser.add_option("-l", "--latency", action = "store_true", help = "Print latency to output ascii file. Temporary.") parser.add_option("--waveform", metavar = "string", default = "half_sine_gaussian", help = "Specifies the waveform used for matched filtering. Possible options: (half_sine_gaussian, sine_gaussian). Default = half_sine_gaussian") - parser.add_option("--save-hdf", action = "store_true", default = False, help = "If set, will save hdf5 files to disk straight from dataframe once every cadence") parser.add_option("--out-path", metavar = "path", default = ".", help = "Write to this path. Default = .") # Condor commands diff --git a/gstlal-ugly/share/feature_extractor/Makefile.gstlal_feature_extractor_offline b/gstlal-ugly/share/feature_extractor/Makefile.gstlal_feature_extractor_offline index b897edb32d686030ebaa429ddb7242f1147b39db..a2fcd59b23e353b5647b79368ebda6f1a13db3f6 100644 --- a/gstlal-ugly/share/feature_extractor/Makefile.gstlal_feature_extractor_offline +++ b/gstlal-ugly/share/feature_extractor/Makefile.gstlal_feature_extractor_offline @@ -27,19 +27,25 @@ CHANNEL_LIST = H1_O2_standard_channel_list.txt # NOTE: * if max_serial_streams > max_parallel_streams, all jobs will be parallelized by channel # * if max_parallel_streams > num_channels in channel list, all jobs will be processed serially, with processing driven by max_serial_streams # * any other combination will produce a mix of parallelization by channels and processing channels serially per job -MAX_PARALLEL_STREAMS = 300 -MAX_SERIAL_STREAMS = 110 +MAX_PARALLEL_STREAMS = 600 +MAX_SERIAL_STREAMS = 210 # Maximum number of concurrent reads from the same frame file, done to prevent I/O locks CONCURRENCY = 1 -# Length of time to process for a given job -SEGMENT_LENGTH = 3500 +# length of time to process for a given job +SEGMENT_LENGTH = 4000 -# Waveform type and parameter space settings -WAVEFORM = half_sine_gaussian -MISMATCH = 0.05 -QHIGH = 100 +# Parameter space config of waveforms +MISMATCH = 0.03 +QHIGH = 40 + +# waveforms +WAVEFORM = sine_gaussian + +# save preferences +SAVE_CADENCE = 20 +PERSIST_CADENCE = 200 # Detector CLUSTER:=$(shell hostname -d) @@ -99,9 +105,11 @@ dag : frame.cache plots $(CHANNEL_LIST) segments.xml.gz --frame-segments-file segments.xml.gz \ --frame-segments-name datasegments \ --local-frame-caching \ - --waveform $(WAVEFORM) \ + --cadence $(SAVE_CADENCE) \ + --persist-cadence $(PERSIST_CADENCE) \ --channel-list $(CHANNEL_LIST) \ --out-path $(OUTPATH) \ + --waveform $(WAVEFORM) \ --max-serial-streams $(MAX_SERIAL_STREAMS) \ --max-parallel-streams $(MAX_PARALLEL_STREAMS) \ --concurrency $(CONCURRENCY) \ @@ -110,7 +118,7 @@ dag : frame.cache plots $(CHANNEL_LIST) segments.xml.gz --qhigh $(QHIGH) \ $(CONDOR_COMMANDS) \ --request-cpu 2 \ - --request-memory 14GB \ + --request-memory 15GB \ --request-disk 12GB \ --verbose \ --disable-web-service