gstlal_feature_extractor_pipe: changed segment splitting so that segments do...

gstlal_feature_extractor_pipe: changed segment splitting so that segments do not get shifted from segment_length arg given, added options in offline Makefile to request save and persist cadence, updated params to reflect more closely offline runs

gstlal-ugly/bin/gstlal_feature_extractor_pipe

@@ -65,12 +65,8 @@ def breakupseg(seg, maxextent, overlap):
 	if abs(seg) < maxextent:
 		return segments.segmentlist([seg])
 
-	# adjust maxextent so that segments are divided roughly equally
-	maxextent = max(int(abs(seg) / (int(abs(seg)) // int(maxextent) + 1)), overlap)
-	maxextent = int(math.ceil(abs(seg) / math.ceil(abs(seg) / maxextent)))
-	end = seg[1]
-
 	seglist = segments.segmentlist()
+	end = seg[1]
 
 	while abs(seg):
 		if (seg[0] + maxextent + overlap) < end:
@@ -151,6 +147,7 @@ def feature_extractor_node_gen(gstlalFeatureExtractorJob, dag, parent_nodes, seg
 						"channel-name":' '.join(channels),
 						"job-id":str(ii + 1).zfill(4),
 						"cadence":options.cadence,
+						"persist-cadence":options.persist_cadence,
 						"max-streams":options.max_serial_streams,
 						"disable-web-service":options.disable_web_service,
 						"local-frame-caching":options.local_frame_caching,
@@ -183,7 +180,8 @@ def parse_command_line():
 	parser.add_option("--local-frame-caching", action = "store_true", help = "Pre-reads frame data and stores to local filespace.")
 	parser.add_option("--description", metavar = "string", default = "GSTLAL_IDQ_TRIGGERS", help = "Set the filename description in which to save the output.")
 	parser.add_option("--save-format", action = "store_true", default = "hdf5", help = "Specifies the save format (ascii or hdf5) of features written to disk. Default = hdf5")
-	parser.add_option("--cadence", type = "int", default = 32, help = "Rate at which to write trigger files to disk. Default = 32 seconds.")
+	parser.add_option("--cadence", type = "int", default = 20, help = "Rate at which to write trigger files to disk. Default = 20 seconds.")
+	parser.add_option("--persist-cadence", type = "int", default = 200, help = "Rate at which to persist trigger files to disk. Default = 200 seconds.")
 	parser.add_option("-m", "--mismatch", type = "float", default = 0.05, help = "Mismatch between templates, mismatch = 1 - minimal match. Default = 0.05.")
 	parser.add_option("-q", "--qhigh", type = "float", default = 100, help = "Q high value for half sine-gaussian waveforms. Default = 100.")
 	parser.add_option("--max-parallel-streams", type = "int", default = 50, help = "Number of streams (sum(channel_i * num_rates_i)) to process in parallel. This gives the maximum number of channels to process for a given job. Default = 50.")
@@ -192,7 +190,6 @@ def parse_command_line():
 	parser.add_option("--segment-length", type = "int", default = 6000, help = "Maximum segment length to process per job. Default = 6000 seconds.")
 	parser.add_option("-l", "--latency", action = "store_true", help = "Print latency to output ascii file. Temporary.")
 	parser.add_option("--waveform", metavar = "string", default = "half_sine_gaussian", help = "Specifies the waveform used for matched filtering. Possible options: (half_sine_gaussian, sine_gaussian). Default = half_sine_gaussian")
-	parser.add_option("--save-hdf", action = "store_true", default = False, help = "If set, will save hdf5 files to disk straight from dataframe once every cadence")
 	parser.add_option("--out-path", metavar = "path", default = ".", help = "Write to this path. Default = .")
 
 	# Condor commands

gstlal-ugly/share/feature_extractor/Makefile.gstlal_feature_extractor_offline

@@ -27,19 +27,25 @@ CHANNEL_LIST = H1_O2_standard_channel_list.txt
 # NOTE: * if max_serial_streams > max_parallel_streams, all jobs will be parallelized by channel
 #       * if max_parallel_streams > num_channels in channel list, all jobs will be processed serially, with processing driven by max_serial_streams
 #       * any other combination will produce a mix of parallelization by channels and processing channels serially per job
-MAX_PARALLEL_STREAMS = 300
-MAX_SERIAL_STREAMS = 110
+MAX_PARALLEL_STREAMS = 600
+MAX_SERIAL_STREAMS = 210
 
 # Maximum number of concurrent reads from the same frame file, done to prevent I/O locks
 CONCURRENCY = 1
 
-# Length of time to process for a given job
-SEGMENT_LENGTH = 3500
+# length of time to process for a given job
+SEGMENT_LENGTH = 4000
 
-# Waveform type and parameter space settings
-WAVEFORM = half_sine_gaussian
-MISMATCH = 0.05
-QHIGH = 100
+# Parameter space config of waveforms
+MISMATCH = 0.03
+QHIGH = 40
+
+# waveforms
+WAVEFORM = sine_gaussian
+
+# save preferences
+SAVE_CADENCE = 20
+PERSIST_CADENCE = 200
 
 # Detector
 CLUSTER:=$(shell hostname -d)
@@ -99,9 +105,11 @@ dag : frame.cache plots $(CHANNEL_LIST) segments.xml.gz
 		--frame-segments-file segments.xml.gz \
 		--frame-segments-name datasegments \
 		--local-frame-caching \
-		--waveform $(WAVEFORM) \
+		--cadence $(SAVE_CADENCE) \
+		--persist-cadence $(PERSIST_CADENCE) \
 		--channel-list $(CHANNEL_LIST) \
 		--out-path $(OUTPATH) \
+		--waveform $(WAVEFORM) \
 		--max-serial-streams $(MAX_SERIAL_STREAMS) \
 		--max-parallel-streams $(MAX_PARALLEL_STREAMS) \
 		--concurrency $(CONCURRENCY) \
@@ -110,7 +118,7 @@ dag : frame.cache plots $(CHANNEL_LIST) segments.xml.gz
 		--qhigh $(QHIGH) \
 		$(CONDOR_COMMANDS) \
 		--request-cpu 2 \
-		--request-memory 14GB \
+		--request-memory 15GB \
 		--request-disk 12GB \
 		--verbose \
 		--disable-web-service