gstlal_feature_extractor + idq_utils.py: modified method of saving features as...

gstlal_feature_extractor + idq_utils.py: modified method of saving features as hdf5 so that max_snr option produces regularly sampled feature rows now

gstlal-ugly/bin/gstlal_feature_extractor

@@ -221,6 +221,13 @@ class MultiChannelHandler(simplehandler.Handler):
 		self.basename = kwargs.pop("basename")
 		self.waveform_type = options.waveform
 
+		# format keys used for saving, etc.
+		self.aggregate_rate = True
+		if self.aggregate_rate:
+			self.keys = list(set([key[0] for key in self.keys]))
+		else:
+			self.keys = [os.path.join(key[0], str(key[1]).zfill(4)) for key in self.keys]
+
 		# format id for aesthetics
 		self.job_id = str(options.job_id).zfill(4)
 		self.subset_id = str(kwargs.pop("subset_id")).zfill(4)
@@ -246,7 +253,7 @@ class MultiChannelHandler(simplehandler.Handler):
 
 		# feature saving properties
 		if options.save_format == 'hdf5':
-			self.fdata = idq_utils.HDF5SeriesFeatureData(columns, keys = self.keys, cadence = self.cadence)
+			self.fdata = idq_utils.HDF5FeatureData(columns, keys = self.keys, cadence = self.cadence)
 
 			if self.is_live:
 				duration = idq_utils.floor_div(int(options.gps_start_time) + self.persist_cadence, self.persist_cadence) - int(options.gps_start_time)
@@ -410,7 +417,11 @@ class MultiChannelHandler(simplehandler.Handler):
 
 			# save iDQ compatible data
 			if options.save_format == 'hdf5':
-				self.fdata.append(feature_row, key = (channel, rate), buftime = buftime)
+				if self.aggregate_rate:
+					key = channel
+				else:
+					key = os.path.join(channel, str(rate).zfill(4))
+				self.fdata.append(feature_row, key = key, buftime = buftime)
 			elif options.save_format == 'ascii':
 				channel_tag = ('%s_%i_%i' %(channel, rate/4, rate/2)).replace(":","_",1)
 				self.fdata.append("%20.9f\t%20.9f\t%20.9f\t%10.3f\t%8.3f\t%8.3f\t%8.3f\t%10.3f\t%s\n" % (start_time, stop_time, trigger_time, freq, row.phase, q, row.chisq, row.snr, channel_tag))

gstlal-ugly/python/idq_utils.py

@@ -217,8 +217,11 @@ class HDF5FeatureData(FeatureData):
 	"""
 	def __init__(self, columns, keys, **kwargs):
 		super(HDF5FeatureData, self).__init__(columns, keys = keys, **kwargs)
+		self.latency = 1
 		self.cadence = kwargs.pop('cadence')
-		self.feature_data = {key: numpy.empty((self.cadence,), dtype = [(column, 'f8') for column in self.columns]) for key in keys}
+		self.dtype = [(column, '<f8') for column in self.columns]
+		self.feature_data = {key: numpy.empty(((self.cadence+self.latency),), dtype = self.dtype) for key in keys}
+		self.last_save_time = None
 		self.clear()
 
 	def dump(self, path, base, start_time, key = None, tmp = False):
@@ -227,13 +230,11 @@ class HDF5FeatureData(FeatureData):
 		"""
 		name = "%d_%d" % (start_time, self.cadence)
 		if key:
-			group = os.path.join(str(key[0]), str(key[1]).zfill(4))
-			create_new_dataset(path, base, self.feature_data[key], name=name, group=group, tmp=tmp)
+			create_new_dataset(path, base, self.feature_data[key], name=name, group=key, tmp=tmp)
 			self.clear(key)
 		else:
 			for key in self.keys:
-				group = os.path.join(str(key[0]), str(key[1]).zfill(4))
-				create_new_dataset(path, base, self.feature_data[key], name=name, group=group, tmp=tmp)
+				create_new_dataset(path, base, self.feature_data[key], name=name, group=key, tmp=tmp)
 			self.clear()
 
 	def append(self, value, key = None, buftime = None):
@@ -241,8 +242,10 @@ class HDF5FeatureData(FeatureData):
 		Append a trigger row to data structure
 		"""
 		if buftime and key:
-			idx = buftime % self.cadence
-			self.feature_data[key][idx] = numpy.array([value[column] for column in self.columns])
+			self.last_save_time = floor_div(buftime, self.cadence)
+			idx = int(numpy.floor(value.trigger_time)) - self.last_save_time
+			if numpy.isnan(self.feature_data[key][idx][self.columns[0]]) or (value.snr > self.feature_data[key][idx]['snr']):
+				self.feature_data[key][idx] = numpy.array(value, dtype=self.dtype)
 
 	def clear(self, key = None):
 		if key:
@@ -268,13 +271,11 @@ class HDF5SeriesFeatureData(FeatureData):
 		"""
 		name = "%d_%d" % (start_time, self.cadence)
 		if key:
-			group = os.path.join(str(key[0]), str(key[1]).zfill(4))
-			create_new_dataset(path, base, numpy.array(self.feature_data[key], dtype=self.dtype), name=name, group=group, tmp=tmp)
+			create_new_dataset(path, base, numpy.array(self.feature_data[key], dtype=self.dtype), name=name, group=key, tmp=tmp)
 			self.clear(key)
 		else:
 			for key in self.keys:
-				group = os.path.join(str(key[0]), str(key[1]).zfill(4))
-				create_new_dataset(path, base, numpy.array(self.feature_data[key], dtype=self.dtype), name=name, group=group, tmp=tmp)
+				create_new_dataset(path, base, numpy.array(self.feature_data[key], dtype=self.dtype), name=name, group=key, tmp=tmp)
 			self.clear()
 
 	def append(self, value, key = None, buftime = None):