Commit 9394fd2a authored by Patrick Godwin's avatar Patrick Godwin
Browse files

gstlal_feature_combiner: allow saving features to a separate directory,...

gstlal_feature_combiner: allow saving features to a separate directory, optionally pass in start, end times
parent 27a55d79
Pipeline #71331 passed with stages
in 30 minutes and 53 seconds
......@@ -36,6 +36,9 @@ import shutil
import h5py
import numpy
from ligo.segments import segment, segmentlist
from gstlal import aggregator
from gstlal.fxtools import utils
# =============================================================================
......@@ -52,11 +55,14 @@ def parse_command_line():
group = optparse.OptionGroup(parser, "Combiner Options", "General settings for configuring the file combiner.")
group.add_option("-v","--verbose", default=False, action="store_true", help = "Print to stdout in addition to writing to automatically generated log.")
group.add_option("--start-time", type = "int", help = "Set the start time to combine features.")
group.add_option("--end-time", type = "int", help = "Set the end time to combine features.")
group.add_option("--log-level", type = "int", default = 10, help = "Sets the verbosity of logging. Default = 10.")
group.add_option("--rootdir", metavar = "path", default = ".", help = "Sets the root directory where features, logs, and metadata are stored.")
group.add_option("--basename", metavar = "string", default = "GSTLAL_IDQ_FEATURES", help = "Sets the basename for files written to disk. Default = GSTLAL_IDQ_FEATURES")
group.add_option("--instrument", metavar = "string", default = "H1", help = "Sets the instrument for files written to disk. Default = H1")
group.add_option("--tag", metavar = "string", default = "test", help = "Sets the name of the tag used. Default = 'test'")
group.add_option("--outdir", metavar = "path", help = "If set, chooses an alternate directory to save the features to. Default = --rootdir")
parser.add_option_group(group)
opts, args = parser.parse_args()
......@@ -81,6 +87,11 @@ if __name__ == "__main__":
verbose=options.verbose
)
### define gps bounds to grab features
start_time = options.start_time if options.start_time else -numpy.inf
end_time = options.end_time if options.end_time else numpy.inf
file_segs = segmentlist([segment(start_time, end_time)])
### get base temp directory
if '_CONDOR_SCRATCH_DIR' in os.environ:
tmp_dir = os.environ['_CONDOR_SCRATCH_DIR']
......@@ -93,6 +104,11 @@ if __name__ == "__main__":
ifo=options.instrument[0],
)
cache = sorted(utils.path2cache(options.rootdir, pattern), key=lambda x: x.segment)
### filter cache with segments
cache = [entry for entry in cache if file_segs.intersects_segment(entry.segment)]
### group by segment
grouped_cache = [(seg, list(group)) for seg, group in itertools.groupby(cache, key=lambda x: x.segment)]
### combine features in each stride
......@@ -122,7 +138,16 @@ if __name__ == "__main__":
for dset in features[channel].keys():
utils.create_new_dataset(tmp_dir, filename, features[channel][dset], name=dset, group=channel, tmp=True, metadata=metadata)
final_path = os.path.join(dirname, filename)+".h5"
### determine final location for features
if options.outdir:
start_time = int(filename.split('-')[2])
basename = '-'.join([options.instrument[0], options.basename])
base_path = utils.to_trigger_path(options.outdir, basename, start_time)
aggregator.makedir(base_path)
final_path = os.path.join(base_path, filename)+".h5"
else:
final_path = os.path.join(dirname, filename)+".h5"
tmp_path = os.path.join(tmp_dir, filename)+".h5.tmp"
logger.info('saving features to: {}'.format(final_path))
shutil.move(tmp_path, final_path)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment