Gitlab will migrate to a new storage backend starting 0300 UTC on 2020-04-04. We do not anticipate a maintenance window for this migration. Performance may be impacted over the weekend. Thanks for your patience.

Commit 95043e3b authored by Patrick Godwin's avatar Patrick Godwin

snax: update naming throughout, including base features directory. add...

snax: update naming throughout, including base features directory. add --features-path to offline DAG
parent 7a2c2c4f
......@@ -169,6 +169,9 @@ def parse_command_line():
parser.add_option("--request-disk", default = "50GB", metavar = "integer", help = "set the requested node local scratch space size needed, default = 50GB")
parser.add_option("--condor-command", action = "append", default = [], metavar = "command=value", help = "set condor commands of the form command=value; can be given multiple times")
# Feature saving options
parser.add_option("--features-path", metavar = "path", help = "If set, chooses an alternate directory to save the features to. Default = --out-path")
options, filenames = parser.parse_args()
# set max parallel streams to options.max_streams for use in data_source_info for splitting up channel lists to process in parallel
......@@ -242,6 +245,9 @@ feature_combiner_options = {
"tag": "offline",
}
if options.features_path:
feature_combiner_options.update({"outdir": options.features_path})
for seg in seglist_range(data_source_info.seg[0], data_source_info.seg[1], 50000):
parent_nodes = [node for (i, job_seg), node in feature_extractor_nodes.items() if seg.intersects(job_seg)]
these_options = dict(feature_combiner_options)
......
......@@ -259,7 +259,7 @@ duration = options.feature_end_time - options.feature_start_time
logdir = os.path.join(options.out_path, 'logs', options.job_id)
aggregator.makedir(logdir)
logger = utils.get_logger('gstlal-feature-extractor_%d-%d' % (options.feature_start_time, duration), rootdir=logdir, verbose=options.verbose)
logger = utils.get_logger('gstlal_snax_extract: %d-%d' % (options.feature_start_time, options.feature_end_time), rootdir=logdir, verbose=options.verbose)
logger.info("writing log to %s" % logdir)
#
......
......@@ -55,7 +55,7 @@ def parse_command_line():
group.add_option("--log-level", type = "int", default = 10, help = "Sets the verbosity of logging. Default = 10.")
group.add_option("--rootdir", metavar = "path", default = ".", help = "Sets the root directory where logs and metadata are stored.")
group.add_option("--features-path", metavar = "path", default = ".", help = "Write features to this path. Default = .")
group.add_option("--basename", metavar = "string", default = "GSTLAL_IDQ_FEATURES", help = "Sets the basename for files written to disk. Default = GSTLAL_IDQ_FEATURES")
group.add_option("--basename", metavar = "string", default = "SNAX_FEATURES", help = "Sets the basename for files written to disk. Default = SNAX_FEATURES")
group.add_option("--instrument", metavar = "string", default = "H1", help = "Sets the instrument for files written to disk. Default = H1")
group.add_option("--tag", metavar = "string", default = "test", help = "Sets the name of the tag used. Default = 'test'")
group.add_option("--waveform", type="string", default = "sine_gaussian", help = "Set the waveform used for producing features. Default = sine_gaussian.")
......
......@@ -513,7 +513,7 @@ def append_options(parser):
group = optparse.OptionGroup(parser, "Data Saving Options", "Adjust parameters used for saving/persisting features to disk as well as directories specified")
group.add_option("--out-path", metavar = "path", default = ".", help = "Write to this path. Default = .")
group.add_option("--description", metavar = "string", default = "GSTLAL_IDQ_FEATURES", help = "Set the filename description in which to save the output.")
group.add_option("--description", metavar = "string", default = "SNAX_FEATURES", help = "Set the filename description in which to save the output.")
group.add_option("--save-format", metavar = "string", default = "hdf5", help = "Specifies the save format (hdf5/kafka/bottle) of features written to disk. Default = hdf5")
group.add_option("--feature-mode", metavar = "string", default = "timeseries", help = "Specifies the mode for which features are generated (timeseries/etg). Default = timeseries")
group.add_option("--data-transfer", metavar = "string", default = "table", help = "Specifies the format of features transferred over-the-wire (table/row). Default = table")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment