Skip to content
Snippets Groups Projects
Commit d3038f99 authored by Duncan Brown's avatar Duncan Brown
Browse files

allow better control over input data and exlusion of segments

parent 5ac06875
No related branches found
No related tags found
No related merge requests found
......@@ -79,7 +79,8 @@ parser = OptionParser(
description = "Publishes XML files into the segment database")
parser.add_option("-r", "--server", metavar = "HOST:PORT", help = "connect to ldbd on HOST:PORT (default port 30020)")
parser.add_option("-d", "--spool-directory", metavar = "DIR", help = "use DIR as spool directory")
parser.add_option("-s", "--state-file", metavar = "FILE", help = "read published and excluded segments from FILE")
parser.add_option("-P", "--pid-file", metavar = "FILE", help = "use FILE as process lock file")
parser.add_option("-D", "--input-directory", metavar = "DIR", help = "look for input files in DIR")
parser.add_option("-l", "--log-file", metavar = "FILE", help = "use FILE as log file")
parser.add_option("-L", "--log-level", metavar = "LEVEL", default = "INFO", help = "set logging level to LEVEL")
......@@ -105,20 +106,23 @@ if options.ping:
print msg
sys.exit(0)
if not options.spool_directory:
raise ValueError, "missing argument --spool-directory"
if not options.state_file:
raise ValueError, "missing argument --state-file"
if not options.pid_file:
raise ValueError, "missing argument --pid-file"
if not options.input_directory:
raise ValueError, "missing argument --input-directory"
if not options.log_file:
raise ValueError, "missing argument --log-file"
lock_file = os.path.join(options.spool_directory,'publish_dqxml.lock')
if os.access(lock_file,os.F_OK):
raise RuntimeError, "lock file exists: %s" % lock_file
else:
fp = open(lock_file, "w")
fp.write("lock")
fp.close()
# check for an existing lock file
if os.access(options.pid_file,os.F_OK):
raise RuntimeError, "lock file exists: %s" % options.pid_file
# create lock file
fp = open(options.pid_file, "w")
fp.write("lock")
fp.close()
try:
# setup the output file
......@@ -135,22 +139,25 @@ try:
logger.setLevel(eval("logging." + options.log_level))
logger.info("ligolw_publish_dqxml starting")
logger.debug("server = " + options.server)
logger.debug("spool directory = " + options.spool_directory)
logger.debug("pid file = " + options.pid_file)
logger.debug("input directory = " + options.input_directory)
logger.debug("log file = " + options.log_file)
logger.debug("log level = " + options.log_level)
# read in the published file and get the published segment list
state_file = os.path.join(options.spool_directory,"published.xml")
logger.debug("reading state from %s" % state_file)
indoc = ligolw_utils.load_url(state_file, gz = (state_file or "stdin").endswith(".gz"))
logger.debug("reading state from %s" % options.state_file)
indoc = ligolw_utils.load_url(options.state_file, gz = (options.state_file or "stdin").endswith(".gz"))
published_segments = segmentdb_utils.find_segments(indoc,"P1:PUBLISHED:0")
exclude_segments = segmentdb_utils.find_segments(indoc,"P1:EXCLUDE:0")
logger.debug("published segments = %s" % str(published_segments))
logger.debug("excluded segments = %s" % str(exclude_segments))
# FIXME this will break Sep 14 2011 01:46:24 UTC
all_time = segments.segmentlist([segments.segment(0,999999999)])
# make a list of the files that need to be inserted
pending_segments = all_time - published_segments
pending_segments = (all_time - published_segments) - exclude_segments
logger.info("pending segments = %s" % str(pending_segments))
pending_files = lal.Cache()
for s in pending_segments:
pending_files += lal.Cache.from_urls(segmentdb_utils.get_all_files_in_range(options.input_directory,s[0],s[1]),coltype=int).sieve(segment=s)
......@@ -179,12 +186,16 @@ try:
else:
published_segments |= segments.segmentlist([f.segment])
segment_def_id = segmentdb_utils.add_to_segment_definer(outdoc,proc_id,"P1","PUBLISHED",0)
segmentdb_utils.add_to_segment(outdoc,proc_id,segment_def_id,published_segments)
excl_def_id = segmentdb_utils.add_to_segment_definer(outdoc,proc_id,"P1","EXCLUDE",0)
pub_def_id = segmentdb_utils.add_to_segment_definer(outdoc,proc_id,"P1","PUBLISHED",0)
segmentdb_utils.add_to_segment(outdoc,proc_id,pub_def_id,exclude_segments)
segmentdb_utils.add_to_segment(outdoc,proc_id,excl_def_id,published_segments)
logger.debug("published segments = %s" % str(published_segments))
logger.debug("excluded segments = %s" % str(exclude_segments))
# write the new segment state file on top of the old one
logger.debug("writing state to %s" % state_file)
ligolw_utils.write_filename(outdoc, state_file)
logger.debug("writing state to %s" % options.state_file)
ligolw_utils.write_filename(outdoc, options.state_file)
except Exception, e:
try:
......@@ -192,9 +203,9 @@ except Exception, e:
except:
pass
print >>sys.stderr, "runtime error (%s)" % str(e)
os.unlink(lock_file)
os.unlink(options.pid_file)
sys.exit(1)
logger.info("exiting")
os.unlink(lock_file)
os.unlink(options.pid_file)
sys.exit(0)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment