Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • duncanmmacleod/dqsegdb-client
  • computing/dqsegdb/client
2 results
Show changes
Commits on Source (5)
Expand all Hide whitespace changes
Inline Side-by-side
bin/ligolw_segment_insert_dqsegdb deleted 100755 → 0
View file @ 2e34d3ac
#!/usr/bin/env python3
# Copyright (C) 2014-2020 Syracuse University, European Gravitational Observatory, and Christopher Newport University. Written by Ryan Fisher and Gary Hemming. See the NOTICE file distributed with this work for additional information regarding copyright ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
###################################################################
"""
This allows the user to publish known and active segments from segment files
into the data quality segments database (DQSegDB).
Example:
ligolw_segment_insert_dqsegdb -t https://segments-backup.ligo.org --insert -i G1 --name=TEST_FLAG_007 --version=1 -e 'test explanation' -c 'test comment' -S known.txt -G active.txt
"""
from __future__ import print_function
import socket
import os
import socket
import atexit
import pwd
import time
import sys
import logging
import logging.handlers
from optparse import OptionParser
from dqsegdb import apicalls
import json
from six.moves import StringIO
from six.moves.urllib.parse import urlparse
try:
import pyRXPU as pyRXP
except ImportError:
try:
import pyRXP
except ImportError as e:
print("""
Error: unable to import the pyRXP module.
You must have pyRXP installed and in your PYTHONPATH to run %s.
%s
""" %(sys.argv[0], e), file=sys.stderr)
from gpstime import gpsnow
try:
from glue import ldbd
from ligo import segments
from glue.ligolw import lsctables
from glue.ligolw import ligolw
from glue.ligolw.utils import process
from glue.ligolw import types as ligolwtypes
except ImportError as e:
print("""
Error: unable to import modules from glue.
Check that glue is correctly installed and in your PYTHONPATH.
%s
""" % e, file=sys.stderr)
sys.exit(1)
##################################################################
from dqsegdb._version import get_versions
__author__ = "Ryan Fisher <ryan.fisher@ligo.org>"
__version__ = get_versions()['version']
__id__ = get_versions()['full-revisionid']
__date__ = get_versions()['date']
__verbose_msg__ = "version: " + get_versions()['version'] + "\nfull-revisionid: " + get_versions()['full-revisionid'] + "\ndirty: " + str(get_versions()['dirty']) \
+ "\nerror: " + str(get_versions()['error']) + "\ndate: " + get_versions()['date']
__formatted_date__ = __date__[0:10] + " " + __date__[11:19] + " +0000"
#__formatted_date__ = __date__[0:10] + " " + __date__[11:19] + " " + __date__[19:25] # it would be nice to fix this so that the time isn't off by a few hours
#__src__ = git_version.status
__src__ = "Status: dirty = " + str(get_versions()['dirty'])
PROGRAM_NAME = sys.argv[0].replace('./','')
PROGRAM_PID = os.getpid()
try:
USER_NAME = os.getlogin()
except:
USER_NAME = pwd.getpwuid(os.getuid())[0]
##################################################################
# Command Line
##################################################################
def parse_command_line():
parser = OptionParser()
parser.add_option("-p", "--ping", action = "store_true", default = None, help = "Ping the target server")
parser.add_option("-t", "--segment-url", action = "store", metavar = "URL", default = None, help = "Users have to specify protocol 'https://' for a secure connection in the segment database url. For example, '--segment-url=https://segments.ligo.org'. No need to specify port number.")
parser.add_option("-o", "--output", action = "store", metavar = "FILE", default = None,help = "Write segments to FILE rather than the segment database")
parser.add_option("-j", "--identity", action = "store", metavar = "IDENTITY", default = None,help = "Set the subject line of the server's service certificate to IDENTITY")
parser.add_option("-I", "--insert", action = "store_true", default = 0, help = "Insert segments to the segment database for a new flag or a new version of an existing flag; note: there is a bug that currently limits segment files to ~24k lines; see https://github.com/ligovirgo/dqsegdb/issues/86")
parser.add_option("-A", "--append", action = "store_true", default = 0, help = "Append segments to an existing segment type (existing flag AND version); note: there is a bug that currently limits segment files to ~24k lines; see https://github.com/ligovirgo/dqsegdb/issues/86")
parser.add_option("-H", "--ignore-future", action = "store_true", default = 0, help = "Allow insert or append to add segments to future times (i.e., times occurring after the time of segment insertion).")
parser.add_option("-F", "--ignore-append-check", action = "store_true", default = 0, help = "Allow --append to insert segments prior to the latest existing segment.")
parser.add_option("-D", "--debug", action = "store_true", default = 0, help = "Set debug flag True")
parser.add_option("-i", "--ifos", action = "store", metavar = "IFOS", default = None, help = "Set the segment interferometer to IFOS (e.g., H1)")
parser.add_option("-n", "--name", action = "store", metavar = "NAME", default = None, help = "Set the name of the segment (flag) to NAME (e.g., DMT-BADMONTH)")
parser.add_option("-v", "--version", action = "store", metavar = "VERSION", default = None, help = "Set the numeric version of the segment (flag) to VERSION (e.g., 1)")
parser.add_option("-e", "--explain", action = "store", metavar = "EXPLAIN", default = None, help = "Set the segment_definer:comment to COMMENT. This should explaining WHAT this flag mean (e.g., \"Light dip 10%\"). Required when --insert/-I is specified.")
parser.add_option("-c", "--comment", action = "store", metavar = "COMMENT", default = None, help = "Set the segment_summary:comment to COMMENT. This should explaining WHY these segments were inserted (e.g., \"Created from hveto results\")")
parser.add_option("-S", "--summary-file", action = "store", metavar = "FILE", default = None, help = "Read the segment_summary rows from FILE. This should be a file containing the GPS start and end times over which the flag was defined (known) (i.e., the union of on and off)")
parser.add_option("-G", "--segment-file", action = "store", metavar = "FILE", default = None, help = "Read the segment rows from FILE. This should containin the GPS start and end times over which the flag was active")
parser.add_option("-B", "--active-indicates-ifo-goodness", action = "store_true", default = False, help = "Sets the variable in the database to indicate that the flag is active when something is good in the ifo state. Default is False, so active flags normally indicate something is wrong.")
return parser.parse_args()
(options, args) = parse_command_line()
# Make sure all necessary command line arguments are properly given
errmsg = ""
if not options.segment_url:
errmsg += "Error: --segment-url must be specified\n"
if not options.ifos:
errmsg += "Error: --ifos must be specified\n"
if not options.name:
errmsg += "Error: --name must be specified\n"
if not options.version:
errmsg += "Error: --version must be specified\n"
if (not options.explain) and (not options.append):
errmsg += "Error: --explain must be specified\n"
if not options.comment:
errmsg += "Error: --comment must be specified\n"
if not options.summary_file:
errmsg += "Error: --summary-file must be specified\n"
if not options.segment_file:
errmsg += "Error: --segment-file must be specified\n"
if options.insert and options.append:
errmsg += "Error: Exactly one of [--insert | --append] can be specified\n"
if (not options.insert) and (not options.append):
errmsg += "One of --help, --insert, or --append must be specified\n"
#o if (not options.ping) and (not options.insert) and (not options.append):
#o errmsg += "One of --ping, --help, --insert, --append must be specified\n"
# --ignore-append-check is too dangerous to allow for direct DB access;
# user must dump to XML, check it, and be authorized to insert via ldbdc
if (not options.output and options.ignore_append_check):
errmsg += "--ignore-append-check may only be used with --output-file\n"
if options.debug:
debug=True
else:
debug=False
if options.active_indicates_ifo_goodness:
use_xml=False
active_indicates_ifo_goodness=True
else:
use_xml=True
active_indicates_ifo_goodness=False # default
if len(errmsg) and not options.ping:
print(errmsg, file=sys.stderr)
print("Run\n %s --help\nfor more information." % sys.argv[0], file=sys.stderr)
sys.exit(1)
def publishAndExit(filepath,debug=False):
result=callInsertMultipleDQXMLThreaded(filepath,debug)
if not result:
raise Exception("Call to InsertMultipleDQXMLFileThreaded returned failure status: Publishing unsuccessful!")
else:
# We're done here
print("Insert worked!")
sys.exit()
def del_file(name):
os.remove(name)
def callInsertMultipleDQXMLThreaded(filepath,debug):
logger = logging.getLogger('ligolw_publish_dqxml_dqsegdb')
log_file=filepath.split('.xml')[0]+'.log'
handler = logging.handlers.RotatingFileHandler(log_file, 'a', 1024**3, 3)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(eval("logging." + "DEBUG"))
infiles=[filepath]
result=apicalls.InsertMultipleDQXMLFileThreaded(infiles,logger,options.segment_url,hackDec11=False,debug=debug,threads=1)
return result
#Create an xml parser, a ligo_lw document parser, and the document
xmlparser = pyRXP.Parser()
lwtparser = ldbd.LIGOLwParser()
segment_md = ldbd.LIGOMetadata(xmlparser,lwtparser)
########################################################################
# Construct local table process, process_params and segment_definer
########################################################################
# Table process and process_params will be populated by calling the
# process/process_params utility
doc = ligolw.Document()
doc.appendChild(ligolw.LIGO_LW())
ligolwtypes.FromPyType[type(True)] = ligolwtypes.FromPyType[type(0)]
process_id = process.register_to_xmldoc(doc, PROGRAM_NAME, options.__dict__,
version = __id__,
cvs_entry_time = __formatted_date__, comment = 'ligolw_segment_insert_dqsegdb client insert or append').process_id
##########################################################################
# Process segment-file and summary-file
##########################################################################
# create the total time interval and storage for the active segments
intervals = segments.segmentlist()
active_segments = segments.segmentlist()
#-----------------Check version and type existence-------------------------
# Query server for current highest version and flag name
o=urlparse(options.segment_url)
protocol=o.scheme
server=o.netloc
version=int(options.version)
if debug:
print("Determining max version")
maxVersion=apicalls.dqsegdbMaxVersion(protocol,server,options.ifos,options.name)
if debug:
print("maxVersion=%d" % maxVersion)
print("Script call provided version=%d" % version)
if version > maxVersion+1:
# Should never insert or append to a version that is 2 steps up from current version in DB (including current version in DB = 0 case that is returned when flag does not yet exist in the database)
raise ValueError("Version in DB is %d. Version supplied, %d, is too high, and should only be equal to highest version in DB for appends or one greater for inserts." % (maxVersion,version))
if maxVersion == 0:
# Flag doesn't exist in DB
# If I use my publisher, this is fine, since it will take care of it for me
maxEndTime=0 ## Because this flag doesn't exist yet!
elif maxVersion > version:
# Fail!
print("Version in DB (%d) is greater than that specified (%d); exiting with error." % (maxVersion,version))
raise ValueError
elif maxVersion == version and options.insert:
# Fail!
print("Version in DB (%d) is equal to that specified (%d) for insertion; exiting with error." % (maxVersion,version))
raise ValueError
else:
# Proceed to grab maxEndTime from server
if debug:
print("Version approved by checks; determining max end time:")
if options.append:
data_dict,query_url1=apicalls.dqsegdbQueryTimeless(protocol,server,options.ifos,options.name,options.version,include_list_string='known,metadata')
maxEndTime=max([i[1] for i in data_dict['known']])
elif options.insert and version == maxVersion + 1:
maxEndTime=0 # The version is greater than the existing versions by one, so we're ok to go forward, regardless of what the summary segments are in the payload!
else:
print("Specified version is not next version available to insert into database; Existing max version is %d and provided version was %d." % (maxVersion,version))
raise ValueError
#-----------------Determine max_end_time of existing data in database-------------------------
#data_dict,query_url1=apicalls.dqsegdbQueryTimeless(protocol,server,options.ifos,options.name,options.version,include_list_string='known,metadata')
#maxEndTime=max([i[1] for i in data_dict['known']])
### Fix!!! (future upgrade): If I'm going to allow an xml file input, I'm going to have to do the following check on the xml file instead, implement that when I allow that.
#-----------------Check summary-file---------------------------------
# get and check the summary intervals to be inserted
fh = open(options.summary_file, 'r')
line_no = 1
for line in fh.readlines():
sum_line = line.strip().split(" ")
if len(sum_line) != 2:
print("\nERROR: invalid format in your --summary-file row number: %d " % line_no)
print("Invalid interval is %s" % str(sum_line))
sys.exit(1)
current_time = str(int(gpsnow()))
# make sure gps time is 9 digits
#if (len(sum_line[0])!=9 or len(sum_line[1])!=9):
# print("\nERROR: gps time must be 9 digits")
# print("Invalid interval in your --summary-file row number: %d " % line_no)
# print("Invalid interval is %s" % str(sum_line))
# sys.exit(1)
## make sure interval start time is less than and not equal to end time
if (int(sum_line[0]) >= int(sum_line[1])):
print("\nERROR: start_time MUST be less than the end_time")
print("Invalid interval in your --summary-file row number: %d " % line_no)
print("Invalid interval is %s" % str(sum_line))
sys.exit(1)
# make sure interval start time is greater than the max end time in the database
# unless --ignore-append-check is specified or we are inserting a new version
elif (int(sum_line[0]) < maxEndTime) and options.append and (not options.ignore_append_check):
print("\nERROR: summary start_time MUST be greater than the max summary end_time %d in the database" % maxEndTime)
print("Invalid interval in your --summary-file row number: %d" % line_no)
print("Invalid interval is %s" % str(sum_line))
sys.exit(1)
# make sure interval end time is less than current time
elif (int(sum_line[1]) > int(current_time)) and not options.ignore_future:
print("\nERROR: summary end_time cannot be greater than the current time")
print("Invalid interval in your --summary-file row number: %d " % line_no)
print("Invalid interval is %s" % str(sum_line))
sys.exit(1)
# if interval is valid, append it to intervals list
else:
this_sum = segments.segment(int(sum_line[0]),int(sum_line[1]))
intervals.append(this_sum)
line_no += 1
intervals.coalesce()
#-----------------Check segment-file---------------------------------
# get and check the active segments to be inserted
fh = open(options.segment_file, 'r')
line_no = 1
for line in fh.readlines():
seg_line = line.strip().split(" ")
if len(seg_line) != 2:
print("\nERROR: invalid format in your --segment-file row number: %d " % line_no)
print("Invalid segment is %s" % str(seg_line))
sys.exit(1)
# make sure gps time is 9 digits
#if (len(seg_line[0]) != 9 or len(seg_line[1])!=9):
# print("\nERROR: gps time must be 9 digits")
# print("Invalid segment in your --segment-file row number: %d" % line_no)
# print("Invalid segment is %s" % str(seg_line))
# sys.exit(1)
# make sure segment start_time is less than end_time:
if int(seg_line[0])>=int(seg_line[1]):
print("\nERROR: segment start_time MUST be less than the end_time")
print("Invalid segment in your --segment-file row number: %d" % line_no)
print("Invalid segment is %s" % str(seg_line))
sys.exit(1)
# make sure segment falls in the summary intervals specified in the --summary-file
this_seg = segments.segment(int(seg_line[0]),int(seg_line[1]))
if this_seg not in intervals:
print("\nERROR: segment cannot fall outside of the summary intervals specified in your --summary-file")
print("Invalid segment in your --segment-file row number: %d" % line_no)
print("Invalid segment is %s" % str(seg_line))
sys.exit(1)
# Otherwise, append this segment to the list of active segments
active_segments.append(this_seg)
line_no += 1
active_segments.coalesce()
if debug:
print("Beginning INSERT part of code")
#######################################################################################
# Process INSERT or APPEND
#######################################################################################
if options.insert or options.append:
if use_xml:
# For the new DQSEGDB, the creator_db is gone and the seg_def_id is determined server side from the ifo,name,version tuple. Thus, inserts and appends appear identical to the server, and only the checks done above need to differentiate them.
#--------------------------------------------------------------------------#
# create segment_definer table
seg_def_table = lsctables.New(lsctables.SegmentDefTable,columns = [
'segment_def_id', 'process_id','ifos','name', 'version','comment'])
doc.childNodes[0].appendChild(seg_def_table)
segment_definer = lsctables.SegmentDef()
seg_def_id = seg_def_table.get_next_id()
segment_definer.segment_def_id = seg_def_id
segment_definer.process_id = process_id
segment_definer.ifos = options.ifos
segment_definer.name = options.name
segment_definer.version = int(options.version)
segment_definer.comment = options.explain
seg_def_table.append(segment_definer)
#---------------- create local segment table ------------#
segment_table = lsctables.New(lsctables.SegmentTable, columns = [
'segment_id','process_id','segment_def_id','start_time','end_time'])
doc.childNodes[0].appendChild(segment_table)
for this_seg in active_segments:
segment = lsctables.Segment()
seg_id = segment_table.get_next_id()
segment.segment_id = seg_id
segment.process_id = process_id
segment.segment_def_id = seg_def_id
segment.start_time = int(this_seg[0])
segment.end_time = int(this_seg[1])
segment_table.append(segment)
#-------------- create local segment_summary table ------------#
seg_sum_table = lsctables.New(lsctables.SegmentSumTable, columns = [
'segment_sum_id', 'process_id', 'segment_def_id', 'start_time', 'end_time', 'comment'])
doc.childNodes[0].appendChild(seg_sum_table)
for this_sum in intervals:
segment_summary = lsctables.SegmentSum()
seg_sum_id = seg_sum_table.get_next_id()
segment_summary.segment_sum_id = seg_sum_id
segment_summary.process_id = process_id
segment_summary.segment_def_id = seg_def_id
segment_summary.start_time = this_sum[0]
segment_summary.end_time = this_sum[1]
segment_summary.comment = options.comment
seg_sum_table.append(segment_summary)
fake_file = StringIO()
doc.write(fake_file)
if options.output:
if debug:
print("Writing out to file and then exiting.")
fp = open(options.output,"w")
fp.write(fake_file.getvalue())
fp.close()
else:
if debug:
print("Writing to temporary file, then calling callInsertMultipleDQXMLThreaded")
filepath='/tmp/ligolw_segment_insert_'+str(time.time())+'.xml'
if not debug:
atexit.register(del_file,filepath) #
fp = open(filepath,'w')
fp.write(fake_file.getvalue())
fp.close()
publishAndExit(filepath,debug)
#result=callInsertMultipleDQXMLThreaded(filepath)
#if not result:
# raise Exception("Call to InsertMultipleDQXMLFileThreaded returned failure status: Publishing unsuccessful!")
#else:
# # We're done here
# if debug:
# print("Insert worked!")
# sys.exit()
else: # Direct to JSON
## Result JSON should look like this:
'''
{
"name":"DCH-RYAN_TEST_MAY192016",
"insert_history":[
{
"process_metadata":{
"process_start_timestamp":1147688345,
"uid":"rfisher",
"args":[
"--comment",
"active_means_ifo_badness_test",
"--insert",
"None",
"--ignore-append-check",
"0",
"--name",
"DCH-RYAN_TEST_MAY192016",
"--segment-file",
"test_segments.txt",
"--explain",
"active_means_ifo_badness_test",
"--summary-file",
"test_summaries.txt",
"--version",
"2",
"--append",
"0",
"--debug",
"None",
"--segment-url",
"https://segments-dev.ligo.org",
"--ignore-future",
"0",
"--ifos",
"L1"
],
"pid":1072128,
"fqdn":"ldas-pcdev1",
"name":"/home/rfisher/DQSEGDB_Mar12/dqsegdb/bin/ligolw_segment_insert_dqsegdb"
},
"insertion_metadata":{
"comment":"/dq/L1/DCH-RYAN_TEST_MAY192016/2",
"insert_data_stop":971610010,
"insert_data_start":971610000,
"auth_user":"rfisher",
"timestamp":1147688345
}
}
],
"active":[
[
971610000,
971610010
]
],
"version":2,
"known":[
[
971610000,
971610010
]
],
"ifo":"L1",
"metadata":{
"flag_description":"active_means_ifo_badness_test",
"further_info_url":"active_means_ifo_badness_test",
"deactivated":false,
"active_indicates_ifo_badness":null,
"flag_version_comment":"ligolw_segment_insert_dqsegdb client insert or append"
}
}
'''
# Ok, first need to build the dictionary object:
# flag_versions[(ifo,name,version)] = InsertFlagVersion(ifo,name,version)
# new_seg_summary = segments.segmentlist([segments.segment(start_time,end_time)])
# flag_versions[(ifo,name,version)].appendKnown(new_seg_summary)
# flag_versions[(ifo,name,version)].flag_description=str(flag_versions_numbered[i]['comment'])
# flag_versions[(ifo,name,version)].provenance_url=seg_sum_comment
# flag_versions[i].flag_version_comment=process_dict[pid]['process_comment']
# insert_history_dict = {}
# process_name=i['process_metadata']['name']
# process_pid=i['process_metadata']['pid']
# process_uid=i['process_metadata']['uid']
# insert_history_dict['process_metadata'] = process_dict[pid]['process_metadata']
# insert_history_dict['insertion_metadata'] = {}
# insert_history_dict['insertion_metadata']['insert_data_stop'] = stop
# insert_history_dict['insertion_metadata']['insert_data_start'] = start
# insert_history_dict['insertion_metadata']['timestamp'] = _UTCToGPS(time.gmtime())
# insert_history_dict['insertion_metadata']['auth_user']=process.get_username()
# insert_history_dict['insertion_metadata']['comment'] = '/dq/'+'/'.join([str(ifo),str(name),str(version)])
# flag_versions[i].insert_history.append(insert_history_dict)
# new_seg = segments.segmentlist([segments.segment(start_time,end_time)])
# flag_versions[(ifo,name,version)].appendActive(new_seg)
# flag_versions[i].coalesceInsertHistory()
# for i in flag_versions.values():
# i.buildFlagDictFromInsertVersion()
# url=i.buildURL(server)
# print(json.dumps(i.flagDict))
# patchWithFailCases(i,url,debug,logger,testing_options)
from dqsegdb.jsonhelper import InsertFlagVersion
flag_version=InsertFlagVersion(options.ifos,options.name,int(options.version))
# args might be something like options.__dict__
for this_sum in intervals:
start_time=this_sum[0]
end_time=this_sum[1]
new_seg_summary = segments.segmentlist([segments.segment(start_time,end_time)])
flag_version.appendKnown(new_seg_summary)
for this_seg in active_segments:
### Fix!!! Make sure these work with ints and floats!
start_time=this_seg[0]
end_time=this_seg[1]
new_seg = segments.segmentlist([segments.segment(start_time,end_time)])
flag_version.appendActive(new_seg)
flag_version.flag_description=options.explain
flag_version.provenance_url=options.comment
flag_version.active_indicates_ifo_badness=(not active_indicates_ifo_goodness)
flag_version.flag_version_comment = 'ligolw_segment_insert_dqsegdb client insert or append'
insert_history_dict={}
process_dict={}
process_dict[PROGRAM_PID] = {}
process_dict[PROGRAM_PID]['process_metadata']={}
process_dict[PROGRAM_PID]['process_metadata']['name'] = PROGRAM_NAME
process_dict[PROGRAM_PID]['process_metadata']['pid'] = PROGRAM_PID
process_dict[PROGRAM_PID]['process_metadata']['uid'] = USER_NAME
arglist=[]
for i in process.process_params_from_dict(options.__dict__):
arglist.append(i[0])
if i[2]:
arglist.append(i[2])
else:
arglist.append("None")
process_dict[PROGRAM_PID]['process_metadata']['args'] = arglist #[i[0],i[2] for i in process.process_params_from_dict(options.__dict__)
process_dict[PROGRAM_PID]['process_metadata']['fqdn'] = socket.gethostname()
process_dict[PROGRAM_PID]['process_metadata']['process_start_timestamp'] = current_time
insert_history_dict['process_metadata'] = process_dict[PROGRAM_PID]['process_metadata']
insert_history_dict['insertion_metadata'] = {}
insert_history_dict['insertion_metadata']['insert_data_start'] = min(min(intervals))
insert_history_dict['insertion_metadata']['insert_data_stop'] = max(max(intervals))
insert_history_dict['insertion_metadata']['timestamp'] = current_time
insert_history_dict['insertion_metadata']['auth_user']= USER_NAME #process.get_username()
insert_history_dict['insertion_metadata']['comment'] = '/dq/'+'/'.join([str(options.ifos),str(options.name),str(options.version)])
flag_version.insert_history.append(insert_history_dict)
flag_version.coalesceInsertHistory()
flag_version.buildFlagDictFromInsertVersion()
url=flag_version.buildURL(options.segment_url)
logger = logging.getLogger('ligolw_publish_dqxml_dqsegdb')
filepath='/tmp/ligolw_segment_insert_'+str(time.time())+'.xml'
log_file=filepath.split('.xml')[0]+'.log'
handler = logging.handlers.RotatingFileHandler(log_file, 'a', 1024**3, 3)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(eval("logging." + "DEBUG"))
#manual_debug=True
manual_debug=False
if manual_debug:
import pdb
pdb.set_trace()
#print json.dumps(flag_version.flagDict)
apicalls.patchWithFailCases(flag_version,url,debug,logger)
dqsegdb/tools/__init__.py 0 → 100644
View file @ 0b5de874
# Copyright (C) 2023 Cardiff University
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Tools for DQSegDB.
"""
bin/ligolw_dq_query_dqsegdb → dqsegdb/tools/ligolw_dq_query.py 100755 → 100644
View file @ 0b5de874
#!/usr/bin/env python3
# Copyright (C) 2014-2020 Syracuse University, European Gravitational Observatory, and Christopher Newport University. Written by Ryan Fisher and Gary Hemming. See the NOTICE file distributed with this work for additional information regarding copyright ownership.
#
# This program is free software: you can redistribute it and/or modify
......@@ -77,7 +76,7 @@ __verbose_msg__ = "version: " + get_versions()['version'] + "\nfull-revisionid:
#
def parse_command_line():
def parse_command_line(args=None):
"""
Parse the command line, return an options object
"""
......@@ -106,7 +105,7 @@ def parse_command_line():
parser.add_option("-f", "--tom-formatting", action = "store_true", help = "Tom Dent requested formatting option: condensed output into single lines.")
parser.add_option("-b", "--seperate-good-from-bad", action = "store_true", help = "Seperates flags marked by the database as being active when the ifo is in a bad state from those that are active when the ifo is in a good state.")
options, time = parser.parse_args()
options, time = parser.parse_args(args=args)
# Make sure we have required arguments
if options.segment_url:
......@@ -291,8 +290,9 @@ def get_full_name(ifo, name, version):
# =============================================================================
#
if __name__ == '__main__':
options, database_location, time = parse_command_line()
def main(args=None):
options, database_location, time = parse_command_line(args=args)
o=urlparse(options.segment_url)
protocol=o.scheme
......@@ -474,4 +474,5 @@ if __name__ == '__main__':
fh.close()
if __name__ == "__main__":
sys.exit(main())
dqsegdb/tools/ligolw_segment_insert.py 0 → 100644
View file @ 0b5de874
This diff is collapsed.
bin/ligolw_segment_query_dqsegdb → dqsegdb/tools/ligolw_segment_query.py 100755 → 100644
View file @ 0b5de874
#!/usr/bin/env python3
#
# Copyright (C) 2014-2020 Syracuse University, European Gravitational Observatory, and Christopher Newport University. Written by Ryan Fisher and Gary Hemming. See the NOTICE file distributed with this work for additional information regarding copyright ownership.
#
# This program is free software: you can redistribute it and/or modify
......@@ -107,7 +105,7 @@ __formatted_date__ = __date__[0:10] + " " + __date__[11:19] + " +0000"
#
def parse_command_line():
def parse_command_line(args=None):
"""
Parse the command line, return an options object
"""
......@@ -148,7 +146,7 @@ def parse_command_line():
parser.add_option("-o", "--output-file", metavar = "output_file", help = "File to which output should be written. Defaults to stdout.")
#parser.add_option("-v", "--debug", metavar="debug", help= "Turn on debugging.")
options, others = parser.parse_args()
options, others = parser.parse_args(args=args)
# Make sure we have exactly one thing to do
count = 0
......@@ -691,7 +689,7 @@ def setup_files(dir_name, gps_start_time, gps_end_time):
# =============================================================================
#
if __name__ == '__main__':
def main(args=None):
options, database_location, file_location = parse_command_line()
# Ping the database and exit if requested
......@@ -762,3 +760,5 @@ if __name__ == '__main__':
os.remove(temp_db)
if __name__ == '__main__':
sys.exit(main())
bin/ligolw_segments_from_cats_dqsegdb → dqsegdb/tools/ligolw_segments_from_cats.py 100755 → 100644
View file @ 0b5de874
#!/usr/bin/env python3
# Copyright (C) 2009 Larne Pekowsky
# Copyright (C) 2014-2020 Syracuse University, European Gravitational Observatory, and Christopher Newport University. Written by Ryan Fisher and Gary Hemming. See the NOTICE file distributed with this work for additional information regarding copyright ownership.
#
......@@ -93,7 +92,7 @@ __verbose_msg__ = "version: " + get_versions()['version'] + "\nfull-revisionid:
#
def parse_command_line():
def parse_command_line(args=None):
"""
Parse the command line, return an options object
"""
......@@ -123,7 +122,7 @@ def parse_command_line():
parser.add_option("-e", "--gps-end-time", metavar = "gps_end_time", help = "End of GPS time range.")
parser.add_option("-g", "--debug", metavar = "debug", help = "Print debug information.")
options, others = parser.parse_args()
options, others = parser.parse_args(args=args)
if not options.veto_file:
raise ValueError("missing required argument --veto-file")
......@@ -259,12 +258,12 @@ class ContentHandler(ligolw.LIGOLWContentHandler):
# =============================================================================
#
if __name__ == '__main__':
def main(args=None):
# Settings
dbtables.get_column_info = kludge_get_column_info
options, db_location, file_location, loaded_file, debug = parse_command_line()
options, db_location, file_location, loaded_file, debug = parse_command_line(args=args)
if options.non_integer:
allow_non_int=True
......@@ -453,3 +452,7 @@ if __name__ == '__main__':
if debug:
print("If you see this message, your call to ligolw_segments_from_cats was successful.")
print("You may want to follow up on any warnings printed before this point; they will indicate any flags that didn't exist in the database but were in the veto definer file.")
if __name__ == "__main__":
sys.exit(main())
setup.py
View file @ 0b5de874
......@@ -36,6 +36,14 @@ setup(name=PACKAGENAME,
packages=packagenames,
ext_modules=[],
scripts=scripts,
entry_points={
"console_scripts": [
"ligolw_dq_query_dqsegdb = dqsegdb.tools.ligolw_dq_query:main",
"ligolw_segment_insert_dqsegdb = dqsegdb.tools.ligolw_segment_insert:main",
"ligolw_segment_query_dqsegdb = dqsegdb.tools.ligolw_segment_query:main",
"ligolw_segments_from_cats_dqsegdb = dqsegdb.tools.ligolw_segments_from_cats:main",
],
},
setup_requires=['setuptools'],
install_requires=[
'gpstime',
......