Commit 849e47c0 authored by Salvatore Vitale's avatar Salvatore Vitale Committed by Carl-Johan Haster

Fix lib gdb

parent 198023c6
......@@ -56,8 +56,13 @@ def cbcBayesGraceDBinfo(gid=None,samples=None,skymap=None,analysis='LALInference
g=ligo.gracedb.rest.GraceDb()
if samples is not None:
samples=os.path.realpath(samples)
peparser=bppu.PEOutputParser('common')
commonResultsObj=peparser.parse(open(samples,'r'))
if '.hdf' in samples or '.h5' in samples:
peparser = bppu.PEOutputParser('hdf5')
commonResultsObj=peparser.parse(samples)
else:
peparser=bppu.PEOutputParser('common')
commonResultsObj=peparser.parse(open(samples,'r'))
try:
pos = bppu.BurstPosterior(commonResultsObj)
pars=['frequency','quality','hrss']
......@@ -78,18 +83,29 @@ def cbcBayesGraceDBinfo(gid=None,samples=None,skymap=None,analysis='LALInference
outstr+='<tr><td align=left>%s %s</td>'%(i,units[i])
outstr+='<td align=left>%.3f &plusmn; %.3f</td></tr>'%(pos[i].samples[which][0],pos[i].stdev)
if bcifile is not None and os.path.isfile(bcifile):
bci=np.loadtxt(bcifile)
bci=np.loadtxt(bcifile)
else: bci=None
if bci is not None:
outstr+='<tr><td align=left>logBCI</td>'
outstr+='<td align=center>%.2f</td></tr>'%(bci)
bsn=None
if bsnfile is not None and os.path.isfile(bsnfile):
bsn=np.loadtxt(bsnfile)
else: bsn=None
bsn=bsn[0]
else:
try:
import h5py
with h5py.File(samples,'r') as h5grp:
tmp=h5grp['lalinference']['lalinference_nest'].attrs
bsn=tmp['log_bayes_factor']
except Exception,e:
print("Could not obtain BNS\n")
print(e)
if bsn is not None:
outstr+='<tr><td align=left>logBSN</td>'
outstr+='<td align=center>%.2f</td></tr>'%(bsn[0])
outstr+='<td align=center>%.2f</td></tr>'%(bsn)
outstr+='</table>'
if email is not None and bci is not None:
......@@ -132,8 +148,8 @@ def cbcBayesGraceDBinfo(gid=None,samples=None,skymap=None,analysis='LALInference
bsn=np.loadtxt(bsnfile)
else: bsn=None
tag=['sky_loc']
if bci is not None and bsn[0] is not None:
if bsn[0]>5. and bci>2.:
if bci is not None and bsn is not None:
if bsn>5. and bci>2.:
tag.append('lvem')
g.writeLog(gid,message,filename=skymap,tagname=tag)
......@@ -144,10 +160,10 @@ if __name__=='__main__':
from optparse import OptionParser
parser=OptionParser(USAGE)
parser.add_option("-g","--gid", dest="gid",help="GraceDB id", metavar="G123456",default=None)
parser.add_option("-s","--samples",dest="samples",help="posterior_samples.dat",default=None)
parser.add_option("-s","--samples",dest="samples",help="posterior_samples.hdf5/dat",default=None)
parser.add_option("--analysis",help="Prefix to use for the graceDB entries. Should be the name of the analysis (default LALInference)",default='LALInference')
parser.add_option("--bci",dest="bci",help="coherence test file: bci.dat",default=None)
parser.add_option("--bsn",dest="bsn",help="evidence file: bsn.dat",default=None)
parser.add_option("--bsn",dest="bsn",help="evidence file: bsn.dat [Deprecated, now is read from samples.hdf5]",default=None)
parser.add_option("--skymap",dest="skymap",help="FITS file skymap",default=None)
parser.add_option("--message",dest="message",type='str', help="Message to go with skymap uplaod",default=None)
parser.add_option('--email',dest='email',help="Will email when run is done.",default=None)
......
......@@ -762,6 +762,7 @@ class LALInferencePipelineDAG(pipeline.CondorDAG):
self.gracedbjob.set_grid_site('local')
self.mapjob = SkyMapJob(cp, os.path.join(self.basepath,'skymap.sub'), self.logpath)
self.plotmapjob = PlotSkyMapJob(cp, os.path.join(self.basepath,'plotskymap.sub'),self.logpath)
self.postruninfojob=PostRunInfoJob(self.config,os.path.join(self.basepath,'postrungdbinfo.sub'),self.logpath,dax=self.is_dax())
# Process the input to build list of analyses to do
self.events=self.setup_from_inputs()
......@@ -794,6 +795,9 @@ class LALInferencePipelineDAG(pipeline.CondorDAG):
if self.config.has_option('analysis','upload-to-gracedb'):
if self.config.getboolean('analysis','upload-to-gracedb'):
self.add_gracedb_FITSskymap_upload(self.events[0],engine=self.engine)
if self.config.has_option('condor','gdbinfo') and self.config.has_option('analysis','ugid') and self.config.getboolean('analysis','upload-to-gracedb'):
self.add_gracedb_info_node(None,event.GID,analysis='LIB',issky=True)
self.dagfilename="lalinference_%s-%s"%(self.config.get('input','gps-start-time'),self.config.get('input','gps-end-time'))
self.set_dag_file(os.path.join(self.basepath,self.dagfilename))
if self.is_dax():
......@@ -1246,8 +1250,14 @@ class LALInferencePipelineDAG(pipeline.CondorDAG):
# To avoid that, LIB will read the gracedDB id to upload info to as an ugid=ID option
# in the analysis section.
ugid=self.config.get('analysis','ugid')
event.GID=ugid
self.add_gracedb_start_node(ugid,'LIB',[sciseg.get_df_node() for sciseg in enginenodes[0].scisegs.values()])
self.add_gracedb_log_node(respagenode,ugid)
self.add_gracedb_log_node(respagenode,ugid,burst=True)
if self.config.has_option('resultspage','email'):
emailto=self.config.get('resultspage','email')
else:
emailto=None
temp_node=self.add_gracedb_info_node(respagenode,ugid,analysis='LIB',email=emailto)
if self.config.has_option('condor','ligo-skymap-plot') and self.config.has_option('condor','ligo-skymap-from-samples'):
if self.engine=='lalinferenceburst': prefix='LIB'
else: prefix='LALInference'
......@@ -1695,6 +1705,8 @@ class LALInferencePipelineDAG(pipeline.CondorDAG):
node.add_parent(parent)
infile=parent.get_pos_file()
node.add_file_arg(infile)
node.append_in_files(infile)
node.set_output_path(outdir)
if gzip_output is not None:
node.set_gzip_output(gzip_output)
......@@ -1710,14 +1722,14 @@ class LALInferencePipelineDAG(pipeline.CondorDAG):
def add_gracedb_start_node(self,gid,name='',parent=None):
node=GraceDBNode(self.gracedbjob,parent=parent,gid=gid,command='log',tag='pe')
node=GraceDBNode(self.gracedbjob,parent=parent,gid=gid,command='create log',tag='pe')
node.set_message(name+' online parameter estimation started.')
self.add_node(node)
return node
def add_gracedb_log_node(self,respagenode,gid):
def add_gracedb_log_node(self,respagenode,gid,burst=False):
nodes=[]
node=GraceDBNode(self.gracedbjob,parent=respagenode,gid=gid,command='log',tag='pe')
node=GraceDBNode(self.gracedbjob,parent=respagenode,gid=gid,command='create log',tag='pe')
resurl=respagenode.webpath.replace(self.gracedbjob.basepath,self.gracedbjob.baseurl)
#node.set_message('online parameter estimation results: '+resurl+'/posplots.html')
node.set_message("LALInference online parameter estimation finished. <a href="+resurl+"/posplots.html>results</a>")
......@@ -1728,21 +1740,21 @@ class LALInferencePipelineDAG(pipeline.CondorDAG):
if self.config.has_option('analysis','add-lvem-tag'):
if self.config.getboolean('analysis','add-lvem-tag'):
tag='pe,lvem'
node=GraceDBNode(self.gracedbjob,parent=respagenode,gid=gid,command='upload',tag=tag)
node.set_filename(respagenode.webpath+'/corner/extrinsic.png')
self.add_node(node)
nodes.append(node)
node=GraceDBNode(self.gracedbjob,parent=respagenode,gid=gid,command='upload',tag='pe')
node.set_filename(respagenode.webpath+'/corner/intrinsic.png')
self.add_node(node)
nodes.append(node)
node=GraceDBNode(self.gracedbjob,parent=respagenode,gid=gid,command='upload',tag='pe')
node.set_filename(respagenode.webpath+'/corner/sourceFrame.png')
self.add_node(node)
nodes.append(node)
if burst is False:
node=GraceDBNode(self.gracedbjob,parent=respagenode,gid=gid,command='upload',tag=tag)
node.set_filename(respagenode.webpath+'/corner/extrinsic.png')
self.add_node(node)
nodes.append(node)
node=GraceDBNode(self.gracedbjob,parent=respagenode,gid=gid,command='upload',tag='pe')
node.set_filename(respagenode.webpath+'/corner/intrinsic.png')
self.add_node(node)
nodes.append(node)
node=GraceDBNode(self.gracedbjob,parent=respagenode,gid=gid,command='upload',tag='pe')
node.set_filename(respagenode.webpath+'/corner/sourceFrame.png')
self.add_node(node)
nodes.append(node)
return nodes
......@@ -1778,6 +1790,49 @@ class LALInferencePipelineDAG(pipeline.CondorDAG):
nodes.append(node)
return nodes
def add_gracedb_info_node(self,respagenode,gid,analysis='LALInference',issky=False,prefix="LIB",email=None):
# if issky=True, this node will upload the FITS file into GDB. BCI and BSN will be used to decide which tags to use
# Otherwise, this node will upload information about parameters and bayes factors
if respagenode is not None:
samples=respagenode.posfile
hdf5samples=respagenode.get_in_files()[0] # This should only be called by LIB, which only uses one file
else:
# Try to find it
resnodes=filter(lambda x: isinstance(x,ResultsPageNode) ,self.get_nodes())
for rs in resnodes:
if len(rs.ifos)>1:
respagenode=rs
samples=respagenode.posfile
hdf5samples=respagenode.get_in_files()[0] # This should only be called by LIB, which only uses one file
if self.postruninfojob.isdefined is False:
return None
if issky is False:
node=PostRunInfoNode(self.postruninfojob,parent=respagenode,gid=gid,samples=hdf5samples)
if email is not None:
node.set_email(email)
else:
skynodes=filter(lambda x: isinstance(x,SkyMapNode) ,self.get_nodes())
for sk in skynodes:
skymap=sk.outdir+'/%s.fits'%prefix
message=' %s FITS sky map'%prefix
node=PostRunInfoNode(self.postruninfojob,parent=sk,gid=gid,samples=None)
node.set_skymap(skymap)
node.set_message(message)
bci=respagenode.get_bcifile()
if bci is not None:
node.set_bci(bci)
bsn=respagenode.get_bsnfile()
if bsn is not None:
node.set_bsn(bsn)
node.set_analysis(analysis)
node.finalize()
self.add_node(node)
return node
def add_rom_weights_node(self,ifo,parent=None):
#try:
#node=self.computeroqweightsnodes[ifo]
......@@ -2402,6 +2457,9 @@ class ResultsPageNode(pipeline.CondorDAGNode):
self.__event=0
self.ifos=None
self.injfile=None
self.bcifile=None
self.bsnfile=None
self.infiles=[]
def set_gzip_output(self,path):
self.add_file_opt('archive',path,file_is_output_file=True)
......@@ -2456,6 +2514,10 @@ class ResultsPageNode(pipeline.CondorDAGNode):
return
self.add_var_arg('--trig '+coinc)
def append_in_files(self,this_file):
self.infiles.append(this_file)
def add_engine_parent(self,node):
"""
Add a parent node which is one of the engine nodes
......@@ -2463,15 +2525,24 @@ class ResultsPageNode(pipeline.CondorDAGNode):
"""
self.add_parent(node)
self.add_file_arg(node.get_pos_file())
self.infiles.append(node.get_pos_file())
self.append_in_files(node.get_pos_file())
def get_pos_file(self): return self.posfile
def get_in_files(self): return self.infiles
def set_bayes_coherent_incoherent(self,bcifile):
self.add_file_opt('bci',bcifile)
self.bcifile=bcifile
def get_bcifile(self):
return self.bcifile
def set_bayes_coherent_noise(self,bsnfile):
self.add_file_opt('bsn',bsnfile)
self.bsnfile=bsnfile
def get_bsnfile(self):
return self.bsnfile
def set_header_file(self,headerfile):
self.add_file_opt('header',headerfile)
......@@ -2720,7 +2791,7 @@ class GraceDBNode(pipeline.CondorDAGNode):
if self.filename:
self.add_var_arg(self.filename+' ')
if self.message:
self.add_var_arg(self.message)
self.add_var_arg("'{}'".format(self.message))
self.__finalized=True
class ROMJob(pipeline.CondorDAGJob,pipeline.AnalysisJob):
......@@ -2960,3 +3031,57 @@ class PlotSkyMapNode(pipeline.CondorDAGNode):
self.add_file_arg(self.fitsfile)
self.add_file_opt('output',self.outfile, file_is_output_file=True)
super(PlotSkyMapNode,self).finalize()
class PostRunInfoJob(pipeline.CondorDAGJob,pipeline.AnalysisJob):
def __init__(self,cp,submitFile,logdir,dax=False):
self.isdefined=True
if not cp.has_option('condor','gdbinfo'):
self.isdefined=False
return
exe=cp.get('condor','gdbinfo')
pipeline.CondorDAGJob.__init__(self,"vanilla",exe)
pipeline.AnalysisJob.__init__(self,cp,dax=dax) # Job always runs locally
if cp.has_option('analysis','accounting_group'):
self.add_condor_cmd('accounting_group',cp.get('analysis','accounting_group'))
self.set_sub_file(os.path.abspath(submitFile))
self.set_stdout_file(os.path.join(logdir,'gdbinfo-$(cluster)-$(process).out'))
self.set_stderr_file(os.path.join(logdir,'gdbinfo-$(cluster)-$(process).err'))
self.add_condor_cmd('getenv','True')
self.add_condor_cmd('RequestMemory','1000')
class PostRunInfoNode(pipeline.CondorDAGNode):
def __init__(self,post_run_info_job,gid=None,parent=None,samples=None):
pipeline.CondorDAGNode.__init__(self,post_run_info_job)
self.bci=None
self.bsn=None
self.analysis='LALInference'
self.__finalized=False
self.set_samples(samples)
self.set_parent(parent)
self.set_gid(gid)
def finalize(self):
self.add_var_opt('analysis',self.analysis)
self.__finalized=True
def set_parent(self,parentnode):
self.add_parent(parentnode)
def set_samples(self,samples):
if samples is not None:
self.add_var_arg('--samples %s'%samples)
def set_skymap(self,skymap):
self.add_var_arg('--skymap %s'%skymap)
def set_message(self,message):
self.add_var_arg("--message '%s'"%message)
def set_email(self,email):
self.add_var_arg('--email %s'%email)
def set_gid(self,gid):
self.add_var_opt('gid',gid)
def set_bci(self,bci):
self.add_file_opt('bci',bci)
def set_bsn(self,bsn):
self.add_file_opt('bsn',bsn)
def set_analysis(self,analysis):
self.analysis=analysis
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment