Added make_triggers_sn.py and mdc_setup_sn.py. (condor_pp_sn.py is not yet complete)

git-svn-id: https://svn.ligo.caltech.edu/svn/bayeswave/branches/craig@157 c56465c9-8126-4a4f-9d7d-ac845eff4865

burstinj/s6/make_cache.py

@@ -9,22 +9,22 @@ from glob import glob
 # -- Cache file format
 # H H1_ER_C00_L1 1074903984 4 file://localhost/archive/frames/ER5/hoft/H1/H-H1_ER_C00_L1-10749/H-H1_ER_C00_L1-1074903984-4.gwf
 
-start = 961545615
-end = 971654415
+start = 870768590
+end = 871215434
 
 # ifo = 'GHLV'
 # frtype = 'BRST_S6'
 
 # basedir = '/archive/frames/S6/Burst-MDC/GHLV-BRST_S6'
-basedir = '/archive/frames/S6/Burst-MDC/GHLV-ELPTC_S6'
+basedir = '/home/stuver/SNMDCs/SN2007gr/GHLV-SN2007gr_piro5'
 
-sfour = 9615
-efour = 9716
+sfour = 8707
+efour = 8712
 
-outfile = open('elptc_s6.cache','w')
+outfile = open('sn2007piro5.cache','w')
 for four in range(sfour, efour+1):
     print "Starting directory " + str(four)
-    dirname = "{0}/GHLV-ELPTC_S6-{1}/".format(basedir, four)
+    dirname = "{0}/GHLV-SN2007gr_piro5-{1}/".format(basedir, four)
     filelist = glob(dirname + '*')
     
     for filename in filelist:

burstinj/s6/make_triggers.py

@@ -68,8 +68,8 @@ print "Found {0} loud WNB in good times".format(goodloudwnb.size)
 
         
 # -- Set how many triggers we want
-N = 100
-start = 21
+N = 20
+start = 0
 stop = start + N
 
 outfile = open('trigs_sg153.txt','w')

burstinj/s6/make_triggers_sn.py

+import numpy as np
+import sys
+import os
+import re
+
+# -- Helper function for removing triggers outside a trigger list
+def find_in_segs(triglist, segs):
+    indx = np.zeros(len(triglist), dtype=bool)
+    for n in range(len(triglist)):
+        gps = triglist[n]
+        indx[n] = np.any( np.logical_and( gps>(segs['start']+8), gps<(segs['stop']-4)) )
+    return indx
+
+def check_scale(scale):
+    if scale not in possibleScales:
+        print
+        print '**********************'
+        print 'Scale factor not found'
+        print '**********************'
+        print 'Cannot find scale = {0}'.format(scale)
+        print
+        print 'No trigger file created for this scale'
+        scale = 0
+    return scale
+
+# ********************** Directory containing detected supernova triggers *****************************
+
+superFile = '/home/craig.cahillane/bayeswave/branches/craig/supernova/SN2007gr_H1H2L1V1_piro5_rho=3.5/'
+
+# *****************************************************************************************************
+
+# *************** Get info from the Log ****************
+namestring = """
+ GravEn_SimID     SimHrss      SimEgwR2    GravEn_Ampl  Internal_x   Internal_phi   External_x   External_phi  External_psi  FrameGPS     EarthCtrGPS     SimName               SimHpHp      SimHcHc      SimHpHc    GEO      GEOctrGPS       GEOfPlus       GEOfCross       GEOtErr       GEOcErr H1       H1ctrGPS        H1fPlus        H1fCross        H1tErr        H1cErr H2       H2ctrGPS        H2fPlus        H2fCross        H2tErr        H2cErr L1       L1ctrGPS        L1fPlus        L1fCross        L1tErr        L1cErr V1       V1ctrGPS        V1fPlus        V1fCross        V1tErr        V1cErr
+"""
+
+namelist = namestring.split()
+data = np.recfromtxt('BurstMDC-SN2007gr_piro5-Log.txt', names=namelist)
+
+waveform = data['SimName']
+hrss = data['SimHrss']
+phi = data['External_phi']
+x = data['External_x']
+theta = np.arccos(x)
+dec = np.pi/2 - theta
+gps = data['EarthCtrGPS']
+
+#snWaveNames = ['piroM5.0eta0.3fac0.2-plus','piroM5.0eta0.6fac0.2-plus','piroM10.0eta0.3fac0.2-plus','piroM10.0eta0.6fac0.2-plus']
+
+scaleFiles = os.listdir(superFile)
+possibleScales = []
+first_reg =  re.compile(r'scale_factor_\d*\.?\d*')
+second_reg = re.compile(r'\d+\.?\d*')
+for scaleFile in scaleFiles:
+    temp = re.findall(first_reg, scaleFile)
+    if temp:
+        temp2 = re.findall(second_reg,scaleFile)
+        if temp2: # If temp is not empty
+            possibleScales.append(float(temp2[0]))
+
+scale = 0
+
+if len(sys.argv) > 2:
+    print 'Too many arguments: using first arg as scale factor'
+    try:
+        scale = float(sys.argv[1])
+    except ValueError:
+        print 'First argument is not a number'
+        print 'Please try again'
+
+if len(sys.argv) == 2:
+    try:
+        scale = float(sys.argv[1])
+    except ValueError:
+        print 'Argument is not a number'
+        print 'Please try again'
+    
+scale = check_scale(scale)
+
+while scale == 0:
+    print 'Possible Scale Factors (Normalized to 10 kPc)'
+    for num in possibleScales:
+        print num
+    try:
+        scale = float(raw_input('What scale factor would you like for the triggers? '))
+        scale = check_scale(scale)
+    except ValueError:
+        print 'Input is not a number'
+
+scaleint = int(scale)
+if abs(scaleint - scale)< 0.0000001:  # check if scale is an integer
+    scale = scaleint
+scale_file = os.path.join(superFile,'scale_factor_{0}'.format(scale))
+titlelist = ['GoodData','Detected']
+data2 = np.recfromtxt(scale_file,names=titlelist)
+sn_detected = data2['Detected']
+print
+print 'Supernovas detected:', len(sn_detected)
+
+if(len(gps) != len(sn_detected)):
+    print 'ERROR: Number of gps times does not match number of injected supernovae'
+    sys.exit()
+
+totalSnWaveNames = []  # Stores the injected waveform names
+snWaveNames = []       # Stores the names of the waveforms found by the CoherentWaveBurst search at the choosen scale
+gps_detected = []      # Stores the GPS times of detected supernovae
+waveform_detected = [] # Stores the associated waveform model of the supernovae
+for i in range(len(sn_detected)):
+    if waveform[i] not in totalSnWaveNames:
+        totalSnWaveNames.append(waveform[i])
+    if sn_detected[i] == 1:
+        gps_detected.append(gps[i])
+        waveform_detected.append(waveform[i])
+        if waveform[i] not in snWaveNames:
+            snWaveNames.append(waveform[i])
+
+missingSnWaveNames = [] # Stores the waveforms not found at this scale factor
+print len(gps_detected), len(waveform_detected)
+if totalSnWaveNames != snWaveNames:
+    for wavename in totalSnWaveNames:
+        if wavename not in snWaveNames:
+            missingSnWaveNames.append(wavename)
+    print 'The search pipeline did not locate waveform bursts of the type', missingSnWaveNames
+
+# Set path to trigger files, using scale factors as directory names
+curDir = os.path.abspath('.')
+print 'Current Directory:', curDir
+snDir = os.path.join(curDir,'sn_trigs/{0}'.format(scale))
+if not os.path.exists(snDir):
+    os.makedirs(snDir)
+
+# Set number of triggers in each trigger file
+N = 20
+
+for wave in snWaveNames:
+    total_trigs = 0
+    outfile = open( os.path.join( snDir,'trigs_{0}.txt'.format(wave) ) ,'w')
+    for i in range(len(waveform_detected)):
+        if wave == waveform_detected[i]:
+            total_trigs += 1
+            outfile.write('{0}\n'.format(gps_detected[i]))
+            #outfile.write('{0} {1}\n'.format(gps_detected[i],waveform_detected[i]))
+        if total_trigs == N:
+            outfile.close()
+            break
+    outfile.close()

condor/s6/mdc_setup.py

 # -----------------------------------
+# Customized for my directory
+# August 13, 2014, Craig Cahillane
+# -----------------------------------
 # Example Condor setup script for BWB
 # 2014, Jonah Kanner
 # ----------------------------------
@@ -9,29 +12,33 @@ import subprocess
 # ------------------
 # -- Set paramters
 # ------------------
-topdir = '/home/jkanner/baysewave/PEC/mdc_v2'
+topdir = '/home/craig.cahillane/bayeswave/PEC/sn2007'
 # topdir = '/home/jkanner/baysewave/test'
-mdcdir = '/home/jkanner/baysewave/svn/trunk/burstinj/s6'
-waveformList = ['sg153']
-scaleList = ['0.3']
-mdccache = '/home/jkanner/baysewave/svn/trunk/burstinj/s6/brst_s6.cache'
-mdclog = '/home/jkanner/baysewave/svn/trunk/burstinj/s6/BurstMDC-BRST_S6-Log.txt'
-bwb = '/home/jkanner/baysewave/svn/branches/condor_compile/src/BayesWaveBurst'
+mdcdir = '/home/craig.cahillane/bayeswave/branches/craig/burstinj/s6'
+waveformList = ['piro5']
+scaleList = ['0.075','0.15','0.3','1']
+#scaleList = Marak's Supernova scale factor file
+mdccache = '/home/craig.cahillane/bayeswave/branches/craig/burstinj/s6/sn2007piro5.cache'
+mdclog = '/home/craig.cahillane/bayeswave/branches/craig/burstinj/s6/BurstMDC-SN2007_piro5-Log.txt'
+bwb = '/home/craig.cahillane/bayeswave/branches/craig/src/BayesWaveBurst'
 ifoList = ['H1', 'L1']
-frtypeList = ['H1_LDAS_C02_L2', 'L1_LDAS_C02_L2']
+frtypeList = ['H1_RDS_C03_L2', 'L1_RDS_C03_L2']
+#frtypeList = ['H1_LDAS_C02_L2', 'L1_LDAS_C02_L2']
 
 bwbargsfmt = """--ifo H1 --H1-flow 40 --H1-channel H1:LDAS-STRAIN \
        --ifo L1 --L1-flow 40 --L1-channel L1:LDAS-STRAIN \
        --H1-cache {wavedir}/H1.cache \
        --L1-cache {wavedir}/L1.cache \
-       --trigtime {gps} --srate 1024 --seglen 4 \
+       --trigtime {gps} --srate 4096 --seglen 4 \ 
        --bayesLine --PSDstart {gps} --PSDlength 4 \
-       --Niter 2000000 --NCmin 15  \
+       --Niter 2000 --NCmin 25 --NCmax 25 \
        --bayesLine --gnuplot \
        --MDC-channel [H1:GW-H,L1:GW-H]   \
        --MDC-cache [{cachefile},{cachefile}] \
        --MDC-prefactor {scale} \
+       --noAdaptTemperature --tempSpacing 1.5 --clusterProposal \
 """
+#Niter = 2000000, srate = 1024
 
 # -- Alternate parameters
 # waveformList = ['sg153', 'lwnb', 'qwnb']

condor/s6/mdc_setup_sn.py

+# -----------------------------------
+# Customized for my directory
+# August 13, 2014, Craig Cahillane
+# -----------------------------------
+# Example Condor setup script for BWB
+# 2014, Jonah Kanner
+# ----------------------------------
+import os
+import numpy as np
+import subprocess
+import glob
+import sys
+
+# ------------------
+# -- Set paramters
+# ------------------
+topdir = '/home/craig.cahillane/bayeswave/PEC/sn2007'
+# topdir = '/home/jkanner/baysewave/test'
+mdcdir = '/home/craig.cahillane/bayeswave/branches/craig/burstinj/s6/sn_trigs'
+#waveformList = ['piroM5.0eta0.3fac0.2-plus','piroM5.0eta0.6fac0.2-plus','piroM10.0eta0.3fac0.2-plus','piroM10.0eta0.6fac0.2-plus']
+#scaleList = ['0.075','0.15','0.3','1']
+#scaleList = Marak's Supernova scale factor file
+mdccache = '/home/craig.cahillane/bayeswave/branches/craig/burstinj/s6/sn2007piro5.cache'
+mdclog = '/home/craig.cahillane/bayeswave/branches/craig/burstinj/s6/BurstMDC-SN2007_piro5-Log.txt'
+bwb = '/home/craig.cahillane/bayeswave/branches/craig/src/BayesWaveBurst'
+ifoList = ['H1', 'L1']
+frtypeList = ['H1_RDS_C03_L2', 'L1_RDS_C03_L2']
+#frtypeList = ['H1_LDAS_C02_L2', 'L1_LDAS_C02_L2']
+channelname = 'LSC-STRAIN'
+#channelname = 'LDAS-STRAIN'
+
+bwbargsfmt = """--ifo H1 --H1-flow 40 --H1-channel H1:{channel} \
+       --ifo L1 --L1-flow 40 --L1-channel L1:{channel} \
+       --H1-cache {wavedir}/H1.cache \
+       --L1-cache {wavedir}/L1.cache \
+       --trigtime {gps} --srate 4096 --seglen 4 \ 
+       --bayesLine --PSDstart {gps} --PSDlength 4 \
+       --Niter 2000 --NCmin 25 --NCmax 25 \
+       --bayesLine --gnuplot \
+       --MDC-channel [H1:GW-H,L1:GW-H]   \
+       --MDC-cache [{cachefile},{cachefile}] \
+       --MDC-prefactor {scale} \
+       --noAdaptTemperature --tempSpacing 1.5 --clusterProposal \
+"""
+#Numbers for sg153, lwnb:  Niter = 2000000, srate = 1024
+
+# -- Alternate parameters
+# waveformList = ['sg153', 'lwnb', 'qwnb']
+# mdccache = '/home/jkanner/baysewave/svn/trunk/burstinj/s6/brst_s6.cache'
+# scaleList = ['0.15', '0.3', '0.6', '1.2']
+# topdir = '/home/jkanner/baysewave/test'
+
+# ---------------------------------
+# -- Define templates for run files
+# ---------------------------------
+submit_str = """
+executable={bwb}
+universe=standard
+arguments={bwbargs}
+output={top}/condorOut.txt
+error={top}/condorError.txt
+log={top}/condorLog.txt
+notification=never
+should_transfer_files=YES
+when_to_transfer_output = ON_EXIT
+stream_error=True
+stream_output=True
+queue 1
+"""
+
+# --------------
+# Fill scaleList
+# --------------
+scaleList = os.listdir(mdcdir)
+
+# -_-_-_-_-_-_-_-_-
+# Fill waveformList
+# _-_-_-_-_-_-_-_-_
+waveformList={}
+for scale in scaleList:
+    tempdir = os.path.join(mdcdir,scale)
+    waveformList[scale] = os.listdir(tempdir)
+
+# -----------------
+# Begin Main Script
+# -----------------
+dagfile = open( os.path.join(topdir, 'submit_all.sh'), 'w')
+
+for scale in scaleList:
+    scaledir = os.path.join(topdir,scale)
+    if not os.path.exists(scaledir): 
+        os.makedirs(scaledir)
+    for waveform in waveformList[scale]:
+        trigfile = os.path.join(mdcdir,'{0}/{1}'.format(scale,waveform))
+        trigList = np.loadtxt(trigfile)
+        wavedir = os.path.join(scaledir,waveform)
+        if not os.path.exists(wavedir):
+            os.makedirs(wavedir)
+        # ------------------------------------------------
+        # Call LIGO Data find for whole waveform directory
+        # ------------------------------------------------
+        start = int(trigList.min()) - 50
+        end   = int(trigList.max()) + 50
+        for ifo, frtype in zip(ifoList,frtypeList):
+            cachefilefmt = os.path.join(wavedir, '{0}.cache')
+            ldfcmd = "ligo_data_find --observatory {o} --type {frtype} -s {start} -e {end} --lal-cache | grep file > {cachefile}".format(o=ifo[0], frtype=frtype, cachefile = cachefilefmt.format(ifo), start=start, end=end)
+            print "Calling LIGO data find ..."
+            print ldfcmd
+            subprocess.call(ldfcmd, shell=True)
+        
+        for trig in trigList:
+            trig = str(trig)
+            trigdir = os.path.join(wavedir, 'job_'+trig)
+            if not os.path.exists(trigdir): os.makedirs(trigdir)
+            
+            bwbargs = bwbargsfmt.format(channel=channelname, gps=trig, wavedir=wavedir, cachefile=mdccache, scale=scale) 
+            submitname = os.path.join(trigdir, 'submit.txt')
+            submitfile = open( submitname, 'w' )
+            submitfile.write(submit_str.format(top=trigdir, bwb=bwb, bwbargs=bwbargs))
+            submitfile.close()
+            
+            dagfile.write("cd {0}\n".format(trigdir))
+            dagfile.write("condor_submit submit.txt\n\n")
+
+dagfile.close()       
+        
+
+
+#for waveform in waveformList:
+#    trigfile = os.path.join(mdcdir, 'trigs_{0}.txt'.format(waveform))
+#    trigList = np.loadtxt(trigfile)
+#    wavedir = os.path.join(topdir, waveform)
+#    if not os.path.exists(wavedir): os.makedirs(wavedir)
+#
+#    # ------------------------------------------------
+#    # Call LIGO Data find for whole waveform directory
+#    # ------------------------------------------------
+#    start = int(trigList.min()) - 50
+#    end   = int(trigList.max()) + 50
+#    for ifo, frtype in zip(ifoList,frtypeList):
+#        cachefilefmt = os.path.join(wavedir, '{0}.cache')
+#        ldfcmd = "ligo_data_find --observatory {o} --type {frtype} -s {start} -e {end} --lal-cache | grep file > {cachefile}".format(o=ifo[0], frtype=frtype, cachefile = cachefilefmt.format(ifo), start=start, end=end)
+#        print "Calling LIGO data find ..."
+#        print ldfcmd
+#        subprocess.call(ldfcmd, shell=True)
+#
+#    for scale in scaleList:
+#        scaledir = os.path.join(wavedir, scale)
+#        if not os.path.exists(scaledir): os.makedirs(scaledir)
+#        
+#        for trig in trigList:
+#            trig = str(trig)
+#            trigdir = os.path.join(scaledir, 'job_'+trig)
+#            if not os.path.exists(trigdir): os.makedirs(trigdir)
+#            
+#            bwbargs = bwbargsfmt.format(gps=trig, wavedir=wavedir, cachefile=mdccache, scale=scale) 
+#            submitname = os.path.join(trigdir, 'submit.txt')
+#            submitfile = open( submitname, 'w' )
+#            submitfile.write(submit_str.format(top=trigdir, bwb=bwb, bwbargs=bwbargs))
+#            submitfile.close()
+#            
+#            dagfile.write("cd {0}\n".format(trigdir))
+#            dagfile.write("condor_submit submit.txt\n\n")
+#
+#dagfile.close()
+print "... I'm afraid I can't do that."
+print "... Just kidding, I'm done."

postprocess/plot_waveform.py

@@ -8,7 +8,6 @@ import argparse
 import os
 import pwd
 
-
 # -------------------------------------------------
 # Define helper function to extract median waveform
 # -------------------------------------------------

postprocess/s6/condor_pp.py

@@ -3,7 +3,7 @@ import glob
 import os
 
 # -- Parameters
-mdclog = '/home/jkanner/baysewave/svn/trunk/burstinj/s6/BurstMDC-BRST_S6-Log.txt'
+mdclog = '/home/craig.cahillane/bayeswave/branches/craig/burstinj/s6/BurstMDC-BRST_S6-Log.txt'
 
 runfile = open('run_pp.sh', 'w')
 dirlist = glob.glob('*/job*')
@@ -18,7 +18,7 @@ for directory in dirlist:
     doitfile.write("""#! /bin/sh
 cd {directory}
 source /home/jkanner/.bashrc.mine
-python /home/jkanner/baysewave/svn/trunk/postprocess/skymap/skyview.py --mdc {mdclog}
+python /home/craig.cahillane/bayeswave/branches/craig/postprocess/plot_waveform.py --mdc=True
 """.format(directory=directory, mdclog=mdclog))
     doitfile.close()