Fixed a few condor bugs.

BayesWaveUtils/bayeswave_pipe/bayeswave_pipe_utils.py

@@ -996,8 +996,8 @@ def condor_job_config(job_type, condor_job, config_parser):
     #
     if config_parser.getboolean('condor', 'osg-deploy'):
         # --- Force downstream jobs to run locally
-        if job_type in ['bayeswave_post', 'bayeswave_fpeak', 'megaplot.py',
-                        'megasky.py']:
+        if job_type in ['bayeswave_post', 'bayeswave_fpeak', 'bayeswave_clean_frame',
+                         'megaplot.py', 'megasky.py']:
             requires.append("(GLIDEIN_SITE=?=undefined)")
         else:
             try:
@@ -1605,12 +1605,14 @@ class bayeswave_clean_frameJob(pipeline.CondorDAGJob, pipeline.AnalysisJob):
         self.add_condor_cmd('initialdir', workdir)
         self.set_sub_file(os.path.join(workdir, 'bayeswave_clean_frame.sub'))
 
+
         self.set_stdout_file(os.path.join(workdir, 'logs',
-            'BayesWaveCleanFrame_$(macrooutputDir)-$(cluster)-$(process).out'))
+            'BayesWaveCleanFrame_$(macrooutdir)-$(macroifo)-$(cluster)-$(process).out'))
         self.set_stderr_file(os.path.join(workdir, 'logs',
-            'BayesWaveCleanFrame_$(macrooutputDir)-$(cluster)-$(process).err'))
+            'BayesWaveCleanFrame_$(macrooutdir)-$(macroifo)-$(cluster)-$(process).err'))
         self.set_log_file(os.path.join(workdir, 'logs',
-            'BayesWaveCleanFrame_$(macrooutputDir)-$(cluster)-$(process).log'))
+            'BayesWaveCleanFrame_$(macrooutdir)-$(macroifo)-$(cluster)-$(process).log'))
+
 
         if cp.has_option('condor','arch'):
             self.add_condor_cmd('+arch',cp.get('condor','arch'))
@@ -1633,13 +1635,13 @@ class bayeswave_clean_frameJob(pipeline.CondorDAGJob, pipeline.AnalysisJob):
         if cp.getboolean('condor', 'transfer-files'):
 
             # File transfers
-            transferstring='$(macrooutputDir),datafind'
+            transferstring='$(macrooutdir),datafind'
 
             if cp.getboolean('condor','copy-frames'):
                 transferstring += ',$(macroframes)'
 
             self.add_condor_cmd('transfer_input_files', transferstring)
-            self.add_condor_cmd('transfer_output_files', '$(macrooutputDir)')
+            self.add_condor_cmd('transfer_output_files', '$(macrooutdir)')
 
         clean_suffix = cp.get('bayeswave_clean_frame_options', 'clean-suffix')
         self.add_opt('clean-suffix', clean_suffix)

BayesWaveUtils/scripts/bayeswave_pipe

@@ -1186,9 +1186,6 @@ for t,trigger in enumerate(trigger_list.triggers):
         # Add options for bayeswave_clean_frame node
         #
         if opts.bayeswave_clean_frame:
-            clean_frame_outdir = os.path.join(outputDir, 'clean_frame')
-            if not os.path.exists(clean_frame_outdir): 
-                os.makedirs(clean_frame_outdir)
             bayeswave_clean_frame_nodes = []
             if cp.has_option('bayeswave_options', 'fullOnly'):
                 model_type = 'full'
@@ -1199,6 +1196,7 @@ for t,trigger in enumerate(trigger_list.triggers):
                 clean_frame_node =  pipe_utils.bayeswave_clean_frameNode(bayeswave_clean_frame_job)
                 clean_frame_node.set_trigtime(trigger.trigger_time)
                 clean_frame_node.set_ifo(ifo)
+                clean_frame_node.set_segment_start(segment_start)
                  
                 trigtime = int(trigger.trigger_time)
                 example_ts = fr.read_timeseries(cache_files[ifo], channel_names[ifo], trigtime-1,2.0 )
@@ -1216,9 +1214,9 @@ for t,trigger in enumerate(trigger_list.triggers):
                 clean_frame_node.set_frame_srate(clean_frame_srate)
                 clean_frame_node.set_seglen(trigger.seglen)            
             
-                glitch_param_file = os.path.join(outputDir, 'chains', '%s_params_%s.dat'%(model_type, ifo))
+                glitch_param_file = os.path.join(outputDir, 'chains', '%s_params_%s.dat.0'%(model_type, ifo))
                 clean_frame_node.set_glitch_param_file(glitch_param_file)
-                clean_frame_node.set_outdir(clean_frame_outdir)
+                clean_frame_node.set_outdir(outputDir)
 
                 clean_frame_node.set_channel_name(channel_names[ifo])
                 clean_frame_node.set_frame_type(frtype_list[ifo])

src/BayesWaveCleanFrame.c

 /*
- *  Copyright (C) 2018 Neil J. Cornish, Tyson B. Littenberg, James A. Clark, Jonah B. Kanner
+ *  Copyright (C) 2018 Neil J. Cornish, Tyson B. Littenberg, James A. Clark, Jonah B. Kanner,
+ *  Sudarshan Ghonge
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -25,6 +26,7 @@
 #include <math.h>
 #include <string.h>
 #include <fftw3.h>
+#include <sys/stat.h>
 #include <gsl/gsl_math.h>
 #include <gsl/gsl_rng.h>
 #include <gsl/gsl_randist.h>
@@ -91,7 +93,7 @@ struct CleanFrameData
 void print_usage();
 void parse_command_line_args(int argc, char **argv, struct CleanFrameData *data);
 
-static void output_frame(REAL8TimeSeries *timeData, REAL8TimeSeries *timeRes, REAL8TimeSeries *timeGlitch, CHAR *frameType, CHAR *ifo, CHAR *outdir);
+static void output_frame(REAL8TimeSeries *timeData, REAL8TimeSeries *timeRes, REAL8TimeSeries *timeGlitch, CHAR *frameType, CHAR *ifo, CHAR *cleandir);
 
 void printProgress (double percentage)
 {
@@ -181,6 +183,10 @@ int main(int argc, char *argv[])
   version = data->clean_suffix;
   outdir = data->outdir;
 
+  char cleandir[MAXSTRINGSIZE];
+  sprintf(cleandir,"%s/clean_frame", data->outdir);
+  mkdir(cleandir,S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+
   char outframeType[MAXSTRINGSIZE];
   char outframeChannel[MAXSTRINGSIZE];
   char outframeGlitchChannel[MAXSTRINGSIZE];
@@ -349,7 +355,7 @@ int main(int argc, char *argv[])
   
 
   // Output cleaned data!
-  output_frame(timeData, timeRes, timeGlitch, outframeType, data->ifo, outdir);
+  output_frame(timeData, timeRes, timeGlitch, outframeType, data->ifo, cleandir);
   XLALDestroyREAL8TimeSeries(timeData);
 
   fprintf(stdout,"\n");
@@ -368,7 +374,7 @@ static void output_frame(REAL8TimeSeries *timeData,
                          REAL8TimeSeries *timeGlitch,
                          CHAR *frameType,
                          CHAR *ifo,
-                         CHAR *outdir)
+                         CHAR *cleandir)
 {
   CHAR fname[2048];
   INT4 duration;
@@ -400,7 +406,7 @@ static void output_frame(REAL8TimeSeries *timeData,
 
   /* get frame filename */
   duration = gpsEnd - gpsStart;
-  snprintf( fname, FILENAME_MAX, "%s/%c-%s-%d-%d.gwf", outdir, ifo[0], frameType, gpsStart, duration );
+  snprintf( fname, FILENAME_MAX, "%s/%c-%s-%d-%d.gwf", cleandir, ifo[0], frameType, gpsStart, duration );
 
   /* define frame */
   frame = XLALFrameNew( &timeData->epoch, duration, "LIGO", 0, 1, detectorFlags );