Merge branch 'pipeline-enhancements' into 'master'

Pipeline enhancements

See merge request !106

BayesWaveUtils/bayeswave_pipe/bayeswave_pipe_utils.py

@@ -885,6 +885,19 @@ def condor_job_config(job_type, condor_job, config_parser):
 
     requires=[]
 
+    # If the process exits successfully after receiving this signal, intermediate file transfer will occur
+    # "success" occurs when the process exits on signal CheckpointExitSignal
+    # Otherwise, "success" occurs when the process exits with code CheckpointExitCode
+    condor_job.add_condor_cmd('+CheckpointExitBySignal', False) #  
+    condor_job.add_condor_cmd('+CheckpointExitSignal', '"SIGTERM"')
+    condor_job.add_condor_cmd('+CheckpointExitCode', 130)
+    condor_job.add_condor_cmd('+SuccessCheckpointExitBySignal', False) #  
+    condor_job.add_condor_cmd('+SuccessCheckpointExitSignal', '"SIGTERM"')
+    condor_job.add_condor_cmd('+SuccessCheckpointExitCode', 130)
+    condor_job.add_condor_cmd('+WantFTOnCheckpoint', True)
+    condor_job.add_condor_cmd('+CheckpointSig', 130)
+    #condor_job.add_condor_cmd('+SpoolOnEvict', False)
+
     #
     # Singularity configuration
     #
@@ -924,18 +937,6 @@ def condor_job_config(job_type, condor_job, config_parser):
         else:
             condor_job.add_condor_cmd('when_to_transfer_output', 'ON_EXIT')
 
-        # If the process exits successfully after receiving this signal, intermediate file transfer will occur
-        # "success" occurs when the process exits on signal CheckpointExitSignal
-        # Otherwise, "success" occurs when the process exits with code CheckpointExitCode
-        condor_job.add_condor_cmd('+CheckpointExitBySignal', False) #  
-        condor_job.add_condor_cmd('+CheckpointExitSignal', '"SIGTERM"')
-        condor_job.add_condor_cmd('+CheckpointExitCode', 130)
-        condor_job.add_condor_cmd('+SuccessCheckpointExitBySignal', False) #  
-        condor_job.add_condor_cmd('+SuccessCheckpointExitSignal', '"SIGTERM"')
-        condor_job.add_condor_cmd('+SuccessCheckpointExitCode', 130)
-        condor_job.add_condor_cmd('+WantFTOnCheckpoint', True)
-        condor_job.add_condor_cmd('+CheckpointSig', 130)
-        #condor_job.add_condor_cmd('+SpoolOnEvict', False)
 
  
         # Time limit before job self-evicts to preserve checkpointing

BayesWaveUtils/scripts/bayeswave_pipe

@@ -192,7 +192,8 @@ def parser():
     parser.add_option("--osg-deploy", default=False, action="store_true")
     #parser.add_option("--max-runtime", type=float, default=84600)
     #parser.add_option("--resume-time", type=float, default=300)
-    parser.add_option("--transfer-files", default=False, action="store_true")
+    parser.add_option("--transfer-files", default=True, action="store_true")
+    parser.add_option("--shared-filesystem", default=False, action="store_true")
     parser.add_option("--singularity", default=None)
 
 
@@ -366,11 +367,20 @@ else:
 
 
 # File transfers
-# Usually only need this for the OSG but it's useful to support them elsewhere
 try:
-    cp.getboolean('condor', 'transfer-files')
+    cp.getboolean('condor', 'shared-filesystem')
 except:
-    cp.set('condor', 'transfer-files', str(opts.transfer_files))
+    cp.set('condor', 'shared-filesystem', str(opts.shared_filesystem))
+
+if not cp.getboolean('condor', 'shared-filesystem'):
+    try:
+        cp.getboolean('condor', 'transfer-files')
+    except:
+        cp.set('condor', 'transfer-files', str(opts.transfer_files))
+else:
+        cp.set('condor', 'transfer-files', str(False))
+
+
 
 # CVMFS check:
 # if BayesWave executables or singularity image path contains