Added support for submitting jobs to the Open Science Grid

This PR updates bilby_pipe to support running on the Open Science Grid. The highlights are

• the bilby_pipe command-line script now supports the --osg option
• updated options for vanilla-universe checkpointing (cc @james-clark)

To support the above, I chose to do a bit of restructuring in the relevant methods to clean up how the submit files are generated, including

• moved out/err/log file path options to a method
• moved checkpointing options to a method
• moved file transfer options to a method and fixed directory names for transfer_output_files
• track extra_lines as a list, not a monolithic str (this is natively supported by pycondor anyway)
• support appending requirements to self.requirements (required for OSG options)

Fixes #103 (closed).

bilby_pipe/main.py

@@ -629,7 +629,8 @@ class Dag(object):
                 input_files_to_transfer.append(distance_marg_cache_file)
             extra_lines.extend(self._condor_file_transfer_lines(
                 input_files_to_transfer,
-                [self.inputs.result_directory],
+                [self._relative_topdir(self.inputs.outdir,
+                                       self.initialdir)],
             ))
 
         if self.inputs.osg:
@@ -878,7 +879,7 @@ class Dag(object):
         if self.inputs.transfer_files or self.inputs.osg:
             extra_lines.extend(self._condor_file_transfer_lines(
                 [str(self.inputs.ini)] + files,
-                [webdir],
+                [self._relative_topdir(self.inputs.outdir, self.initialdir)],
             ))
 
         if self.inputs.osg:
@@ -988,6 +989,9 @@ class Dag(object):
             # htcondor >= 8.9.x (probably)
             "+SuccessCheckpointExitBySignal = False",
             "+SuccessCheckpointExitCode = {}".format(code),
+            # ask condor to provide the checkpoint signals
+            "+WantCheckpointSignal = True",
+            "+CheckpointSig = \"SIGTERM\"",
         ]
 
     @staticmethod
@@ -1001,6 +1005,19 @@ class Dag(object):
             "stream_output = True",
         ]
 
+    @staticmethod
+    def _relative_topdir(path, reference):
+        """Returns the top-level directory name of a path relative
+        to a reference
+        """
+        try:
+            return str(Path(path).resolve().relative_to(reference))
+        except ValueError as exc:
+            exc.args = (
+                "cannot format {} relative to {}".format(path, reference),
+            )
+            raise
+
     def _osg_submit_options(self, executable, has_ligo_frames=False):
         """Returns the extra submit lines and requirements to enable running
         a job on the Open Science Grid