allow dynamic memory requests in DAG layer definitions

gstlal/python/dags/layers/__init__.py

@@ -51,6 +51,10 @@ class Layer:
 	transfer_files
 		Whether to leverage Condor file transfer for moving around
 		files. On by default.
+	dynamic_memory
+		Whether to dynamically increase memory request if jobs are
+		put on hold due to going over memory requested.
+		Off by default.
 	requirements
 		Additional key-value pairs in the submit description.
 	inputs
@@ -73,6 +77,7 @@ class Layer:
 	log_dir: Optional[str] = "logs"
 	retries: Optional[int] = 3
 	transfer_files: Optional[bool] = True
+	dynamic_memory: Optional[bool] = False
 	requirements: Optional[dict] = field(default_factory=dict)
 	inputs: Optional[dict] = field(default_factory=dict)
 	outputs: Optional[dict] = field(default_factory=dict)
@@ -91,6 +96,7 @@ class Layer:
 			"universe": self.universe,
 			"executable": dagutils.which(self.executable),
 			"arguments": self._arguments(),
+			"periodic_release": "(HoldReasonCode == 5)",
 			**self.requirements,
 		}
 
@@ -104,7 +110,6 @@ class Layer:
 				submit_options["should_transfer_files"] = "YES"
 				submit_options["when_to_transfer_output"] = "ON_SUCCESS"
 				submit_options["success_exit_code"] = 0
-				submit_options['periodic_release'] = '(HoldReasonCode == 5)'
 				submit_options["preserve_relative_paths"] = True
 			if inputs:
 				submit_options["transfer_input_files"] = inputs
@@ -119,6 +124,16 @@ class Layer:
 		# extra boilerplate submit opts
 		submit_options["notification"] = "never"
 
+		# set dynamic memory opts if requested
+		if self.dynamic_memory:
+			base_memory = submit_options["request_memory"]
+			submit_options["+MemoryUsage"] = f"( {base_memory} ) * 2 / 3"
+			submit_options["request_memory"] = "( MemoryUsage ) * 3 / 2"
+			submit_options["periodic_release"] = " && ".join([
+				submit_options["periodic_release"],
+				"((CurrentTime - EnteredCurrentStatus > 180) && (HoldReasonCode != 34))"
+			])
+
 		return {
 			"name": self.name,
 			"submit_description": htcondor.Submit(submit_options),