systemd_generator: generator boots all systems on large test stand.

support/systemd/generator/cdsrfm.py

@@ -3,10 +3,12 @@ from front_end import Process
 
 class CDSRFMProcesses(object):
     def __init__(self, target_dir):
-        self.target_dir=target_dir
+        self.target_dir = target_dir
 
-    def epics(self):
-        return Process(True, "rts-cdsrfm-epics", "rts-cdsrfm-epics")
+    @staticmethod
+    def epics():
+        return Process("rts-cdsrfm-epics.service", "rts-cdsrfm-epics.service")
 
-    def module(self):
-        return Process(True, "rts-cdsrfm-module", "rts-cdsrfm-module")
\ No newline at end of file
+    @staticmethod
+    def module():
+        return Process("rts-cdsrfm-module.service", "rts-cdsrfm-module.service")

support/systemd/generator/fe_generator

+#!/bin/bash
+
+source /etc/advligorts/env
+set -a
+source /etc/advligorts/systemd_env
+source /etc/advligorts/systemd_env_`hostname`
+set +a
+
+cd /usr/lib/fe_generator
+
+python3 main.py $@
\ No newline at end of file

support/systemd/generator/front_end.py

@@ -5,16 +5,20 @@ import os.path as path
 
 
 class Process(object):
-    def __init__(self, is_in_world, start, end):
-        self.is_in_world = is_in_world
+    def __init__(self, start, end, first_service=None):
         self.start = start
         self.end = end
+        if first_service is None:
+            self.first_service = start
+        else:
+            self.first_service = first_service
 
 
 class FrontEndProcesses(object):
     def __init__(self, target_dir):
         self.target_dir = target_dir
 
+    def models(self):
         target_name = path.join(self.target_dir, "rts-models.target")
         with open(target_name, "wt") as f:
             f.write("""[Unit]
@@ -23,52 +27,87 @@ Description=All models
 
     @staticmethod
     def dolphin_port():
-        return Process(False, 'rts-dolphin-port.service', 'rts-dolphin-port.service')
+        return Process('rts-dolphin-port.service', 'rts-dolphin-port.service')
 
     def dolphin_drivers(self):
         # link up from one to the next
+        targ_name = "rts-dolphin-driver.target"
+        targ_path = path.join(self.target_dir, targ_name)
         services = ['dis_kosif.service', 'dis_ix.service', 'dis_irm.service', 'dis_sisci.service',
                     'dis_nodemgr.service']
+        with open(targ_path, "wt") as f:
+            f.write(f"""[Unit]
+Description=Dolphin IX drivers
+Wants={" ".join(services)}""")
+
         self.serialize_units(services)
-        return Process(False, 'dis_kosif.service', 'dis_nodemgr.service')
+        return Process(targ_name, 'dis_nodemgr.service', 'dis_kosif.service',)
 
     def epics_only_models(self, models):
-        target_name = path.join(self.target_dir, "rts-epics-only-models.target")
-        with open(target_name, "wt") as f:
-            f.write("""[Unit]
-Description=All epics only models            
-""")
         services = [f"rts-epics@{model}.service" for model in models]
+
+        target_name = "rts-epics-only-models.target"
+        target_path = path.join(self.target_dir, target_name)
+        with open(target_path, "wt") as f:
+            f.write(f"""[Unit]
+Description=All epics only models
+Wants={" ".join(services)}            
+""")
+
         self.serialize_units(services)
         for service in services:
-            self.link_to(service, target_name)
+            self.part_of(service, target_name)
 
-        self.link_to("rts-epics-only-models.target", "rts-models.target")
+        self.link_to(target_name, "rts-models.target")
+        self.part_of(target_name, "rts-models.target")
 
-        return Process(True, "rts-epics-only-models.target", services[-1])
+        return Process(target_name, services[-1], services[0])
 
     def iop_model(self, model):
-        link_path = path.join(self.target_dir, "rts-iop-model.target")
-        os.symlink(f"rts@{model}.target", link_path)
+        target_path = path.join(self.target_dir, "rts-iop-model.target")
+        with open(target_path, "wt") as f:
+            f.write(f"""[Unit]
+Description=The IPO model.    
+Wants=rts@{model}.target                    
+""")
+
+        self.part_of(f"rts@{model}.target", "rts-iop-model.target")
 
         self.link_to("rts-iop-model.target", "rts-models.target")
+        self.part_of("rts-iop-model.target", "rts-models.target")
 
-        return Process(True, f"rts-iop-model.target", f"rts-awgtpman@{model}.service")
+        return Process(f"rts-iop-model.target", f"rts-awgtpman@{model}.service",
+                       f"rts-epics@{model}.service")
 
-    def user_models(self, models):
-        target_name = path.join(self.target_dir, "rts-user-models.target")
-        with open(target_name, "wt") as f:
+    def user_models(self, models, iop_model):
+        target_name = "rts-user-models.target"
+        target_path = path.join(self.target_dir, target_name)
+        with open(target_path, "wt") as f:
             f.write("""[Unit]
 Description=All user models            
 """)
-        for i in range(1, len(models)):
-            self.after(f"rts@{models[i]}.target", f"rts-epics@{models[i-1]}.service")
+
+        services = []
+        for model in models:
+            services += [f"rts-epics@{model}.service", f"rts-module@{model}.service",
+                         f"rts-awgtpman@{model}.service"]
+
+        self.serialize_units(services)
+
         for model in models:
             self.link_to(f"rts@{model}.target", target_name)
+            self.part_of(f"rts@{model}.target", target_name)
+
+            #make sure iop is running before running any user models
+            self.requires(f"rts-epics@{model}.service", f"rts-module@{iop_model}.service")
+            self.after(f"rts-epics@{model}.service", f"rts-module@{iop_model}.service")
+
+        self.link_to(target_name, "rts-models.target")
+        self.part_of(target_name, "rts-models.target")
 
-        self.link_to("rts-user-models.target", "rts-models.target")
 
-        return Process(True, f"rts-user-models.target", f"rts-awgtpman@{models[-1]}.service")
+        return Process(f"rts-user-models.target", services[-1],
+                       services[0])
 
     def edcs(self, edcs):
         """
@@ -76,7 +115,7 @@ Description=All user models
         """
         services = [f"rts-edc_{edc}.service" for edc in edcs]
         self.serialize_units(services)
-        return Process(True, services[0], services[-1])
+        return Process(services[0], services[-1])
 
     def streaming(self, options):
         # check transport specifier exists
@@ -85,7 +124,9 @@ Description=All user models
             raise Exception(f"option '{ts}' must be determined when reading options")
 
         # set up port
-        fpath = path.join(self.target_dir, "rts-daq.network")
+        network_dir = "/run/systemd/network"
+        os.makedirs(network_dir, exist_ok=True)
+        fpath = path.join(network_dir, "rts-daq.network")
         with open(fpath, "wt") as f:
             f.write(f"""[Match]
 Name={options['DAQ_ETH_DEV']}
@@ -113,8 +154,9 @@ Wants={" ".join(services)}
 
         for service in services:
             self.link_to(service, targ_unit_name)
+            self.part_of(service, targ_unit_name)
 
-        return Process(True, services[0], services[-1])
+        return Process(targ_unit_name, services[-1], services[0])
 
     def serialize_units(self, services):
         """
@@ -123,8 +165,15 @@ Wants={" ".join(services)}
         for i in range(1, len(services)):
             self.after(services[i], services[i-1])
 
+    def serialize_processes(self, processes):
+        """
+        Take a list of processes and put one after the other
+        """
+        for i in range(1, len(processes)):
+            self.after(processes[i].first_service, processes[i-1].end)
+
     def create_world_target(self):
-        with open(path.join(self.target_dir, "rts-world.target", "wt")) as world:
+        with open(path.join(self.target_dir, "rts-world.target"), "wt") as world:
             world.write("""[Unit]
 Description=All model and streaming services for Front End servers
 
@@ -138,11 +187,12 @@ WantedBy=multi-user.target
 
         :return:
         """
-        wants_path = path.join(self.target_dir, f"{target_name}.wants")
-        os.makedirs(wants_path, exist_ok=True)
-        targ = path.join("..", unit_name)
-        link = path.join(wants_path, unit_name)
-        os.symlink(targ, link)
+        override_path = path.join(self.target_dir, f"{target_name}.d")
+        os.makedirs(override_path, exist_ok=True)
+        conf_path = path.join(override_path, f"wants_{unit_name}.conf")
+        with open(conf_path, "wt") as f:
+            f.write(f"""[Unit]
+Wants={unit_name}""")
 
     def part_of(self, unit_name, parent_name):
         """
@@ -150,7 +200,7 @@ WantedBy=multi-user.target
         """
         unit_d = path.join(self.target_dir, f"{unit_name}.d")
         os.makedirs(unit_d, exist_ok=True)
-        fname = path.join(unit_d, f"part of {parent_name}.conf")
+        fname = path.join(unit_d, f"part_of_{parent_name}.conf")
         with open(fname, "wt") as conf:
             conf.write(f"""[Unit]
 PartOf={parent_name}
@@ -162,8 +212,20 @@ PartOf={parent_name}
         """
         after_d = path.join(self.target_dir, f"{after_name}.d")
         os.makedirs(after_d, exist_ok=True)
-        fname = path.join(after_d, f"after {before_name}.conf")
+        fname = path.join(after_d, f"after_{before_name}.conf")
         with open(fname, "wt") as conf:
             conf.write(f"""[Unit]
 After={before_name}
+""")
+
+    def requires(self, after_name, before_name):
+        """
+        Make the systemd unit after_name require before_name is finished
+        """
+        after_d = path.join(self.target_dir, f"{after_name}.d")
+        os.makedirs(after_d, exist_ok=True)
+        fname = path.join(after_d, f"require_{before_name}.conf")
+        with open(fname, "wt") as conf:
+            conf.write(f"""[Unit]
+Requires={before_name}
 """)

support/systemd/generator/log.py

+def klog(line):
+    with open("/dev/kmsg","wt") as f:
+        f.write(f"fe_generator: {line}\n")
\ No newline at end of file

support/systemd/generator/main.py

@@ -6,7 +6,12 @@
 from sequencer import Sequencer
 from options import get_options
 import sys
+from log import klog
+
 
 if __name__ == '__main__':
-    seq = Sequencer(get_options(), sys.argv[1])
+    klog(f"in python with args {sys.argv}")
+    seq = Sequencer(get_options(), sys.argv[2])
+    klog(f"Sequencer created")
     seq.create_start_sequence()
+    klog(f"sequence complete")

support/systemd/generator/options.py

@@ -23,20 +23,21 @@ class Variable(object):
 # net every expected variable is set up here, only those that need translation from a string
 # or need a default value
 variables = {
-    'USER_MODELS': Variable(split),
-    'EPICS_ONLY_MODELS': Variable(split),
-    'EDC': Variable(split),
+    'USER_MODELS': Variable(split, default=[]),
+    'EPICS_ONLY_MODELS': Variable(split, default=[]),
+    'EDC': Variable(split, default=[]),
     'IS_DOLPHIN_NODE': Variable(boolean, False),
     'DAQ_STREAMING': Variable(boolean, False),
+    'CDSRFM': Variable(boolean, False),
+    'START_MODELS': Variable(boolean, False),
 }
 
 
 def get_options():
     # first check /etc/advligorts/systemd_env.. files
-    options = {}
-    read_env_file("/etc/advligorts/systemd_env", options)
+    options = {key: value for key,value in os.environ.items()}
     host_name = os.uname()[1]
-    read_env_file(f"/etc/advligorts/systemd_env_{host_name}", options)
+
 
     # also if rtsystab is available, prefer its unit list
 
@@ -82,40 +83,27 @@ def get_options():
     options['HAS_IOP_MODEL'] = 'IOP_MODEL' in options
     options['HAS_USER_MODELS'] = len(filtered_user_models) > 0
 
-    options['HAS_EDC'] = "EDCS" in options and len(options["EDCS"]) > 0
+    options['HAS_EDC'] = "EDC" in options and len(options["EDC"]) > 0
 
     if 'cps_xmit_args' in options:
         options['transport_specifier'] = 'cps_xmit'
 
-
-def read_env_file(fname, options):
-    try:
-        with open(fname, "rt") as f:
-            for line in f.readlines():
-                if line.strip()[0] == '#':
-                    continue
-                # ignore anything without an equal sign
-                if line.find('=') < 0:
-                    continue
-                pair = line.split('=')
-                key = pair[0].strip()
-                value = pair[1].strip()
-                options[key] = value
-    except IOError:
-        # don't do anything if file not there
-        pass
+    return options
 
 
 def read_rtsystab(fname, options, host_name):
     try:
         with open(fname, "rt") as f:
-            for line in f.readlines():
-                if line.strip()[0] == '#':
+            for line_raw in f.readlines():
+                line = line_raw.strip()
+                if len(line) <= 0:
+                    continue
+                if line[0] == '#':
                     continue
                 words = [word.strip() for word in line.split()]
                 if len(words) > 1 and words[0] == host_name:
                     options["IOP_MODEL"] = words[1]
-                    options["USER_MODELS"] = words[2:]
+                    options["USER_MODELS"] = " ".join(words[2:])
                     return
     except IOError:
         pass

support/systemd/generator/sequencer.py

@@ -3,7 +3,8 @@
 from front_end import FrontEndProcesses
 from cdsrfm import CDSRFMProcesses
 import os.path as path
-import os
+from log import klog
+
 
 class Sequencer(object):
 
@@ -19,72 +20,90 @@ class Sequencer(object):
 
     def create_start_sequence(self):
         if not self.options["START_MODELS"]:
+            klog("START_MODELS is false. Quitting")
             return
         if self.options["CDSRFM"]:
-            sequence = self.create_cdsrfm_start_sequence(CDSRFMProcesses(self.target_dir))
+            klog("is CDSRFM host")
+            before_world, world = self.create_cdsrfm_start_sequence(self.processes,
+                                                         CDSRFMProcesses(self.target_dir))
         else:
-            sequence = self.create_frontend_start_sequence()
-        self.link_sequence(sequence)
+            klog("is standard front end host")
+            before_world, world = self.create_frontend_start_sequence()
+        self.link_sequence(before_world, world)
 
     def create_frontend_start_sequence(self):
-        sequence = []
+        before_world = []
+        world = []
+        models = False
         if self.options["HAS_DOLPHIN_PORT"] and self.options["IS_DOLPHIN_NODE"]:
-            sequence.append(self.processes.dolphin_port())
-            sequence.append(self.delay(15, "wait_for_dolphin_port"))
+            before_world.append(self.processes.dolphin_port())
+            before_world.append(self.delay(15, "dolphin_port"))
         if self.options["IS_DOLPHIN_NODE"]:
-            sequence.append(self.processes.dolphin_drivers())
-            sequence.append(self.delay(15), "wait_for_dolphin_driver")
+            before_world.append(self.processes.dolphin_drivers())
+        before_world.append(self.delay(30, "startup"))
         if self.options["HAS_EPICS_ONLY_MODELS"]:
-            sequence.append(self.processes.epics_only_models(self.options["EPICS_ONLY_MODELS"]))
+            world.append(self.processes.epics_only_models(self.options["EPICS_ONLY_MODELS"]))
+            models = True
         if self.options["HAS_IOP_MODEL"]:
-            sequence.append(self.processes.iop_model(self.options["IOP_MODEL"]))
+            world.append(self.processes.iop_model(self.options["IOP_MODEL"]))
+            models = True
         if self.options["HAS_EDC"]:
-            sequence.append(self.processes.edcs(self.options["EDCS"]))
+            world.append(self.processes.edcs(self.options["EDC"]))
         if self.options["DAQ_STREAMING"]:
-            sequence.append(self.processes.streaming(self.options))
+            world.append(self.processes.streaming(self.options))
         if self.options["HAS_USER_MODELS"]:
-            sequence.append(self.processes.user_models(self.options["USER_MODELS"]))
-        return sequence
+            if self.options["HAS_IOP_MODEL"]:
+                world.append(self.processes.user_models(self.options["USER_MODELS"],
+                                                        self.options["IOP_MODEL"]))
+                models = True
+            else:
+                klog("Can't have user models without an IOP model")
+        if models:
+            self.processes.models()
+        return before_world, world
 
-    @staticmethod
-    def create_cdsrfm_start_sequence(front_end_processes, cdsrfm_processes):
-        sequence = [
+    def create_cdsrfm_start_sequence(self, front_end_processes, cdsrfm_processes):
+        before_world = [
             front_end_processes.dolphin_port(),
+            self.delay(15, "dolphin_port"),
             front_end_processes.dolphin_drivers(),
+            self.delay(30, "startup"),]
+        world = [
+            cdsrfm_processes.module(),
             cdsrfm_processes.epics(),
-            cdsrfm_processes.module()]
-        return sequence
+            ]
+        return before_world, world
 
-    def link_sequence(self, sequence):
+    def link_sequence(self, before_world, world):
         self.processes.create_world_target()
 
         # link the first of each process to multi-user or to the world target
-        for process in sequence:
-            if process.is_in_world:
-                self.processes.wanted_by(process.start, "rts-world.target")
-                self.processes.part_of(process.start, "rts-world.target")
-            else:
-                self.processes.link_to(process.start, "multi-user.target")
+        for process in before_world:
+            self.processes.link_to(process.start, "multi-user.target")
+        for process in world:
+            self.processes.link_to(process.start, "rts-world.target")
+            self.processes.part_of(process.start, "rts-world.target")
+        if len(before_world) > 0:
+            self.processes.after("rts-world.target", before_world[-1].end)
+        self.processes.link_to("rts-world.target", "multi-user.target")
+        self.processes.serialize_processes(before_world + world)
 
-        for i in range(1, len(sequence)):
-            self.processes.after(sequence[i].start, sequence[i-1].end)
 
     class Delay(object):
-        is_in_world = False
-
         def __init__(self, unit_name):
             self.start = unit_name
             self.end = unit_name
+            self.first_service = unit_name
 
     def delay(self, time_s, name):
-        file_name = f"{name}.service"
+        file_name = f"rts-delay-{name}.service"
         file_path = path.join(self.target_dir, file_name)
         with open(file_path, "wt") as f:
-            f"""[Unit]
+            f.write(f"""[Unit]
 Description=Delay for {time_s} seconds
 
 [Service]
 ExecStartPre=/bin/sleep {time_s}
-ExecStart=/usr/bin/echo 'finished waiting for {name}'
-"""
-        return self.Delay(file_name)
\ No newline at end of file
+ExecStart=/bin/echo 'finished waiting for {name}'
+""")
+        return self.Delay(file_name)