added some unit tests for checkdaqconfig.py. Now accepts hardcoded paths in...

added some unit tests for checkdaqconfig.py.  Now accepts hardcoded paths in master template, but will look in expected paths if the paths in the file are dummies (template variables for example).

.gitignore

@@ -36,3 +36,6 @@ src/daqd/autom4te.cache
 
 #kernel module build files
 modules.order
+
+#python compile files
+*__pycache__*
\ No newline at end of file

src/daqd/util/checkdaqconfig.py

@@ -45,9 +45,9 @@ parser.add_argument('-m', dest="master_template",
                     help="path to master file template, default: $target/target/daq/master.in", default="")
 parser.add_argument('-i', dest="ini_path", help="path to .ini files, default: $target/chans/daq", default="")
 parser.add_argument('-p', dest="par_path", help="path to .par files, default: $target/target/gds/param", default="")
-parser.add_argument('-d', dest="daq_name", help="name of DAQ (daq0, daq1)", required=True, default="")
 parser.add_argument('-b', dest="daq_base_path",
                     help="base path for daq running directories, default: $target", default="")
+parser.add_argument('daq_name', help="name of DAQ (daq0, daq1)")
 
 args = parser.parse_args()
 
@@ -88,7 +88,7 @@ daq_name = args.daq_name
 if target_path == "" and used_target:
     print("WARNING: target path is empty.", file=sys.stderr)
     print("Either a default path could not be created, or an empty target path was specified.", file=sys.stderr)
-    print("Specify a non-epty target directory with the -t option\n")
+    print("Specify a non-empty target directory with the -t option\n")
 
 # Print all paths
 print("Target directory:\t" + target_path)
@@ -103,12 +103,36 @@ os.makedirs(hash_archives, exist_ok=True)
 
 # do they exist?
 
-missing_paths = [p for p in [archive_path, master_template, ini_path, par_path, daq_base_path] if not path.exists(p)]
+missing_paths = [p for p in [archive_path, master_template, daq_base_path] if not path.exists(p)]
 
 if missing_paths:
     print("ERROR: could not find " + "\nERROR: could not find ".join(missing_paths), file=sys.stderr)
     sys.exit(2)
 
+if not path.exists(ini_path):
+    print(f"WARNING: {ini_path} does not exist.  Paths to .ini file must be hardcoded in master template.")
+
+if not path.exists(par_path):
+    print(f"WARNING: {par_path} does not exist.  Paths to .par file must be hardcoded in master template.")
+
+def replace_dummy_path(file_path, default_path):
+    """
+    Replace a possible dummy path in file_path with default_path.
+    Replace the path only if file_path does not exist.
+    If default_path/file_path also doesn't exist, throw an exception.
+
+    :param file_path: an absolute path to a file
+    :param default_path:
+    :return: if file_path exists, return file_poth, or else return default_path/basename(fname)
+    """
+
+    if path.exists(file_path):
+        return file_path
+    new_path = path.join(default_path, path.basename(file_path))
+    if path.exists(new_path):
+        return new_path
+    raise Exception(f"{file_path} could not be found and {new_path} could not be found")
+
 # get paths to all files in master file.
 try:
     with open(master_template) as f:
@@ -117,9 +141,12 @@ except IOError as e:
     print(f"Failed to open {master_template}: " + str(e), file=sys.stderr)
     sys.exit(1)
 
-master_names = [path.basename(i.strip()) for i in master_lines if i.strip()[0] != "#"]
-ini_files = [f"{ini_path}/{i}" for i in master_names if i[-3:] == "ini"]
-par_files = [f"{par_path}/{i}" for i in master_names if i[-3:] == "par"]
+master_paths = [i.strip() for i in master_lines if i.strip()[0] != "#"]
+ini_files = [replace_dummy_path(i, ini_path) for i in master_paths if i[-3:] == "ini"]
+par_files = [replace_dummy_path(i, par_path) for i in master_paths if i[-3:] == "par"]
+
+
+
 for f in par_files:
     print(f)
 ini_files.sort()
@@ -284,22 +311,20 @@ with open(lockfile, "w") as lock_f:
     new_hash_file = f"{hash_archives}/{new_archive_basename}/{hash_file_name}"
     os.makedirs(new_archive)
 
-    # create master file
-    with open(master_template) as mf:
-        master_contents = mf.read()
-
-    with open(new_master, "w") as mf:
-        t_vars = {"base_path": new_archive}
-        mf.write(Template(master_contents).substitute(t_vars))
-
     # copy in config files
     print("copy in config files")
+    master_entries = []
     for f in ini_files + par_files:
         fb = path.basename(f)
         dest = f"{new_archive}/{fb}"
         shutil.copyfile(f, dest)
+        master_entries.append(dest)
     print("copy complete")
 
+    # create master file
+    with open(new_master, "w") as mf:
+        mf.write("\n".join(master_entries))
+
     # link to master
     create_link(new_archive, master_link)
     update_daq_archives(new_archive)

src/daqd/util/checkdaqconfig_test.py

+#!/usr/bin/python
+
+# to run these tests: python3 -m unittest <this-file>
+
+import unittest
+import os
+import os.path as path
+import shutil
+
+test_dir = "checkdaqconfig_testfiles"
+test_hash = "7d134aad8c722085dda3b78b2ba20233"
+
+def get_test_dir(test_folder):
+    global test_dir
+
+    base_dir, _ = path.split(__file__)
+    abs_test_dir = path.join(base_dir, test_dir)
+    return path.join(abs_test_dir, test_folder)
+
+def test_master(self, files):
+    """
+    Test that the master file is pointing to precisely the set of files specified by files
+    and that it is pointing into the right hash directory.
+
+    :param self: A TestCase with self.hash_path defined
+    :param files:
+    :return: None
+    """
+    with open(f"{self.hash_path}/master", "rt") as f:
+        for l in f.readlines():
+            line = l.strip()
+            if len(line) > 0:
+                fpath, fname = path.split(line)
+                self.assertEqual(fpath, self.hash_path, "master file pointing to wrong directory")
+                self.assertIn(fname, files, f"{fname} not among expected files in master")
+                if fname in files:
+                    files.remove(fname)
+        self.assertEquals(len(files), 0, "some files expected in master were not found")
+
+def test_link(self, daq_name):
+    """
+    Test that the running directory is a link to the right hash file.
+
+    :param self:
+    :return:
+    """
+    link_path = f"{self.base_path}/{daq_name}/running"
+    self.assertTrue(path.lexists(link_path), "running doesn't exist")
+    self.assertTrue(path.islink(link_path), "running not a link")
+    real_path = path.realpath(link_path)
+    self.assertEqual(real_path, self.hash_path, f"running pointing to wrong path: {real_path}")
+
+class TargetTestCase(unittest.TestCase):
+    def setUp(self):
+        self.base_path = get_test_dir("target_path")
+        self.archive_path = f"{self.base_path}/target/daq/archive"
+        self.hash_path = f"{self.archive_path}/hash_archive/{test_hash}"
+        self.tearDown()
+
+    def tearDown(self):
+        shutil.rmtree(self.archive_path, ignore_errors=True)
+        shutil.rmtree(f"{self.base_path}/daq0", ignore_errors=True)
+
+    def test_default_paths(self):
+        status = os.system(f"./checkdaqconfig.py -t {self.base_path} daq0")
+        self.assertEqual(status, 0, f"default run exited with status {status}")
+        self.assertTrue(path.exists(self.hash_path), "hash directory not created or created with the wrong hash")
+        files = set(["test1.ini", "test1.par"])
+        test_master(self, files)
+        test_link(self, "daq0")
+        self.assertTrue(path.exists(f"{self.archive_path}/daqconfig.log"), "log file not created")
+
+    def test_specified_paths(self):
+        status = os.system(f"./checkdaqconfig.py -a {self.base_path}/target/daq/archive -m {self.base_path}/target/daq/master.in -i {self.base_path}/chans/daq -p {self.base_path}/target/gds/param -b {self.base_path} daq0")
+        self.assertEqual(status, 0, f"default run exited with status {status}")
+        self.assertTrue(path.exists(self.hash_path), "hash directory not created or created with the wrong hash")
+        files = set(["test1.ini", "test1.par"])
+        test_master(self, files)
+        test_link(self, "daq0")
+        self.assertTrue(path.exists(f"{self.archive_path}/daqconfig.log"), "log file not created")
+
+class ReplaceTestCase(unittest.TestCase):
+    def setUp(self):
+        self.base_path = get_test_dir("replace")
+        self.archive_path = f"{self.base_path}/target/daq/archive"
+        self.hash_path = f"{self.archive_path}/hash_archive/{test_hash}"
+        shutil.rmtree(self.base_path, ignore_errors=True)
+        shutil.copytree(get_test_dir("replace.base"), self.base_path, symlinks=True)
+        with open(f"{self.base_path}/target/daq/master.in", "at") as f:
+            f.write(f"\n$base_path/test1.par\n")
+
+    def tearDown(self):
+        shutil.rmtree(self.base_path, ignore_errors=True)
+
+    def test_replace_link(self):
+        status = os.system(f"./checkdaqconfig.py -t {self.base_path} daq0")
+        self.assertEqual(status, 0, f"default run exited with status {status}")
+        self.assertTrue(path.exists(self.hash_path), "hash directory not created or created with the wrong hash")
+        files = set(["test1.ini", "test1.par"])
+        test_master(self, files)
+        test_link(self, "daq0")
+        self.assertTrue(path.exists(f"{self.archive_path}/daqconfig.log"), "log file not created")
+
+class NotALinkTestCase(unittest.TestCase):
+    def setUp(self):
+        self.base_path = get_test_dir("not_a_link")
+        self.archive_path = f"{self.base_path}/target/daq/archive"
+        self.hash_path = f"{self.archive_path}/hash_archive/{test_hash}"
+        self.tearDown()
+
+    def tearDown(self):
+        shutil.rmtree(self.archive_path, ignore_errors=True)
+
+    def test_not_a_link_blocks(self):
+        status = os.system(f"./checkdaqconfig.py -t {self.base_path} daq0")
+        self.assertNotEqual(status, 0, f"default run should have failed")
+        self.assertFalse(path.islink(f"{self.base_path}/daq0/running"), "should not be a link")
+
+class HardPathTestCase(unittest.TestCase):
+    def setUp(self):
+        self.base_path = get_test_dir("hard_path")
+        self.archive_path = f"{self.base_path}/target/daq/archive"
+        self.hash_path = f"{self.archive_path}/hash_archive/{test_hash}"
+        self.master_file = f"{self.base_path}/target/daq/master.in"
+        self.tearDown()
+
+        with open(self.master_file, "wt") as mf:
+            mf.write(f"{self.base_path}/chans1/daq/test1.ini\n")
+            mf.write(f"{self.base_path}/target/gds1/param/test1.par\n")
+
+    def tearDown(self):
+        shutil.rmtree(self.archive_path, ignore_errors=True)
+        shutil.rmtree(f"{self.base_path}/daq0", ignore_errors=True)
+        try:
+            os.remove(self.master_file)
+        except FileNotFoundError:
+            pass
+
+    def test_hardcoded_paths(self):
+        status = os.system(f"./checkdaqconfig.py -t {self.base_path} daq0")
+        self.assertEqual(status, 0, f"default run exited with status {status}")
+        self.assertTrue(path.exists(self.hash_path), "hash directory not created or created with the wrong hash")
+        files = set(["test1.ini", "test1.par"])
+        test_master(self, files)
+        test_link(self, "daq0")
+        self.assertTrue(path.exists(f"{self.archive_path}/daqconfig.log"), "log file not created")
+
+if __name__ == "__main__":
+    unittest.main()

src/daqd/util/checkdaqconfig_testfiles/hard_path/chans1/daq/test1.ini

+Hello from test1.ini
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/hard_path/target/gds1/param/test1.par

+Hello from test1.par
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/not_a_link/chans/daq/test1.ini

+Hello from test1.ini
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/not_a_link/target/daq/master.in

+$base_path/test1.ini
+$base_path/test1.par
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/not_a_link/target/gds/param/test1.par

+Hello from test1.par
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/replace.base/chans/daq/test1.ini

+Hello from test1.ini
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/replace.base/daq0/running

+/home/erik.vonreis/projects/advligorts/src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/hash_archive/a9a1f767278c2af3fd6f025e6dc30ecb
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/daq0/240620_14:12:51

+/home/erik.vonreis/projects/advligorts/src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/hash_archive/a9a1f767278c2af3fd6f025e6dc30ecb
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/daq0/240620_14:20:36

+/home/erik.vonreis/projects/advligorts/src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/hash_archive/a9a1f767278c2af3fd6f025e6dc30ecb
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/daqconfig.log

+2020-06-24 14:12:51-0700,daq0,a9a1f767278c2af3fd6f025e6dc30ecb, a9a1f767278c2af3fd6f025e6dc30ecb

src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/hash_archive/a9a1f767278c2af3fd6f025e6dc30ecb/master

+/home/erik.vonreis/projects/advligorts/src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/hash_archive/a9a1f767278c2af3fd6f025e6dc30ecb/test1.ini
+#/home/erik.vonreis/projects/advligorts/src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/hash_archive/a9a1f767278c2af3fd6f025e6dc30ecb/test1.par
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/archive/hash_archive/a9a1f767278c2af3fd6f025e6dc30ecb/test1.ini

+Hello from test1.ini
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/replace.base/target/daq/master.in

+$base_path/test1.ini
+#$base_path/test1.par
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/replace.base/target/gds/param/test1.par

+Hello from test1.par
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/target_path/chans/daq/test1.ini

+Hello from test1.ini
\ No newline at end of file

src/daqd/util/checkdaqconfig_testfiles/target_path/target/daq/master.in

+$base_path/test1.ini
+$base_path/test1.par
\ No newline at end of file