Pytest enhancements

Adds

1. A pytest marker generate that marks a test as generating test data. These tests will only be run when the --generate flag is given, and all other tests won't be run in this case. This prevents accidentally overwriting test data.
2. A pytest fixture compare_noise which checks whether all noises in a budget are the same as in another budget. A list of noises passing, failing, or skipped is printed and an assertion error is raised if any traces are different.

The unit tests test_save_budgets and test_check_noise are added to test/budgets/test_budgets.py making use of the new fixture and marker. The compare_noise fixture is useful because it quickly checks if noises are the same and fails if any are different. So you can have pytest both continuously plotting a budget and quickly checking if any noises have changed by doing for example

ptw -- test/budgets/test_budgets.py -s -k test_check_noise
ptw -- test/budgets/test_budgets.py -s -k test_load[Aplus]

conftest.py

@@ -40,6 +40,14 @@ def pytest_addoption(parser):
         help="Do not preclear tpaths",
     )
 
+    parser.addoption(
+        "--generate",
+        action="store_true",
+        default=False,
+        dest="generate",
+        help="Generate test data",
+    )
+
 
 @pytest.fixture
 def plot(request):
@@ -270,3 +278,83 @@ def relfile_test(_file_, request, pre = None, post = None, fname = None):
     else:
         return relfile(_file_, testname, fname = fname)
 
+
+@pytest.fixture
+def compare_noise(pprint):
+    """
+    Fixture to compare two sets of traces
+
+    A list of noises passed, failed, and skipped are printed. Comparisons are
+    skipped if the psd's are sufficiently small (controlled by psd_tol) indicating
+    that the noise is essentially zero or if a trace is missing.
+
+    An assertion error is raised if any noises fail.
+    """
+    import numpy as np
+
+    def compare(traces, ref_traces, psd_tol=1e-52):
+        passed = []
+        failed = []
+        skipped = []
+        if ref_traces.budget:
+            for ref_trace in ref_traces:
+                if np.all(ref_trace.psd < psd_tol):
+                    skipped.append(ref_trace.name)
+                    continue
+
+                try:
+                    trace = traces[ref_trace.name]
+                except KeyError:
+                    skipped.append(ref_trace.name)
+                    continue
+
+                if np.allclose(trace.psd, ref_trace.psd, atol=0):
+                    passed.append(trace.name)
+                else:
+                    failed.append(trace.name)
+        else:
+            if np.allclose(ref_traces.psd, traces.psd, atol=0):
+                passed.append(traces.name)
+            else:
+                failed.append(traces.name)
+
+        pprint('Noises failed:')
+        pprint(40 * '-')
+        for noise in failed:
+            pprint(noise)
+        pprint(40 * '+')
+        pprint('Noises passed:')
+        pprint(40 * '-')
+        for noise in passed:
+            pprint(noise)
+        pprint(40 * '+')
+        pprint('Noises skipped:')
+        pprint(40 * '-')
+        for noise in skipped:
+            pprint(noise)
+
+        assert len(failed) == 0
+
+    return compare
+
+
+def pytest_collection_modifyitems(config, items):
+    """
+    Modifies tests to be selectively skipped with command line options
+
+    https://docs.pytest.org/en/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option
+    """
+    # run tests marked as generate if and only if --generate is given
+    # skip all others in this case
+    if config.getoption('--generate'):
+        skip = pytest.mark.skip(
+            reason='only running tests that generate data')
+        for item in items:
+            if 'generate' not in item.keywords:
+                item.add_marker(skip)
+    else:
+        skip = pytest.mark.skip(
+            reason='generates test data: needs --generate to run')
+        for item in items:
+            if 'generate' in item.keywords:
+                item.add_marker(skip)