Added back heuristic-based power-loss testing

The main change here from the previous test framework design is:

1. Powerloss testing remains in-process, speeding up testing.

2. The state of a test, included all powerlosses, is encoded in the
   test id + leb16 encoded powerloss string. This means exhaustive
   testing can be run in CI, but then easily reproduced locally with
   full debugger support.

   For example:

   ./scripts/test.py test_dirs#reentrant_many_dir#10#1248g1g2 --gdb

   Will run the test test_dir, case reentrant_many_dir, permutation #10,
   with powerlosses at 1, 2, 4, 8, 16, and 32 cycles. Dropping into gdb
   if an assert fails.

The changes to the block-device are a work-in-progress for a
lazily-allocated/copy-on-write block device that I'm hoping will keep
exhaustive testing relatively low-cost.

scripts/test.py

@@ -73,8 +73,6 @@ class TestCase:
         self.in_ = config.pop('in',
             config.pop('suite_in', None))
 
-        self.normal = config.pop('normal',
-            config.pop('suite_normal', True))
         self.reentrant = config.pop('reentrant',
             config.pop('suite_reentrant', False))
 
@@ -159,7 +157,6 @@ class TestSuite:
             # a couple of these we just forward to all cases
             defines = config.pop('defines', {})
             in_ = config.pop('in', None)
-            normal = config.pop('normal', True)
             reentrant = config.pop('reentrant', False)
 
             self.cases = []
@@ -172,7 +169,6 @@ class TestSuite:
                     'suite': self.name,
                     'suite_defines': defines,
                     'suite_in': in_,
-                    'suite_normal': normal,
                     'suite_reentrant': reentrant,
                     **case}))
 
@@ -181,7 +177,6 @@ class TestSuite:
                 set(case.defines) for case in self.cases))
 
             # combine other per-case things
-            self.normal = any(case.normal for case in self.cases)
             self.reentrant = any(case.reentrant for case in self.cases)
 
         for k in config.keys():
@@ -236,6 +231,12 @@ def compile(**args):
             f.write = write
             f.writeln = writeln
 
+            f.writeln("// Generated by %s:" % sys.argv[0])
+            f.writeln("//")
+            f.writeln("// %s" % ' '.join(sys.argv))
+            f.writeln("//")
+            f.writeln()
+
             # redirect littlefs tracing
             f.writeln('#define LFS_TRACE_(fmt, ...) do { \\')
             f.writeln(8*' '+'extern FILE *test_trace; \\')
@@ -366,10 +367,10 @@ def compile(**args):
                 f.writeln(4*' '+'.id = "%s",' % suite.id())
                 f.writeln(4*' '+'.name = "%s",' % suite.name)
                 f.writeln(4*' '+'.path = "%s",' % suite.path)
-                f.writeln(4*' '+'.types = %s,'
-                    % ' | '.join(filter(None, [
-                        'TEST_NORMAL' if suite.normal else None,
-                        'TEST_REENTRANT' if suite.reentrant else None])))
+                f.writeln(4*' '+'.flags = %s,'
+                    % (' | '.join(filter(None, [
+                        'TEST_REENTRANT' if suite.reentrant else None]))
+                        or 0))
                 if suite.defines:
                     # create suite define names
                     f.writeln(4*' '+'.define_names = (const char *const[]){')
@@ -384,10 +385,10 @@ def compile(**args):
                     f.writeln(12*' '+'.id = "%s",' % case.id())
                     f.writeln(12*' '+'.name = "%s",' % case.name)
                     f.writeln(12*' '+'.path = "%s",' % case.path)
-                    f.writeln(12*' '+'.types = %s,'
-                        % ' | '.join(filter(None, [
-                            'TEST_NORMAL' if case.normal else None,
-                            'TEST_REENTRANT' if case.reentrant else None])))
+                    f.writeln(12*' '+'.flags = %s,'
+                        % (' | '.join(filter(None, [
+                            'TEST_REENTRANT' if case.reentrant else None]))
+                            or 0))
                     f.writeln(12*' '+'.permutations = %d,'
                         % len(case.permutations))
                     if case.defines:
@@ -461,12 +462,13 @@ def runner(**args):
             '--error-exitcode=4',
             '-q'])
 
-    # filter tests?
-    if args.get('normal'):    cmd.append('-n')
-    if args.get('reentrant'): cmd.append('-r')
+    # other context
     if args.get('geometry'):
         cmd.append('-G%s' % args.get('geometry'))
 
+    if args.get('powerloss'):
+        cmd.append('-p%s' % args.get('powerloss'))
+
     # defines?
     if args.get('define'):
         for define in args.get('define'):
@@ -476,12 +478,13 @@ def runner(**args):
 
 def list_(**args):
     cmd = runner(**args)
-    if args.get('summary'):         cmd.append('--summary')
-    if args.get('list_suites'):     cmd.append('--list-suites')
-    if args.get('list_cases'):      cmd.append('--list-cases')
-    if args.get('list_paths'):      cmd.append('--list-paths')
-    if args.get('list_defines'):    cmd.append('--list-defines')
-    if args.get('list_geometries'): cmd.append('--list-geometries')
+    if args.get('summary'):          cmd.append('--summary')
+    if args.get('list_suites'):      cmd.append('--list-suites')
+    if args.get('list_cases'):       cmd.append('--list-cases')
+    if args.get('list_paths'):       cmd.append('--list-paths')
+    if args.get('list_defines'):     cmd.append('--list-defines')
+    if args.get('list_geometries'):  cmd.append('--list-geometries')
+    if args.get('list_powerlosses'): cmd.append('--list-powerlosses')
 
     if args.get('verbose'):
         print(' '.join(shlex.quote(c) for c in cmd))
@@ -598,11 +601,12 @@ def run_stage(name, runner_, **args):
     passed_suite_perms = co.defaultdict(lambda: 0)
     passed_case_perms = co.defaultdict(lambda: 0)
     passed_perms = 0
+    powerlosses = 0
     failures = []
     killed = False
 
     pattern = re.compile('^(?:'
-            '(?P<op>running|finished|skipped) '
+            '(?P<op>running|finished|skipped|powerloss) '
                 '(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)'
             '|' '(?P<path>[^:]+):(?P<lineno>\d+):(?P<op_>assert):'
                 ' *(?P<message>.*)' ')$')
@@ -613,6 +617,7 @@ def run_stage(name, runner_, **args):
         nonlocal passed_suite_perms
         nonlocal passed_case_perms
         nonlocal passed_perms
+        nonlocal powerlosses
         nonlocal locals
 
         # run the tests!
@@ -659,6 +664,9 @@ def run_stage(name, runner_, **args):
                         last_id = m.group('id')
                         last_output = []
                         last_assert = None
+                    elif op == 'powerloss':
+                        last_id = m.group('id')
+                        powerlosses += 1
                     elif op == 'finished':
                         passed_suite_perms[m.group('suite')] += 1
                         passed_case_perms[m.group('case')] += 1
@@ -766,6 +774,8 @@ def run_stage(name, runner_, **args):
                                 len(expected_case_perms))
                                 if not args.get('by_cases') else None,
                             '%d/%d perms' % (passed_perms, expected_perms),
+                            '%dpls!' % powerlosses
+                                if powerlosses else None,
                             '\x1b[31m%d/%d failures\x1b[m'
                                 % (len(failures), expected_perms)
                                 if failures else None]))))
@@ -785,6 +795,7 @@ def run_stage(name, runner_, **args):
     return (
         expected_perms,
         passed_perms,
+        powerlosses,
         failures,
         killed)
     
@@ -806,33 +817,34 @@ def run(**args):
 
     expected = 0
     passed = 0
+    powerlosses = 0
     failures = []
-    for type, by in it.product(
-            ['normal', 'reentrant'],
-            expected_case_perms.keys() if args.get('by_cases')
-                else expected_suite_perms.keys() if args.get('by_suites')
-                else [None]):
+    for by in (expected_case_perms.keys() if args.get('by_cases')
+            else expected_suite_perms.keys() if args.get('by_suites')
+            else [None]):
         # rebuild runner for each stage to override test identifier if needed
         stage_runner = runner(**args | {
-            'test_ids': [by] if by is not None else args.get('test_ids', []),
-            'normal': type == 'normal',
-            'reentrant': type == 'reentrant'})
+            'test_ids': [by] if by is not None else args.get('test_ids', [])})
 
         # spawn jobs for stage
-        expected_, passed_, failures_, killed = run_stage(
-            '%s %s' % (type, by or 'tests'), stage_runner, **args)
+        expected_, passed_, powerlosses_, failures_, killed = run_stage(
+            by or 'tests', stage_runner, **args)
         expected += expected_
         passed += passed_
+        powerlosses += powerlosses_
         failures.extend(failures_)
         if (failures and not args.get('keep_going')) or killed:
             break
 
     # show summary
     print()
-    print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs'
+    print('\x1b[%dmdone:\x1b[m %s' # %d/%d passed, %d/%d failed%s, in %.2fs'
         % (32 if not failures else 31,
-            passed, expected, len(failures), expected,
-            time.time()-start))
+            ', '.join(filter(None, [
+                '%d/%d passed' % (passed, expected),
+                '%d/%d failed' % (len(failures), expected),
+                '%dpls!' % powerlosses if powerlosses else None,
+                'in %.2fs' % (time.time()-start)]))))
     print()
 
     # print each failure
@@ -844,7 +856,7 @@ def run(**args):
     for failure in failures:
         # show summary of failure
         path, lineno = runner_paths[testcase(failure.id)]
-        defines = runner_defines[failure.id]
+        defines = runner_defines.get(failure.id, {})
 
         print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed'
             % (path, lineno, failure.id,
@@ -913,8 +925,9 @@ def main(**args):
             or args.get('list_cases')
             or args.get('list_paths')
             or args.get('list_defines')
+            or args.get('list_defaults')
             or args.get('list_geometries')
-            or args.get('list_defaults')):
+            or args.get('list_powerlosses')):
         list_(**args)
     else:
         run(**args)
@@ -930,7 +943,7 @@ if __name__ == "__main__":
         help="Description of testis to run. May be a directory, path, or \
             test identifier. Test identifiers are of the form \
             <suite_name>#<case_name>#<permutation>, but suffixes can be \
-            dropped to run any matching tests. Defaults to %r." % TEST_PATHS)
+            dropped to run any matching tests. Defaults to %s." % TEST_PATHS)
     parser.add_argument('-v', '--verbose', action='store_true',
         help="Output commands that run behind the scenes.")
     # test flags
@@ -945,20 +958,21 @@ if __name__ == "__main__":
         help="List the path for each test case.")
     test_parser.add_argument('--list-defines', action='store_true',
         help="List the defines for each test permutation.")
-    test_parser.add_argument('--list-geometries', action='store_true',
-        help="List the disk geometries used for testing.")
     test_parser.add_argument('--list-defaults', action='store_true',
         help="List the default defines in this test-runner.")
+    test_parser.add_argument('--list-geometries', action='store_true',
+        help="List the disk geometries used for testing.")
+    test_parser.add_argument('--list-powerlosses', action='store_true',
+        help="List the available power-loss scenarios.")
     test_parser.add_argument('-D', '--define', action='append',
         help="Override a test define.")
     test_parser.add_argument('-G', '--geometry',
         help="Filter by geometry.")
-    test_parser.add_argument('-n', '--normal', action='store_true',
-        help="Filter for normal tests. Can be combined.")
-    test_parser.add_argument('-r', '--reentrant', action='store_true',
-        help="Filter for reentrant tests. Can be combined.")
+    test_parser.add_argument('-p', '--powerloss',
+        help="Comma-separated list of power-loss scenarios to test. \
+            Defaults to 0,l.")
     test_parser.add_argument('-d', '--disk',
-        help="Use this file as the disk.")
+        help="Redirect block device operations to this file.")
     test_parser.add_argument('-t', '--trace',
         help="Redirect trace output to this file.")
     test_parser.add_argument('-o', '--output',