Changed how fuzz tests are iterated to allow powerloss-fuzz testing

Instead of iterating over a number of seeds in the test itself, the
seeds are now permuted as a part of normal test defines.

This lets each seed take advantage of other test features, mainly the
ability to test powerlosses heuristically.

This is probably how it should have been done in the first place, but
the permutation tests can't do this since the number of permutations
changes as the size of the test input changes. The test define system
can't handle that very well.

The tradeoffs here are:

- We can't do cross-fuzz checks, such as the balance checks in the rbyd
  tests, though those really should be moved to benchmarks anyways.

- The large number of cheap fuzz permutations skews the total
  permutation count, though I'm not sure this matters.

  before: 3083 permutations (-Gnor)
  after: 409893 permutations (-Gnor)

scripts/bench.py

@@ -758,20 +758,22 @@ def find_ids(runner, bench_ids=[], **args):
     # find suite/case by id
     bench_ids_ = []
     for id in bench_ids:
+        # strip permutation
+        name, *_ = id.split(':', 1)
         bench_ids__ = []
         # resolve globs
-        if '*' in id:
+        if '*' in name:
             bench_ids__.extend(suite
                 for suite in expected_suite_perms.keys()
-                if fnmatch.fnmatch(suite, id))
+                if fnmatch.fnmatch(suite, name))
             bench_ids__.extend(case_
                 for case_ in expected_case_perms.keys()
-                if fnmatch.fnmatch(case_, id))
+                if fnmatch.fnmatch(case_, name))
         # literal suite
-        elif id in expected_suite_perms:
+        elif name in expected_suite_perms:
             bench_ids__.append(id)
         # literal case
-        elif id in expected_case_perms:
+        elif name in expected_case_perms:
             bench_ids__.append(id)
 
         # no suite/case found? error

scripts/test.py

@@ -767,20 +767,22 @@ def find_ids(runner, test_ids=[], **args):
     # find suite/case by id
     test_ids_ = []
     for id in test_ids:
+        # strip permutation
+        name, *_ = id.split(':', 1)
         test_ids__ = []
         # resolve globs
-        if '*' in id:
+        if '*' in name:
             test_ids__.extend(suite
                 for suite in expected_suite_perms.keys()
-                if fnmatch.fnmatch(suite, id))
+                if fnmatch.fnmatch(suite, name))
             test_ids__.extend(case_
                 for case_ in expected_case_perms.keys()
-                if fnmatch.fnmatch(case_, id))
+                if fnmatch.fnmatch(case_, name))
         # literal suite
-        elif id in expected_suite_perms:
+        elif name in expected_suite_perms:
             test_ids__.append(id)
         # literal case
-        elif id in expected_case_perms:
+        elif name in expected_case_perms:
             test_ids__.append(id)
 
         # no suite/case found? error

tests/t5_dirs.toml

@@ -706,99 +706,97 @@ code = '''
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.PARENT = [false, true]
 defines.REMOUNT = [false, true]
-defines.SAMPLES = 10
+defines.SEED = 'range(10)'
-# -1 => all pseudo-random seeds
+reentrant = true
-#  n => reproduce a specific seed
-defines.SEED = -1
 code = '''
-    // iterate through severals seeds that we can reproduce easily
+    // format once per test
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+    lfs_t lfs;
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+    int err = lfsr_mount(&lfs, cfg);
-            seed++) {
+    if (err) {
-        printf("--- seed: %d ---\n", seed);
-        // reset lfs here each iteration
-        lfs_t lfs;
         lfsr_format(&lfs, cfg) => 0;
         lfsr_mount(&lfs, cfg) => 0;
-        if (PARENT) {
-            lfsr_mkdir(&lfs, "parent") => 0;
-        }
-
-        // set up a simulation to compare against
-        lfs_size_t *sim = malloc(N*sizeof(lfs_size_t));
-        lfs_size_t sim_size = 0;
-
-        uint32_t prng = seed;
-        for (lfs_size_t i = 0; i < N; i++) {
-            // choose a pseudo-random number, truncate to 4 decimals
-            lfs_size_t x = TEST_PRNG(&prng) % 1000;
-
-            // insert into our sim
-            for (lfs_size_t j = 0;; j++) {
-                if (j >= sim_size || sim[j] >= x) {
-                    // already seen? skip
-                    if (sim[j] == x) {
-                        goto next;
-                    }
-
-                    // insert
-                    memmove(&sim[j+1], &sim[j],
-                            (sim_size-j)*sizeof(lfs_size_t));
-                    sim_size += 1;
-                    sim[j] = x;
-
-                    // remount?
-                    if (REMOUNT) {
-                        lfsr_unmount(&lfs) => 0;
-                        lfsr_mount(&lfs, cfg) => 0;
-                        // grm should be zero here
-                        assert(lfs.grm[0] == 0);
-                    }
-                    break;
-                }
-            }
-
-            // create a directory here
-            char name[256];
-            sprintf(name, "%s/dir%04d", (PARENT ? "parent" : ""), x);
-            lfsr_mkdir(&lfs, name) => 0;
-        next:;
-        }
-
-        // test that our directories match our simulation
-        for (lfs_size_t j = 0; j < sim_size; j++) {
-            char name[256];
-            sprintf(name, "%s/dir%04d", (PARENT ? "parent" : ""), sim[j]);
-            struct lfs_info info;
-            lfsr_stat(&lfs, name, &info) => 0;
-            char name2[256];
-            sprintf(name2, "dir%04d", sim[j]);
-            assert(strcmp(info.name, name2) == 0);
-            assert(info.type == LFS_TYPE_DIR);
-        }
-
-        lfsr_dir_t dir;
-        lfsr_dir_open(&lfs, &dir, (PARENT ? "parent" : "/")) => 0;
-        struct lfs_info info;
-        lfsr_dir_read(&lfs, &dir, &info) => 0;
-        assert(strcmp(info.name, ".") == 0);
-        assert(info.type == LFS_TYPE_DIR);
-        lfsr_dir_read(&lfs, &dir, &info) => 0;
-        assert(strcmp(info.name, "..") == 0);
-        assert(info.type == LFS_TYPE_DIR);
-        for (lfs_size_t j = 0; j < sim_size; j++) {
-            char name[256];
-            sprintf(name, "dir%04d", sim[j]);
-            lfsr_dir_read(&lfs, &dir, &info) => 0;
-            assert(strcmp(info.name, name) == 0);
-            assert(info.type == LFS_TYPE_DIR);
-        }
-        lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
-
-        // clean up sim/lfs
-        free(sim);
-        lfsr_unmount(&lfs) => 0;
     }
+
+    if (PARENT) {
+        err = lfsr_mkdir(&lfs, "parent");
+        assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
+    }
+
+    // set up a simulation to compare against
+    lfs_size_t *sim = malloc(N*sizeof(lfs_size_t));
+    lfs_size_t sim_size = 0;
+
+    uint32_t prng = SEED;
+    for (lfs_size_t i = 0; i < N; i++) {
+        // choose a pseudo-random number, truncate to 4 decimals
+        lfs_size_t x = TEST_PRNG(&prng) % 1000;
+
+        // insert into our sim
+        for (lfs_size_t j = 0;; j++) {
+            if (j >= sim_size || sim[j] >= x) {
+                // already seen? skip
+                if (j < sim_size && sim[j] == x) {
+                    goto next;
+                }
+
+                // insert
+                memmove(&sim[j+1], &sim[j],
+                        (sim_size-j)*sizeof(lfs_size_t));
+                sim_size += 1;
+                sim[j] = x;
+                break;
+            }
+        }
+
+        // create a directory here
+        char name[256];
+        sprintf(name, "%s/dir%04d", (PARENT ? "parent" : ""), x);
+        err = lfsr_mkdir(&lfs, name);
+        assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
+    next:;
+    }
+
+    // remount?
+    if (REMOUNT) {
+        lfsr_unmount(&lfs) => 0;
+        lfsr_mount(&lfs, cfg) => 0;
+        // grm should be zero here
+        assert(lfs.grm[0] == 0);
+    }
+
+    // test that our directories match our simulation
+    for (lfs_size_t j = 0; j < sim_size; j++) {
+        char name[256];
+        sprintf(name, "%s/dir%04d", (PARENT ? "parent" : ""), sim[j]);
+        struct lfs_info info;
+        lfsr_stat(&lfs, name, &info) => 0;
+        char name2[256];
+        sprintf(name2, "dir%04d", sim[j]);
+        assert(strcmp(info.name, name2) == 0);
+        assert(info.type == LFS_TYPE_DIR);
+    }
+
+    lfsr_dir_t dir;
+    lfsr_dir_open(&lfs, &dir, (PARENT ? "parent" : "/")) => 0;
+    struct lfs_info info;
+    lfsr_dir_read(&lfs, &dir, &info) => 0;
+    assert(strcmp(info.name, ".") == 0);
+    assert(info.type == LFS_TYPE_DIR);
+    lfsr_dir_read(&lfs, &dir, &info) => 0;
+    assert(strcmp(info.name, "..") == 0);
+    assert(info.type == LFS_TYPE_DIR);
+    for (lfs_size_t j = 0; j < sim_size; j++) {
+        char name[256];
+        sprintf(name, "dir%04d", sim[j]);
+        lfsr_dir_read(&lfs, &dir, &info) => 0;
+        assert(strcmp(info.name, name) == 0);
+        assert(info.type == LFS_TYPE_DIR);
+    }
+    lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
+
+    // clean up sim/lfs
+    free(sim);
+    lfsr_unmount(&lfs) => 0;
 '''