Reintroduced test-defines into the new test_runner

This moves defines entirely into the runtime of the test_runner,
simplifying thing and reducing the amount of generated code that needs
to be build, at the cost of limiting test-defines to uintmax_t types.

This is implemented using a set of index-based scopes (created by
test.py) that allow different layers to override defines from other
layers, accessible through the global `test_define` function.

layers:
1. command-line overrides
2. per-case defines
3. per-geometry defines

.gitignore

@@ -4,6 +4,8 @@
 *.a
 *.ci
 *.csv
+*.t.c
+*.a.c
 
 # Testing things
 blocks/

Makefile

@@ -33,11 +33,13 @@ ASM := $(SRC:%.c=$(BUILDDIR)%.s)
 CGI := $(SRC:%.c=$(BUILDDIR)%.ci)
 
 TESTS ?= $(wildcard tests_/*.toml)
-TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) \
+TEST_SRC ?= $(SRC) \
-	$(SRC:%.c=$(BUILDDIR)%.t.c) \
+		$(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \
-	$(BUILDDIR)runners/test_runner.t.c
+		runners/test_runner.c
+TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) $(TEST_SRC:%.c=$(BUILDDIR)%.t.c)
 TEST_TASRC := $(TEST_TSRC:%.t.c=%.t.a.c)
 TEST_TAOBJ := $(TEST_TASRC:%.t.a.c=%.t.a.o)
+TEST_TADEP := $(TEST_TASRC:%.t.a.c=%.t.a.d)
 
 ifdef DEBUG
 override CFLAGS += -O0
@@ -141,6 +143,7 @@ summary: $(BUILDDIR)lfs.csv
 
 # rules
 -include $(DEP)
+-include $(TEST_TADEP)
 .SUFFIXES:
 .SECONDARY:
 
@@ -202,3 +205,4 @@ clean:
 	rm -f $(TEST_TSRC)
 	rm -f $(TEST_TASRC)
 	rm -f $(TEST_TAOBJ)
+	rm -f $(TEST_TADEP)

runners/test_runner.c

@@ -1,71 +1,363 @@
 
 #include "runners/test_runner.h"
-#include <getopt.h>
+#include "bd/lfs_testbd.h"
 
+#include <getopt.h>
+#include <sys/types.h>
 
 // disk geometries
 struct test_geometry {
     const char *name;
-    lfs_size_t read_size;
+    const uintmax_t *defines;
-    lfs_size_t prog_size;
-    lfs_size_t erase_size;
-    lfs_size_t erase_count;
 };
 
+// Note this includes the default configuration for test pre-defines
+#define TEST_GEOMETRY(name, read, prog, erase, count) \
+    {name, (const uintmax_t[]){ \
+        /* READ_SIZE */         read, \
+        /* PROG_SIZE */         prog, \
+        /* BLOCK_SIZE */        erase, \
+        /* BLOCK_COUNT */       count, \
+        /* BLOCK_CYCLES */      -1, \
+        /* CACHE_SIZE */        (64 % (prog) == 0) ? 64 : (prog), \
+        /* LOOKAHEAD_SIZE */    16, \
+        /* ERASE_VALUE */       0xff, \
+        /* ERASE_CYCLES */      0, \
+        /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \
+    }}
+
 const struct test_geometry test_geometries[] = {
-    // Made up geometries that works well for testing
+    // Made up geometry that works well for testing
-    {"small",    16,   16,     512, (1024*1024)/512},
+    TEST_GEOMETRY("small",    16,   16,     512, (1024*1024)/512),
-    {"medium",   16,   16,    4096, (1024*1024)/4096},
+    TEST_GEOMETRY("medium",   16,   16,    4096, (1024*1024)/4096),
-    {"big",      16,   16, 32*1024, (1024*1024)/(32*1024)},
+    TEST_GEOMETRY("big",      16,   16, 32*1024, (1024*1024)/(32*1024)),
     // EEPROM/NVRAM
-    {"eeprom",    1,    1,     512, (1024*1024)/512},
+    TEST_GEOMETRY("eeprom",    1,    1,     512, (1024*1024)/512),
     // SD/eMMC
-    {"emmc",    512,  512,     512, (1024*1024)/512},
+    TEST_GEOMETRY("emmc",    512,  512,     512, (1024*1024)/512),
-    // NOR flash
+    // NOR flash                       
-    {"nor",       1,    1,    4096, (1024*1024)/4096},
+    TEST_GEOMETRY("nor",       1,    1,    4096, (1024*1024)/4096),
-    // NAND flash
+    // NAND flash                      
-    {"nand",   4096, 4096, 32*1024, (1024*1024)/(32*1024)},
+    TEST_GEOMETRY("nand",   4096, 4096, 32*1024, (1024*1024)/(32*1024)),
 };
+
 const size_t test_geometry_count = (
         sizeof(test_geometries) / sizeof(test_geometries[0]));
 
 
+// test define lookup and management
+const uintmax_t *test_defines[3] = {NULL};
+const bool *test_define_masks[2] = {NULL};
+
+uintmax_t test_define(size_t define) {
+    if (test_define_masks[0] && test_define_masks[0][define]) {
+        return test_defines[0][define];
+    } else if (test_define_masks[1] && test_define_masks[1][define]) {
+        return test_defines[1][define];
+    } else {
+        return test_defines[2][define];
+    }
+}
+
+void test_define_geometry(const struct test_geometry *geometry) {
+    if (geometry) {
+        test_defines[2] = geometry->defines;
+    } else {
+        test_defines[2] = NULL;
+    }
+}
+
+void test_define_case(const struct test_case *case_, size_t perm) {
+    if (case_ && case_->defines) {
+        test_defines[1] = case_->defines[perm];
+        test_define_masks[1] = case_->define_mask;
+    } else {
+        test_defines[1] = NULL;
+        test_define_masks[1] = NULL;
+    }
+}
+
+struct override {
+    const char *name;
+    uintmax_t override;
+};
+
+void test_define_overrides(
+        const struct test_suite *suite,
+        const struct override *overrides,
+        size_t override_count) {
+    if (overrides && override_count > 0) {
+        uintmax_t *defines = malloc(suite->define_count * sizeof(uintmax_t));
+        memset(defines, 0, suite->define_count * sizeof(uintmax_t));
+        bool *define_mask = malloc(suite->define_count * sizeof(bool));
+        memset(define_mask, 0, suite->define_count * sizeof(bool));
+
+        // lookup each override in the suite defines, they may have a
+        // different index in each suite
+        for (size_t i = 0; i < override_count; i++) {
+            ssize_t index = -1;
+            for (size_t j = 0; j < suite->define_count; j++) {
+                if (strcmp(overrides[i].name, suite->define_names[j]) == 0) {
+                    index = j;
+                    break;
+                }
+            }
+
+            if (index >= 0) {
+                defines[index] = overrides[i].override;
+                define_mask[index] = true;
+            }
+        }
+
+        test_defines[0] = defines;
+        test_define_masks[0] = define_mask;
+    } else {
+        free((uintmax_t *)test_defines[0]);
+        test_defines[0] = NULL;
+        free((bool *)test_define_masks[0]);
+        test_define_masks[0] = NULL;
+    }
+}
+
+
+// operations we can do
+void summary(
+        struct override *overrides,
+        size_t override_count) {
+    (void)overrides;
+    (void)override_count;
+    printf("%-36s %7s %7s %7s %7s\n",
+            "", "geoms", "suites", "cases", "perms");
+    size_t cases = 0;
+    size_t perms = 0;
+    for (size_t i = 0; i < test_suite_count; i++) {
+        cases += test_suites[i]->case_count;
+
+        for (size_t j = 0; j < test_suites[i]->case_count; j++) {
+            perms += test_suites[i]->cases[j]->permutations;
+        }
+    }
+
+    printf("%-36s %7zu %7zu %7zu %7zu\n",
+            "TOTAL",
+            test_geometry_count,
+            test_suite_count,
+            cases,
+            test_geometry_count*perms);
+}
+
+void list_suites(
+        struct override *overrides,
+        size_t override_count) {
+    (void)overrides;
+    (void)override_count;
+    printf("%-36s %-12s %7s %7s %7s\n",
+            "id", "suite", "types", "cases", "perms");
+    for (size_t i = 0; i < test_suite_count; i++) {
+        size_t perms = 0;
+        for (size_t j = 0; j < test_suites[i]->case_count; j++) {
+            perms += test_suites[i]->cases[j]->permutations;
+        }
+
+        printf("%-36s %-12s %7s %7zu %7zu\n",
+                test_suites[i]->id,
+                test_suites[i]->name,
+                "n", // TODO
+                test_suites[i]->case_count,
+                test_geometry_count*perms);
+    }
+}
+
+void list_cases(
+        struct override *overrides,
+        size_t override_count) {
+    (void)overrides;
+    (void)override_count;
+    printf("%-36s %-12s %-12s %7s %7s\n",
+            "id", "suite", "case", "types", "perms");
+    for (size_t i = 0; i < test_suite_count; i++) {
+        for (size_t j = 0; j < test_suites[i]->case_count; j++) {
+            printf("%-36s %-12s %-12s %7s %7zu\n",
+                    test_suites[i]->cases[j]->id,
+                    test_suites[i]->name,
+                    test_suites[i]->cases[j]->name,
+                    "n", // TODO
+                    test_geometry_count
+                        * test_suites[i]->cases[j]->permutations);
+        }
+    }
+}
+
+void list_paths(
+        struct override *overrides,
+        size_t override_count) {
+    (void)overrides;
+    (void)override_count;
+    printf("%-36s %-36s\n", "id", "path");
+    for (size_t i = 0; i < test_suite_count; i++) {
+        for (size_t j = 0; j < test_suites[i]->case_count; j++) {
+            printf("%-36s %-36s\n",
+                    test_suites[i]->cases[j]->id,
+                    test_suites[i]->cases[j]->path);
+        }
+    }
+}
+
+void list_defines(
+        struct override *overrides,
+        size_t override_count) {
+    (void)overrides;
+    (void)override_count;
+    // TODO
+}
+
+void list_geometries(
+        struct override *overrides,
+        size_t override_count) {
+    (void)overrides;
+    (void)override_count;
+    printf("%-36s %7s %7s %7s %7s %7s\n",
+            "name", "read", "prog", "erase", "count", "size");
+    for (size_t i = 0; i < test_geometry_count; i++) {
+        test_define_geometry(&test_geometries[i]);
+
+        printf("%-36s %7ju %7ju %7ju %7ju %7ju\n",
+                test_geometries[i].name,
+                READ_SIZE,
+                PROG_SIZE,
+                BLOCK_SIZE,
+                BLOCK_COUNT,
+                BLOCK_SIZE*BLOCK_COUNT);
+    }
+}
+
+void run(
+        struct override *overrides,
+        size_t override_count) {
+    for (size_t i = 0; i < test_suite_count; i++) {
+        test_define_overrides(test_suites[i], overrides, override_count);
+
+        for (size_t j = 0; j < test_suites[i]->case_count; j++) {
+            for (size_t perm = 0;
+                    perm < test_geometry_count
+                        * test_suites[i]->cases[j]->permutations;
+                    perm++) {
+                size_t case_perm = perm / test_geometry_count;
+                size_t geom_perm = perm % test_geometry_count;
+
+                // setup defines
+                test_define_geometry(&test_geometries[geom_perm]);
+                test_define_case(test_suites[i]->cases[j], case_perm);
+
+                // create block device and configuration
+                lfs_testbd_t bd;
+
+                struct lfs_config cfg = {
+                    .context            = &bd,
+                    .read               = lfs_testbd_read,
+                    .prog               = lfs_testbd_prog,
+                    .erase              = lfs_testbd_erase,
+                    .sync               = lfs_testbd_sync,
+                    .read_size          = READ_SIZE,
+                    .prog_size          = PROG_SIZE,
+                    .block_size         = BLOCK_SIZE,
+                    .block_count        = BLOCK_COUNT,
+                    .block_cycles       = BLOCK_CYCLES,
+                    .cache_size         = CACHE_SIZE,
+                    .lookahead_size     = LOOKAHEAD_SIZE,
+                };
+
+                struct lfs_testbd_config bdcfg = {
+                    .erase_value        = ERASE_VALUE,
+                    .erase_cycles       = ERASE_CYCLES,
+                    .badblock_behavior  = BADBLOCK_BEHAVIOR,
+                    .power_cycles       = 0,
+                };
+
+                lfs_testbd_createcfg(&cfg, NULL, &bdcfg) => 0;
+
+                // filter?
+                if (test_suites[i]->cases[j]->filter) {
+                    bool filter = test_suites[i]->cases[j]->filter(
+                            &cfg, case_perm);
+                    if (!filter) {
+                        printf("skipped %s#%zu\n",
+                                test_suites[i]->cases[j]->id,
+                                perm);
+                        continue;
+                    }
+                }
+
+                // run the test
+                printf("running %s#%zu\n", test_suites[i]->cases[j]->id, perm);
+
+                test_suites[i]->cases[j]->run(&cfg, case_perm);
+
+                printf("finished %s#%zu\n", test_suites[i]->cases[j]->id, perm);
+
+                // cleanup
+                lfs_testbd_destroy(&cfg) => 0;
+
+                test_define_geometry(NULL);
+                test_define_case(NULL, 0);
+            }
+        }
+
+        test_define_overrides(NULL, NULL, 0);
+    }
+}
+
+
+
+
 // option handling
 enum opt_flags {
     OPT_HELP            = 'h',
-    OPT_LIST            = 'l',
+    OPT_SUMMARY         = 'Y',
-    OPT_LIST_PATHS      = 1,
+    OPT_LIST_SUITES     = 1,
-    OPT_LIST_DEFINES    = 2,
+    OPT_LIST_CASES      = 'l',
-    OPT_LIST_GEOMETRIES = 3,
+    OPT_LIST_PATHS      = 2,
+    OPT_LIST_DEFINES    = 3,
+    OPT_LIST_GEOMETRIES = 4,
+    OPT_DEFINE          = 'D',
 };
 
+const char *short_opts = "hYlD:";
+
 const struct option long_opts[] = {
-    {"help",            no_argument, NULL, OPT_HELP},
+    {"help",            no_argument,       NULL, OPT_HELP},
-    {"list",            no_argument, NULL, OPT_LIST},
+    {"summary",         no_argument,       NULL, OPT_SUMMARY},
-    {"list-paths",      no_argument, NULL, OPT_LIST_PATHS},
+    {"list-suites",     no_argument,       NULL, OPT_LIST_SUITES},
-    {"list-defines",    no_argument, NULL, OPT_LIST_DEFINES},
+    {"list-cases",      no_argument,       NULL, OPT_LIST_CASES},
-    {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES},
+    {"list-paths",      no_argument,       NULL, OPT_LIST_PATHS},
+    {"list-defines",    no_argument,       NULL, OPT_LIST_DEFINES},
+    {"list-geometries", no_argument,       NULL, OPT_LIST_GEOMETRIES},
+    {"define",          required_argument, NULL, OPT_DEFINE},
     {NULL, 0, NULL, 0},
 };
 
 const char *const help_text[] = {
     "Show this help message.",
+    "Show quick summary.",
+    "List test suites.",
     "List test cases.",
     "List the path for each test case.",
     "List the defines for each test permutation.",
     "List the disk geometries used for testing.",
+    "Override a test define.",
 };
 
 int main(int argc, char **argv) {
-    bool list = false;
+    void (*op)(
-    bool list_paths = false;
+            struct override *overrides,
-    bool list_defines = false;
+            size_t override_count) = run;
-    bool list_geometries = false;
+    struct override *overrides = NULL;
+    size_t override_count = 0;
+    size_t override_cap = 0;
 
     // parse options
     while (true) {
-        int index = 0;
+        int c = getopt_long(argc, argv, short_opts, long_opts, NULL);
-        int c = getopt_long(argc, argv, "hl", long_opts, &index);
         switch (c) {
             // generate help message
             case OPT_HELP: {
@@ -105,19 +397,56 @@ int main(int argc, char **argv) {
                 printf("\n");
                 exit(0);
             }
-            // list flags
+            // summary/list flags
-            case OPT_LIST:
+            case OPT_SUMMARY:
-                list = true;
+                op = summary;
+                break;
+            case OPT_LIST_SUITES:
+                op = list_suites;
+                break;
+            case OPT_LIST_CASES:
+                op = list_cases;
                 break;
             case OPT_LIST_PATHS:
-                list_paths = true;
+                op = list_paths;
                 break;
             case OPT_LIST_DEFINES:
-                list_defines = true;
+                op = list_defines;
                 break;
             case OPT_LIST_GEOMETRIES:
-                list_geometries = true;
+                op = list_geometries;
                 break;
+            // configuration
+            case OPT_DEFINE: {
+                // realloc if necessary
+                override_count += 1;
+                if (override_count > override_cap) {
+                    override_cap = (2*override_cap > 4) ? 2*override_cap : 4;
+                    overrides = realloc(overrides, override_cap
+                            * sizeof(struct override));
+                }
+
+                // parse into string key/uintmax_t value, cannibalizing the
+                // arg in the process
+                char *sep = strchr(optarg, '=');
+                char *parsed = NULL;
+                if (!sep) {
+                    goto invalid_define;
+                }
+                overrides[override_count-1].override
+                        = strtoumax(sep+1, &parsed, 0);
+                if (parsed == sep+1) {
+                    goto invalid_define;
+                }
+
+                overrides[override_count-1].name = optarg;
+                *sep = '\0';
+                break;
+
+invalid_define:
+                fprintf(stderr, "error: invalid define: %s\n", optarg);
+                exit(-1);
+            }
             // done parsing
             case -1:
                 goto getopt_done;
@@ -128,51 +457,16 @@ int main(int argc, char **argv) {
     }
 getopt_done:
 
-    // what do we need to do?
+    for (size_t i = 0; i < override_count; i++) {
-    if (list) {
+        printf("define: %s %ju\n", overrides[i].name, overrides[i].override);
-        printf("%-36s %-12s %-12s %7s %7s\n",
-                "id", "suite", "case", "type", "perms");
-        for (size_t i = 0; i < test_suite_count; i++) {
-            for (size_t j = 0; j < test_suites[i]->case_count; j++) {
-                printf("%-36s %-12s %-12s %7s %7d\n",
-                        test_suites[i]->cases[j]->id,
-                        test_suites[i]->name,
-                        test_suites[i]->cases[j]->name,
-                        "n", // TODO
-                        test_suites[i]->cases[j]->permutations);
-            }
-        }
-
-    } else if (list_paths) {
-        printf("%-36s %-36s\n", "id", "path");
-        for (size_t i = 0; i < test_suite_count; i++) {
-            for (size_t j = 0; j < test_suites[i]->case_count; j++) {
-                printf("%-36s %-36s\n",
-                        test_suites[i]->cases[j]->id,
-                        test_suites[i]->cases[j]->path);
-            }
-        }
-    } else if (list_defines) {
-        // TODO
-    } else if (list_geometries) {
-        printf("%-12s %7s %7s %7s %7s %7s\n",
-                "name", "read", "prog", "erase", "count", "size");
-        for (size_t i = 0; i < test_geometry_count; i++) {
-            printf("%-12s %7d %7d %7d %7d %7d\n",
-                    test_geometries[i].name,
-                    test_geometries[i].read_size,
-                    test_geometries[i].prog_size,
-                    test_geometries[i].erase_size,
-                    test_geometries[i].erase_count,
-                    test_geometries[i].erase_size
-                        * test_geometries[i].erase_count);
-        }
-    } else {
-        printf("remaining: ");
-        for (int i = optind; i < argc; i++) {
-            printf("%s ", argv[i]);
-        }
-        printf("\n");
     }
+
+    // do the thing
+    op(
+            overrides,
+            override_count);
+
+    // cleanup (need to be done for valgrind testing)
+    free(overrides);
 }
 

runners/test_runner.h

@@ -4,11 +4,24 @@
 #include "lfs.h"
 
 
+// generated test configurations
+enum test_type {
+    TEST_NORMAL    = 0x1,
+    TEST_REENTRANT = 0x2,
+    TEST_VALGRIND  = 0x4,
+};
+
 struct test_case {
     const char *id;
     const char *name;
     const char *path;
-    uint32_t permutations;
+    uint8_t types;
+    size_t permutations;
+
+    const uintmax_t *const *defines;
+    const bool *define_mask;
+
+    bool (*filter)(struct lfs_config *cfg, uint32_t perm);
     void (*run)(struct lfs_config *cfg, uint32_t perm);
 };
 
@@ -16,11 +29,33 @@ struct test_suite {
     const char *id;
     const char *name;
     const char *path;
+
+    const char *const *define_names;
+    size_t define_count;
+
     const struct test_case *const *cases;
     size_t case_count;
 };
 
+// TODO remove this indirection
 extern const struct test_suite *test_suites[];
 extern const size_t test_suite_count;
 
+
+// access generated test defines
+uintmax_t test_define(size_t define);
+
+// a few preconfigured defines that control how tests run
+#define READ_SIZE           test_define(0)
+#define PROG_SIZE           test_define(1)
+#define BLOCK_SIZE          test_define(2)
+#define BLOCK_COUNT         test_define(3)
+#define BLOCK_CYCLES        test_define(4)
+#define CACHE_SIZE          test_define(5)
+#define LOOKAHEAD_SIZE      test_define(6)
+#define ERASE_VALUE         test_define(7)
+#define ERASE_CYCLES        test_define(8)
+#define BADBLOCK_BEHAVIOR   test_define(9)
+
+
 #endif

scripts/test_.py

@@ -5,6 +5,7 @@
 
 import glob
 import itertools as it
+import math as m
 import os
 import re
 import shutil
@@ -17,15 +18,25 @@ SUITE_PROLOGUE = """
 #include "runners/test_runner.h"
 #include <stdio.h>
 """
-# TODO handle indention implicity?
-# TODO change cfg to be not by value? maybe not?
 CASE_PROLOGUE = """
-    lfs_t lfs;
+lfs_t lfs;
-    struct lfs_config cfg = *cfg_;
 """
 CASE_EPILOGUE = """
 """
 
+PRE_DEFINES = [
+    'READ_SIZE',
+    'PROG_SIZE',
+    'BLOCK_SIZE',
+    'BLOCK_COUNT',
+    'BLOCK_CYCLES',
+    'CACHE_SIZE',
+    'LOOKAHEAD_SIZE',
+    'ERASE_VALUE',
+    'ERASE_CYCLES',
+    'BADBLOCK_BEHAVIOR',
+]
+
 
 # TODO
 # def testpath(path):
@@ -58,7 +69,25 @@ class TestCase:
         self.code = config.pop('code')
         self.code_lineno = config.pop('code_lineno', None)
 
-        self.permutations = 1
+        # figure out defines and the number of resulting permutations
+        self.defines = {}
+        for k, v in config.pop('defines', {}).items():
+            try:
+                v = eval(v)
+            except:
+                v = v
+
+            if not isinstance(v, str):
+                try:
+                    v = list(v)
+                except:
+                    v = [v]
+            else:
+                v = [v]
+
+            self.defines[k] = v
+
+        self.permutations = m.prod(len(v) for v in self.defines.values())
 
         for k in config.keys():
             print('warning: in %s, found unused key %r' % (self.id(), k),
@@ -122,6 +151,10 @@ class TestSuite:
                     'suite': self.name,
                     **case}))
 
+            # combine pre-defines and per-case defines
+            self.defines = PRE_DEFINES + sorted(
+                set.union(*(set(case.defines) for case in self.cases)))
+
         for k in config.keys():
             print('warning: in %s, found unused key %r' % (self.id(), k),
                 file=sys.stderr)
@@ -129,6 +162,8 @@ class TestSuite:
     def id(self):
         return self.name
             
+            
+            
 
 
 def compile(**args):
@@ -164,13 +199,62 @@ def compile(**args):
                     f.write(suite.code)
                     f.write('\n')
 
-                # create test functions and case structs
+                for i, define in it.islice(
+                        enumerate(suite.defines),
+                        len(PRE_DEFINES), None):
+                    f.write('#define %-24s test_define(%d)\n' % (define, i))
+                f.write('\n')
+
                 for case in suite.cases:
-                    f.write('void __test__%s__%s('
+                    # create case defines
-                        '__attribute__((unused)) struct lfs_config *cfg_, '
+                    if case.defines:
+                        for perm, defines in enumerate(
+                                it.product(*(
+                                    [(k, v) for v in vs]
+                                    for k, vs in case.defines.items()))):
+                            f.write('const uintmax_t '
+                                '__test__%s__%s__%d__defines[] = {\n'
+                                % (suite.name, case.name, perm))
+                            for k, v in sorted(defines):
+                                f.write(4*' '+'[%d] = %s,\n'
+                                    % (suite.defines.index(k), v))
+                            f.write('};\n')
+                            f.write('\n')
+
+                        f.write('const uintmax_t *const '
+                            '__test__%s__%s__defines[] = {\n'
+                            % (suite.name, case.name))
+                        for perm in range(case.permutations):
+                            f.write(4*' '+'__test__%s__%s__%d__defines,\n'
+                                % (suite.name, case.name, perm))
+                        f.write('};\n')
+                        f.write('\n')
+
+                        f.write('const bool '
+                            '__test__%s__%s__define_mask[] = {\n'
+                            % (suite.name, case.name))
+                        for i, k in enumerate(suite.defines):
+                            f.write(4*' '+'%s,\n'
+                                % ('true' if k in case.defines else 'false'))
+                        f.write('};\n')
+                        f.write('\n')
+
+                    # create case filter function
+                    f.write('bool __test__%s__%s__filter('
+                        '__attribute__((unused)) struct lfs_config *cfg, '
                         '__attribute__((unused)) uint32_t perm) {\n'
                         % (suite.name, case.name))
-                    f.write(CASE_PROLOGUE)
+                    f.write(4*' '+'return true;\n')
+                    f.write('}\n')
+                    f.write('\n')
+
+                    # create case run function
+                    f.write('void __test__%s__%s__run('
+                        '__attribute__((unused)) struct lfs_config *cfg, '
+                        '__attribute__((unused)) uint32_t perm) {\n'
+                        % (suite.name, case.name))
+                    f.write(4*' '+'%s\n'
+                        % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' '))
                     f.write('\n')
                     f.write(4*' '+'// test case %s\n' % case.id())
                     if case.code_lineno is not None:
@@ -178,27 +262,49 @@ def compile(**args):
                             % (case.code_lineno, suite.path))
                     f.write(case.code)
                     f.write('\n')
-                    f.write(CASE_EPILOGUE)
+                    f.write(4*' '+'%s\n'
+                        % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' '))
                     f.write('}\n')
                     f.write('\n')
 
+                    # create case struct
                     f.write('const struct test_case __test__%s__%s__case = {\n'
                         % (suite.name, case.name))
                     f.write(4*' '+'.id = "%s",\n' % case.id())
                     f.write(4*' '+'.name = "%s",\n' % case.name)
                     f.write(4*' '+'.path = "%s",\n' % case.path)
+                    f.write(4*' '+'.types = TEST_NORMAL,\n')
                     f.write(4*' '+'.permutations = %d,\n' % case.permutations)
-                    f.write(4*' '+'.run = __test__%s__%s,\n'
+                    if case.defines:
+                        f.write(4*' '+'.defines = __test__%s__%s__defines,\n'
+                            % (suite.name, case.name))
+                        f.write(4*' '+'.define_mask = '
+                            '__test__%s__%s__define_mask,\n'
+                            % (suite.name, case.name))
+                    f.write(4*' '+'.filter = __test__%s__%s__filter,\n'
+                        % (suite.name, case.name))
+                    f.write(4*' '+'.run = __test__%s__%s__run,\n'
                         % (suite.name, case.name))
                     f.write('};\n')
                     f.write('\n')
 
+                # create suite define names
+                f.write('const char *const __test__%s__define_names[] = {\n'
+                    % suite.name)
+                for k in suite.defines:
+                    f.write(4*' '+'"%s",\n' % k)
+                f.write('};\n')
+                f.write('\n')
+
                 # create suite struct
                 f.write('const struct test_suite __test__%s__suite = {\n'
-                    % (suite.name))
+                    % suite.name)
                 f.write(4*' '+'.id = "%s",\n' % suite.id())
                 f.write(4*' '+'.name = "%s",\n' % suite.name)
                 f.write(4*' '+'.path = "%s",\n' % suite.path)
+                f.write(4*' '+'.define_names = __test__%s__define_names,\n'
+                    % suite.name)
+                f.write(4*' '+'.define_count = %d,\n' % len(suite.defines))
                 f.write(4*' '+'.cases = (const struct test_case *const []){\n')
                 for case in suite.cases:
                     f.write(8*' '+'&__test__%s__%s__case,\n'
@@ -216,11 +322,13 @@ def compile(**args):
         # write out a test source
         if 'output' in args:
             with openio(args['output'], 'w') as f:
-                f.write(SUITE_PROLOGUE)
-                f.write('\n')
                 f.write('#line 1 "%s"\n' % args['source'])
                 with open(args['source']) as sf:
                     shutil.copyfileobj(sf, f)
+                f.write('\n')
+
+                f.write(SUITE_PROLOGUE)
+                f.write('\n')
 
                 # add suite info to test_runner.c
                 if args['source'] == 'runners/test_runner.c':