MagnitudeOS/littlefs
1
0
Fork 0
forked from Imagelibrary/littlefs
Code Pull Requests Activity

Added permutations and ranges to test defines

Browse Source 
This is really more work for the bench runner. With this change defines
can be manipulated at a rather high level at runtime. Which should be
useful for generating benchmarks across various dimensions.

The define grammar in the test_runner is now a bit more powerful,
accepting:

1. A single value: -DN=42
2. A list of values, which get permuted: -DN=1,2,3
3. A range: -DN=range(10)
4. Some combo: -DN=1,2,range(3,0,-1)

This is more complex in the test .toml defines, which can also be C
expressions:

1. A single value: define=42
2. A single expression: define='42*42'
3. A list: define=[1,2,3]
4. A comma separated string: define='1,2,3'
5. A range: define='42*range(10)'
6. This mess: define=[1,2,'3,4,range(2)*range(2)+3']
This commit is contained in:
Christopher Haster
2022-09-11 03:10:08 -05:00
parent bfbe44e70d
commit 03c1a4ee2e
6 changed files with 324 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

351
runners/test_runner.c
View File

@@ -274,18 +274,17 @@ void test_define_geometry(const test_geometry_t *geometry) {
// override updates
typedef struct test_override {
const char *name;
intmax_t define;
const intmax_t *defines;
size_t permutations;
} test_override_t;
const test_override_t *test_overrides = NULL;
size_t test_override_count = 0;
void test_define_overrides(
const test_override_t *overrides,
size_t override_count) {
test_overrides = overrides;
test_override_count = override_count;
}
test_define_t *test_override_defines = NULL;
size_t test_override_define_count = 0;
size_t test_override_define_permutations = 1;
size_t test_override_define_capacity = 0;
// suite/perm updates
void test_define_suite(const struct test_suite *suite) {
@@ -308,35 +307,63 @@ void test_define_suite(const struct test_suite *suite) {
// map any overrides
if (test_override_count > 0) {
// first figure out the total size of override permutations
size_t count = 0;
size_t permutations = 1;
for (size_t i = 0; i < test_override_count; i++) {
for (size_t d = 0;
d < lfs_max(
suite->define_count,
TEST_IMPLICIT_DEFINE_COUNT);
d++) {
// define name match?
const char *name = test_define_name(d);
if (name && strcmp(name, test_overrides[i].name) == 0) {
count = d+1;
permutations *= test_overrides[i].permutations;
break;
}
}
}
test_override_define_count = count;
test_override_define_permutations = permutations;
// make sure our override arrays are big enough
if (suite->define_count
> test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count) {
if (count * permutations > test_override_define_capacity) {
// align to power of two to avoid any superlinear growth
size_t ncount = 1 << lfs_npw2(suite->define_count);
test_define_maps[TEST_DEFINE_MAP_OVERRIDE].defines = realloc(
(test_define_t*)test_define_maps[
TEST_DEFINE_MAP_OVERRIDE].defines,
ncount*sizeof(test_define_t));
test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count = ncount;
size_t ncapacity = 1 << lfs_npw2(count * permutations);
test_override_defines = realloc(
test_override_defines,
sizeof(test_define_t)*ncapacity);
test_override_define_capacity = ncapacity;
}
for (size_t i = 0;
i < test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count;
i++) {
((test_define_t*)test_define_maps[
TEST_DEFINE_MAP_OVERRIDE].defines)[i]
= (test_define_t){NULL};
// zero unoverridden defines
memset(test_override_defines, 0,
sizeof(test_define_t) * count * permutations);
const char *name = test_define_name(i);
if (!name) {
continue;
}
// compute permutations
size_t p = 1;
for (size_t i = 0; i < test_override_count; i++) {
for (size_t d = 0;
d < lfs_max(
suite->define_count,
TEST_IMPLICIT_DEFINE_COUNT);
d++) {
// define name match?
const char *name = test_define_name(d);
if (name && strcmp(name, test_overrides[i].name) == 0) {
// scatter the define permutations based on already
// seen permutations
for (size_t j = 0; j < permutations; j++) {
test_override_defines[j*count + d]
= (test_define_t)TEST_LIT(
test_overrides[i].defines[(j/p)
% test_overrides[i].permutations]);
}
for (size_t j = 0; j < test_override_count; j++) {
if (strcmp(name, test_overrides[j].name) == 0) {
((test_define_t*)test_define_maps[
TEST_DEFINE_MAP_OVERRIDE].defines)[i]
= (test_define_t)TEST_LIT(test_overrides[j].define);
// keep track of how many permutations we've seen so far
p *= test_overrides[i].permutations;
break;
}
}
@@ -350,13 +377,20 @@ void test_define_perm(
size_t perm) {
if (case_->defines) {
test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){
case_->defines[perm], suite->define_count};
case_->defines + perm*suite->define_count,
suite->define_count};
} else {
test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){
NULL, 0};
}
}
void test_define_override(size_t perm) {
test_define_maps[TEST_DEFINE_MAP_OVERRIDE] = (test_define_map_t){
test_override_defines + perm*test_override_define_count,
test_override_define_count};
}
void test_define_explicit(
const test_define_t *defines,
size_t define_count) {
@@ -368,7 +402,7 @@ void test_define_cleanup(void) {
// test define management can allocate a few things
free(test_define_cache);
free(test_define_cache_mask);
free((test_define_t*)test_define_maps[TEST_DEFINE_MAP_OVERRIDE].defines);
free(test_override_defines);
}
@@ -522,25 +556,30 @@ static void case_forperm(
// define permutation
test_define_perm(suite, case_, k);
for (size_t g = 0; g < test_geometry_count; g++) {
// define geometry
test_define_geometry(&test_geometries[g]);
test_define_flush();
for (size_t v = 0; v < test_override_define_permutations; v++) {
// define override permutation
test_define_override(v);
if (cycles) {
cb(data, suite, case_, &(test_powerloss_t){
.run=run_powerloss_cycles,
.cycles=cycles,
.cycle_count=cycle_count});
} else {
for (size_t p = 0; p < test_powerloss_count; p++) {
// skip non-reentrant tests when powerloss testing
if (test_powerlosses[p].short_name != '0'
&& !(case_->flags & TEST_REENTRANT)) {
continue;
for (size_t g = 0; g < test_geometry_count; g++) {
// define geometry
test_define_geometry(&test_geometries[g]);
test_define_flush();
if (cycles) {
cb(data, suite, case_, &(test_powerloss_t){
.run=run_powerloss_cycles,
.cycles=cycles,
.cycle_count=cycle_count});
} else {
for (size_t p = 0; p < test_powerloss_count; p++) {
// skip non-reentrant tests when powerloss testing
if (test_powerlosses[p].short_name != '0'
&& !(case_->flags & TEST_REENTRANT)) {
continue;
}
cb(data, suite, case_, &test_powerlosses[p]);
}
cb(data, suite, case_, &test_powerlosses[p]);
}
}
}
@@ -832,11 +871,31 @@ void perm_list_defines(
d < lfs_max(suite->define_count,
TEST_IMPLICIT_DEFINE_COUNT);
d++) {
if (!test_define_ispermutation(d)) {
continue;
if (d < TEST_IMPLICIT_DEFINE_COUNT
|| test_define_ispermutation(d)) {
list_defines_add(defines, d);
}
}
}
list_defines_add(defines, d);
void perm_list_permutation_defines(
void *data,
const struct test_suite *suite,
const struct test_case *case_,
const test_powerloss_t *powerloss) {
struct list_defines_defines *defines = data;
(void)suite;
(void)case_;
(void)powerloss;
// collect permutation_defines
for (size_t d = 0;
d < lfs_max(suite->define_count,
TEST_IMPLICIT_DEFINE_COUNT);
d++) {
if (test_define_ispermutation(d)) {
list_defines_add(defines, d);
}
}
}
@@ -845,22 +904,7 @@ extern const test_geometry_t builtin_geometries[];
static void list_defines(void) {
struct list_defines_defines defines = {NULL, 0, 0};
// yes we do need to define a suite, this does a bit of bookeeping
// such as setting up the define cache
test_define_suite(&(const struct test_suite){0});
// make sure to include builtin geometries here
for (size_t g = 0; builtin_geometries[g].long_name; g++) {
test_define_geometry(&builtin_geometries[g]);
test_define_flush();
// add implicit defines
for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) {
list_defines_add(&defines, d);
}
}
// add permutation defines
// add defines
for (size_t t = 0; t < test_id_count; t++) {
for (size_t i = 0; i < TEST_SUITE_COUNT; i++) {
if (test_ids[t].suite && strcmp(
@@ -932,7 +976,7 @@ static void list_permutation_defines(void) {
test_ids[t].define_count,
test_ids[t].cycles,
test_ids[t].cycle_count,
perm_list_defines,
perm_list_permutation_defines,
&defines);
}
}
@@ -1700,10 +1744,7 @@ const char *const help_text[] = {
int main(int argc, char **argv) {
void (*op)(void) = run;
test_override_t *overrides = NULL;
size_t override_count = 0;
size_t override_capacity = 0;
size_t test_override_capacity = 0;
size_t test_geometry_capacity = 0;
size_t test_powerloss_capacity = 0;
size_t test_id_capacity = 0;
@@ -1803,10 +1844,10 @@ int main(int argc, char **argv) {
case OPT_DEFINE: {
// allocate space
test_override_t *override = mappend(
(void**)&overrides,
(void**)&test_overrides,
sizeof(test_override_t),
&override_count,
&override_capacity);
&test_override_count,
&test_override_capacity);
// parse into string key/intmax_t value, cannibalizing the
// arg in the process
@@ -1815,13 +1856,112 @@ int main(int argc, char **argv) {
if (!sep) {
goto invalid_define;
}
override->define = strtoumax(sep+1, &parsed, 0);
if (parsed == sep+1) {
goto invalid_define;
}
override->name = optarg;
*sep = '\0';
override->name = optarg;
optarg = sep+1;
// parse comma-separated permutations
{
override->defines = NULL;
override->permutations = 0;
size_t override_capacity = 0;
while (true) {
optarg += strspn(optarg, " ");
if (strncmp(optarg, "range", strlen("range")) == 0) {
// range of values
optarg += strlen("range");
optarg += strspn(optarg, " ");
if (*optarg != '(') {
goto invalid_define;
}
optarg += 1;
intmax_t start = strtoumax(optarg, &parsed, 0);
intmax_t stop = -1;
intmax_t step = 1;
// allow empty string for start=0
if (parsed == optarg) {
start = 0;
}
optarg = parsed + strspn(parsed, " ");
if (*optarg != ',' && *optarg != ')') {
goto invalid_define;
}
if (*optarg == ',') {
optarg += 1;
stop = strtoumax(optarg, &parsed, 0);
// allow empty string for stop=end
if (parsed == optarg) {
stop = -1;
}
optarg = parsed + strspn(parsed, " ");
if (*optarg != ',' && *optarg != ')') {
goto invalid_define;
}
if (*optarg == ',') {
optarg += 1;
step = strtoumax(optarg, &parsed, 0);
// allow empty string for stop=1
if (parsed == optarg) {
step = 1;
}
optarg = parsed + strspn(parsed, " ");
if (*optarg != ')') {
goto invalid_define;
}
}
} else {
// single value = stop only
stop = start;
start = 0;
}
if (*optarg != ')') {
goto invalid_define;
}
optarg += 1;
// calculate the range of values
assert(step != 0);
for (intmax_t i = start;
(step < 0)
? i > stop
: (uintmax_t)i < (uintmax_t)stop;
i += step) {
*(intmax_t*)mappend(
(void**)&override->defines,
sizeof(intmax_t),
&override->permutations,
&override_capacity) = i;
}
} else if (*optarg != '\0') {
// single value
intmax_t define = strtoimax(optarg, &parsed, 0);
if (parsed == optarg) {
goto invalid_define;
}
*(intmax_t*)mappend(
(void**)&override->defines,
sizeof(intmax_t),
&override->permutations,
&override_capacity) = define;
} else {
break;
}
optarg = parsed + strspn(parsed, " ");
if (*optarg == ',') {
optarg += 1;
}
}
}
assert(override->permutations > 0);
break;
invalid_define:
@@ -2117,10 +2257,12 @@ powerloss_next:
}
case OPT_STEP: {
char *parsed = NULL;
size_t start = strtoumax(optarg, &parsed, 0);
test_step_start = strtoumax(optarg, &parsed, 0);
test_step_stop = -1;
test_step_step = 1;
// allow empty string for start=0
if (parsed != optarg) {
test_step_start = start;
if (parsed == optarg) {
test_step_start = 0;
}
optarg = parsed + strspn(parsed, " ");
@@ -2130,10 +2272,10 @@ powerloss_next:
if (*optarg == ',') {
optarg += 1;
size_t stop = strtoumax(optarg, &parsed, 0);
test_step_stop = strtoumax(optarg, &parsed, 0);
// allow empty string for stop=end
if (parsed != optarg) {
test_step_stop = stop;
if (parsed == optarg) {
test_step_stop = -1;
}
optarg = parsed + strspn(parsed, " ");
@@ -2143,10 +2285,10 @@ powerloss_next:
if (*optarg == ',') {
optarg += 1;
size_t step = strtoumax(optarg, &parsed, 0);
test_step_step = strtoumax(optarg, &parsed, 0);
// allow empty string for stop=1
if (parsed != optarg) {
test_step_step = step;
if (parsed == optarg) {
test_step_step = 1;
}
optarg = parsed + strspn(parsed, " ");
@@ -2154,6 +2296,10 @@ powerloss_next:
goto step_unknown;
}
}
} else {
// single value = stop only
test_step_stop = test_step_start;
test_step_start = 0;
}
break;
@@ -2317,30 +2463,31 @@ getopt_done: ;
};
}
// register overrides
test_define_overrides(overrides, override_count);
// do the thing
op();
// cleanup (need to be done for valgrind testing)
test_define_cleanup();
free(overrides);
if (test_overrides) {
for (size_t i = 0; i < test_override_count; i++) {
free((void*)test_overrides[i].defines);
}
free((void*)test_overrides);
}
if (test_geometry_capacity) {
free((test_geometry_t*)test_geometries);
free((void*)test_geometries);
}
if (test_powerloss_capacity) {
for (size_t i = 0; i < test_powerloss_count; i++) {
free((lfs_testbd_powercycles_t*)test_powerlosses[i].cycles);
free((void*)test_powerlosses[i].cycles);
}
free((test_powerloss_t*)test_powerlosses);
free((void*)test_powerlosses);
}
if (test_id_capacity) {
for (size_t i = 0; i < test_id_count; i++) {
free((test_geometry_t*)test_ids[i].defines);
free((lfs_testbd_powercycles_t*)test_ids[i].cycles);
free((void*)test_ids[i].defines);
free((void*)test_ids[i].cycles);
}
free((test_id_t*)test_ids);
free((void*)test_ids);
}
}

2
runners/test_runner.h
View File

@@ -43,7 +43,7 @@ struct test_case {
test_flags_t flags;
size_t permutations;
const test_define_t *const *defines;
const test_define_t *defines;
bool (*filter)(void);
void (*run)(struct lfs_config *cfg);

80
scripts/test.py
View File

@@ -62,6 +62,7 @@ class TestCase:
self.defines = set()
self.permutations = []
# defines can be a dict or a list or dicts
suite_defines = config.pop('suite_defines', {})
if not isinstance(suite_defines, list):
suite_defines = [suite_defines]
@@ -69,15 +70,63 @@ class TestCase:
if not isinstance(defines, list):
defines = [defines]
def csplit(v):
# split commas but only outside of parens
parens = 0
i_ = 0
for i in range(len(v)):
if v[i] == ',' and parens == 0:
yield v[i_:i]
i_ = i+1
elif v[i] in '([{':
parens += 1
elif v[i] in '}])':
parens -= 1
if v[i_:].strip():
yield v[i_:]
def parse_define(v):
# a define entry can be a list
if isinstance(v, list):
for v_ in v:
yield from parse_define(v_)
# or a string
elif isinstance(v, str):
# which can be comma-separated values, with optional
# range statements. This matches the runtime define parser in
# the runner itself.
for v_ in csplit(v):
m = re.search(r'\brange\b\s*\('
'(?P<start>[^,\s]*)'
'\s*(?:,\s*(?P<stop>[^,\s]*)'
'\s*(?:,\s*(?P<step>[^,\s]*)\s*)?)?\)',
v_)
if m:
start = (int(m.group('start'), 0)
if m.group('start') else 0)
stop = (int(m.group('stop'), 0)
if m.group('stop') else None)
step = (int(m.group('step'), 0)
if m.group('step') else 1)
if m.lastindex <= 1:
start, stop = 0, start
for x in range(start, stop, step):
yield from parse_define('%s(%d)%s' % (
v_[:m.start()], x, v_[m.end():]))
else:
yield v_
# or a literal value
else:
yield v
# build possible permutations
for suite_defines_ in suite_defines:
self.defines |= suite_defines_.keys()
for defines_ in defines:
self.defines |= defines_.keys()
self.permutations.extend(map(dict, it.product(*(
[(k, v) for v in (vs if isinstance(vs, list) else [vs])]
for k, vs in sorted(
(suite_defines_ | defines_).items())))))
self.permutations.extend(dict(perm) for perm in it.product(*(
[(k, v) for v in parse_define(vs)]
for k, vs in sorted((suite_defines_ | defines_).items()))))
for k in config.keys():
print('%swarning:%s in %s, found unused key %r' % (
@@ -254,13 +303,12 @@ def compile(test_paths, **args):
f.writeln(4*' '+'return %s;' % v)
f.writeln('}')
f.writeln()
f.writeln('const test_define_t *const '
'__test__%s__%s__defines[] = {'
% (suite.name, case.name))
f.writeln('const test_define_t '
'__test__%s__%s__defines[]['
'TEST_IMPLICIT_DEFINE_COUNT+%d] = {'
% (suite.name, case.name, len(suite.defines)))
for defines in case.permutations:
f.writeln(4*' '+'(const test_define_t['
'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % (
len(suite.defines)))
f.writeln(4*' '+'{')
for k, v in sorted(defines.items()):
f.writeln(8*' '+'[%-24s] = {%s, NULL},' % (
k+'_i', define_cbs[v]))
@@ -321,9 +369,10 @@ def compile(test_paths, **args):
write_case_functions(f, suite, case)
else:
if case.defines:
f.writeln('extern const test_define_t *const '
'__test__%s__%s__defines[];'
% (suite.name, case.name))
f.writeln('extern const test_define_t '
'__test__%s__%s__defines[]['
'TEST_IMPLICIT_DEFINE_COUNT+%d];'
% (suite.name, case.name, len(suite.defines)))
if suite.if_ is not None or case.if_ is not None:
f.writeln('extern bool __test__%s__%s__filter('
'void);'
@@ -368,7 +417,8 @@ def compile(test_paths, **args):
f.writeln(12*' '+'.permutations = %d,'
% len(case.permutations))
if case.defines:
f.writeln(12*' '+'.defines = __test__%s__%s__defines,'
f.writeln(12*' '+'.defines '
'= (const test_define_t*)__test__%s__%s__defines,'
% (suite.name, case.name))
if suite.if_ is not None or case.if_ is not None:
f.writeln(12*' '+'.filter = __test__%s__%s__filter,'
@@ -1088,7 +1138,7 @@ if __name__ == "__main__":
help="Output file.")
# runner + test_ids overlaps test_paths, so we need to do some munging here
args = parser.parse_args()
args = parser.parse_intermixed_args()
args.test_paths = [' '.join(args.runner or [])] + args.test_ids
args.runner = args.runner or [RUNNER_PATH]

12
tests/test_dirs.toml
View File

@@ -18,7 +18,7 @@ code = '''
'''
[cases.many_dir_creation]
defines.N = [3,6,9,12,21,33,57,66,72,93,99]
defines.N = 'range(3, 100, 3)'
if = 'N < BLOCK_COUNT/2'
code = '''
lfs_t lfs;
@@ -55,7 +55,7 @@ code = '''
'''
[cases.many_dir_removal]
defines.N = [3,6,9,12,21,33,57,66,72,93,99]
defines.N = 'range(3, 100, 11)'
if = 'N < BLOCK_COUNT/2'
code = '''
lfs_t lfs;
@@ -112,7 +112,7 @@ code = '''
'''
[cases.many_dir_rename]
defines.N = [3,6,9,12,21,33,57,66,72,93,99]
defines.N = 'range(3, 100, 11)'
if = 'N < BLOCK_COUNT/2'
code = '''
lfs_t lfs;
@@ -266,7 +266,7 @@ code = '''
'''
[cases.file_creation]
defines.N = [3,6,9,12,21,33,57,66,72,93,99]
defines.N = 'range(3, 100, 11)'
if = 'N < BLOCK_COUNT/2'
code = '''
lfs_t lfs;
@@ -306,7 +306,7 @@ code = '''
'''
[cases.file_removal]
defines.N = [3,6,9,12,21,33,57,66,72,93,99]
defines.N = 'range(3, 100, 11)'
if = 'N < BLOCK_COUNT/2'
code = '''
lfs_t lfs;
@@ -366,7 +366,7 @@ code = '''
'''
[cases.file_rename]
defines.N = [3,6,9,12,21,33,57,66,72,93,99]
defines.N = 'range(3, 100, 11)'
if = 'N < BLOCK_COUNT/2'
code = '''
lfs_t lfs;

4
tests/test_move.toml
View File

@@ -1586,7 +1586,7 @@ code = '''
# move fix in relocation
[cases.move_fix_relocation]
in = "lfs.c"
defines.RELOCATIONS = [0x0, 0x1, 0x2, 0x3]
defines.RELOCATIONS = 'range(4)'
defines.ERASE_CYCLES = 0xffffffff
code = '''
lfs_t lfs;
@@ -1731,7 +1731,7 @@ code = '''
# move fix in relocation with predecessor
[cases.move_fix_relocation_predecessor]
in = "lfs.c"
defines.RELOCATIONS = [0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7]
defines.RELOCATIONS = 'range(8)'
defines.ERASE_CYCLES = 0xffffffff
code = '''
lfs_t lfs;

2
tests/test_truncate.toml
View File

@@ -285,7 +285,7 @@ code = '''
# more aggressive general truncation tests
[cases.aggressive_truncate]
defines.CONFIG = [0,1,2,3,4,5]
defines.CONFIG = 'range(6)'
defines.SMALLSIZE = 32
defines.MEDIUMSIZE = 2048
defines.LARGESIZE = 8192