From c87361508b0af927ee0a1c60bc9927b7e0ffcd9a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 20 Jul 2025 09:40:46 -0500 Subject: [PATCH] scripts: test.py/bench.py: Added --no-internal to skip internal tests The --no-internal flag avoids building any internal tests/benches (tests/benches with in="lfs3.c"), which can be useful for quickly testing high-level things while refactoring. Refactors tend to break all the internal tests, and it can be a real pain to update everything. Note that --no-internal can be injected into the build with TESTCFLAGS: TESTCFLAGS=--no-internal make test-runner -j \ && ./scripts/test.py -j -b For a curious data point, here's the current number of internal/non-internal tests: suites cases perms total: 24 808 633968/776298 internal: 22 (91.7%) 532 (65.8%) 220316/310247 (34.8%) non-internal: 2 ( 8.3%) 276 (34.2%) 413652/466051 (65.2%) It's interesting to note that while internal tests have more test cases, the non-internal tests generate a larger number of test permutations. This is probably because internal tests tend to target specific corner cases/known failure points, and don't invite much variants. --- While --no-internal may be useful for high-level testing during a refactor, I'm not sure it's a good idea to rely on it for _debugging_ a refactor. The whole point of internal testing is to catch low-level bugs early, with as little unnecessary state as possible. Skipping these to debug integration tests is a bit counterproductive! --- scripts/bench.py | 26 ++++++++++++++++++-------- scripts/test.py | 26 ++++++++++++++++++-------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/scripts/bench.py b/scripts/bench.py index b5cf7526..88e5d5f2 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -247,17 +247,23 @@ class BenchSuite: defines = config.pop('defines', {}) self.cases = [] - for name, case in cases.items(): - self.cases.append(BenchCase( + for name, config_ in cases.items(): + case = BenchCase( config={ 'name': name, - 'path': path + (':%d' % case['lineno'] - if 'lineno' in case else ''), + 'path': path + (':%d' % config_['lineno'] + if 'lineno' in config_ else ''), 'suite': self.name, 'suite_defines': defines, 'suite_in': self.in_, - **case}, - args=args)) + **config_}, + args=args) + + # skipping internal tests? + if args.get('no_internal') and case.in_ is not None: + continue + + self.cases.append(case) # sort for consistency self.cases.sort() @@ -1761,6 +1767,9 @@ if __name__ == "__main__": '-c', '--compile', action='store_true', help="Compile a bench suite or source file.") + comp_parser.add_argument( + '-o', '--output', + help="Output file.") comp_parser.add_argument( '-s', '--source', help="Source file to compile, possibly injecting internal benches.") @@ -1769,8 +1778,9 @@ if __name__ == "__main__": help="Inject these header files into every compiled bench file. " "Defaults to %r." % HEADER_PATHS) comp_parser.add_argument( - '-o', '--output', - help="Output file.") + '--no-internal', + action='store_true', + help="Don't build internal tests.") # do the thing args = parser.parse_intermixed_args() diff --git a/scripts/test.py b/scripts/test.py index 41fc7080..c69dbf03 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -255,19 +255,25 @@ class TestSuite: reentrant = config.pop('reentrant', False) self.cases = [] - for name, case in cases.items(): - self.cases.append(TestCase( + for name, config_ in cases.items(): + case = TestCase( config={ 'name': name, - 'path': path + (':%d' % case['lineno'] - if 'lineno' in case else ''), + 'path': path + (':%d' % config_['lineno'] + if 'lineno' in config_ else ''), 'suite': self.name, 'suite_defines': defines, 'suite_in': self.in_, 'suite_reentrant': reentrant, 'suite_fuzz': self.fuzz_, - **case}, - args=args)) + **config_}, + args=args) + + # skipping internal tests? + if args.get('no_internal') and case.in_ is not None: + continue + + self.cases.append(case) # sort for consistency self.cases.sort() @@ -1796,6 +1802,9 @@ if __name__ == "__main__": '-c', '--compile', action='store_true', help="Compile a test suite or source file.") + comp_parser.add_argument( + '-o', '--output', + help="Output file.") comp_parser.add_argument( '-s', '--source', help="Source file to compile, possibly injecting internal tests.") @@ -1804,8 +1813,9 @@ if __name__ == "__main__": help="Inject these header files into every compiled test file. " "Defaults to %r." % HEADER_PATHS) comp_parser.add_argument( - '-o', '--output', - help="Output file.") + '--no-internal', + action='store_true', + help="Don't build internal tests.") # do the thing args = parser.parse_intermixed_args()