From 007ac97bec727f4da21498b60769ddaa5ce4624d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 6 Nov 2024 15:31:17 -0600 Subject: [PATCH] scripts: Adopted double-indent on multiline expressions This matches the style used in C, which is good for consistency: a_really_long_function_name( double_indent_after_first_newline( single_indent_nested_newlines)) We were already doing this for multiline control-flow statements, simply because I'm not sure how else you could indent this without making things really confusing: if a_really_long_function_name( double_indent_after_first_newline( single_indent_nested_newlines)): do_the_thing() This was the only real difference style-wise between the Python code and C code, so now both should be following roughly the same style (80 cols, double-indent multiline exprs, prefix multiline binary ops, etc). --- scripts/amor.py | 162 +++--- scripts/avg.py | 221 ++++---- scripts/bench.py | 1009 +++++++++++++++++------------------ scripts/changeprefix.py | 96 ++-- scripts/code.py | 409 ++++++++------- scripts/cov.py | 486 ++++++++--------- scripts/crc32c.py | 27 +- scripts/data.py | 409 ++++++++------- scripts/dbgblock.py | 100 ++-- scripts/dbgbmap.py | 449 ++++++++-------- scripts/dbgbtree.py | 397 +++++++------- scripts/dbgcat.py | 93 ++-- scripts/dbgerr.py | 27 +- scripts/dbglfs.py | 942 +++++++++++++++++---------------- scripts/dbgmtree.py | 606 ++++++++++----------- scripts/dbgrbyd.py | 487 ++++++++--------- scripts/dbgtag.py | 213 ++++---- scripts/parity.py | 39 +- scripts/perf.py | 756 +++++++++++++------------- scripts/perfbd.py | 701 +++++++++++++------------ scripts/plot.py | 778 +++++++++++++-------------- scripts/plotmpl.py | 641 ++++++++++++----------- scripts/prettyasserts.py | 119 ++--- scripts/readblock.py | 12 +- scripts/readmdir.py | 85 +-- scripts/readtree.py | 64 ++- scripts/stack.py | 437 ++++++++-------- scripts/structs.py | 389 +++++++------- scripts/summary.py | 438 ++++++++-------- scripts/tailpipe.py | 53 +- scripts/teepipe.py | 28 +- scripts/test.py | 1077 +++++++++++++++++++------------------- scripts/tracebd.py | 540 +++++++++---------- scripts/watch.py | 109 ++-- 34 files changed, 6290 insertions(+), 6109 deletions(-) diff --git a/scripts/amor.py b/scripts/amor.py index 083e339e..0088c14c 100755 --- a/scripts/amor.py +++ b/scripts/amor.py @@ -53,8 +53,8 @@ def collect(csv_paths, renames=[], defines=[]): with openio(path) as f: reader = csv.DictReader(f, restval='') fields.extend( - k for k in reader.fieldnames - if k not in fields) + k for k in reader.fieldnames + if k not in fields) for r in reader: # apply any renames if renames: @@ -90,8 +90,8 @@ def main(csv_paths, output, *, # separate out renames renames = list(it.chain.from_iterable( - ((k, v) for v in vs) - for k, vs in it.chain(by or [], fields or []))) + ((k, v) for v in vs) + for k, vs in it.chain(by or [], fields or []))) if by is not None: by = [k for k, _ in by] if fields is not None: @@ -99,7 +99,7 @@ def main(csv_paths, output, *, if by is None and fields is None: print("error: needs --by or --fields to figure out fields", - file=sys.stderr) + file=sys.stderr) sys.exit(-1) # collect results from csv files @@ -108,24 +108,22 @@ def main(csv_paths, output, *, # if by not specified, guess it's anything not in # iter/size/fields/renames/defines if by is None: - by = [ - k for k in fields_ - if k != iter - and k != size - and k not in (fields or []) - and not any(k == old_k for _, old_k in renames) - and not any(k == k_ for k_, _ in defines)] + by = [k for k in fields_ + if k != iter + and k != size + and k not in (fields or []) + and not any(k == old_k for _, old_k in renames) + and not any(k == k_ for k_, _ in defines)] # if fields not specified, guess it's anything not in # by/iter/size/renames/defines if fields is None: - fields = [ - k for k in fields_ - if k not in (by or []) - and k != iter - and k != size - and not any(k == old_k for _, old_k in renames) - and not any(k == k_ for k_, _ in defines)] + fields = [k for k in fields_ + if k not in (by or []) + and k != iter + and k != size + and not any(k == old_k for _, old_k in renames) + and not any(k == k_ for k_, _ in defines)] # add meas to by if it isn't already present if meas is not None and meas not in by: @@ -161,23 +159,23 @@ def main(csv_paths, output, *, # find amortized results if amor: amors.append(r - | {f: sums[f] / size_ for f in fields} - | ({} if meas is None - else {meas: r[meas]+'+amor'} if meas in r - else {meas: 'amor'})) + | {f: sums[f] / size_ for f in fields} + | ({} if meas is None + else {meas: r[meas]+'+amor'} if meas in r + else {meas: 'amor'})) # also find per-byte results if per: amors.append(r - | {f: r.get(f, 0) / size_ for f in fields} - | ({} if meas is None - else {meas: r[meas]+'+per'} if meas in r - else {meas: 'per'})) + | {f: r.get(f, 0) / size_ for f in fields} + | ({} if meas is None + else {meas: r[meas]+'+per'} if meas in r + else {meas: 'per'})) # write results to CSV with openio(output, 'w') as f: writer = csv.DictWriter(f, - by + [iter] + ([size] if size is not None else []) + fields) + by + [iter] + ([size] if size is not None else []) + fields) writer.writeheader() for r in amors: writer.writerow(r) @@ -187,68 +185,70 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Amortize benchmark measurements.", - allow_abbrev=False) + description="Amortize benchmark measurements.", + allow_abbrev=False) parser.add_argument( - 'csv_paths', - nargs='*', - help="Input *.csv files.") + 'csv_paths', + nargs='*', + help="Input *.csv files.") parser.add_argument( - '-o', '--output', - required=True, - help="*.csv file to write amortized measurements to.") + '-o', '--output', + required=True, + help="*.csv file to write amortized measurements to.") parser.add_argument( - '--amor', - action='store_true', - help="Compute amortized results.") + '--amor', + action='store_true', + help="Compute amortized results.") parser.add_argument( - '--per', - action='store_true', - help="Compute per-byte results.") + '--per', + action='store_true', + help="Compute per-byte results.") parser.add_argument( - '-b', '--by', - action='append', - type=lambda x: ( - lambda k, vs=None: ( - k.strip(), - tuple(v.strip() for v in vs.split(',')) - if vs is not None else ()) - )(*x.split('=', 1)), - help="Group by this field. Can rename fields with new_name=old_name.") + '-b', '--by', + action='append', + type=lambda x: ( + lambda k, vs=None: ( + k.strip(), + tuple(v.strip() for v in vs.split(',')) + if vs is not None else ()) + )(*x.split('=', 1)), + help="Group by this field. Can rename fields with " + "new_name=old_name.") parser.add_argument( - '-m', '--meas', - help="Optional name of measurement name field. If provided, the name " - "will be modified with +amor or +per.") + '-m', '--meas', + help="Optional name of measurement name field. If provided, the " + "name will be modified with +amor or +per.") parser.add_argument( - '-i', '--iter', - required=True, - help="Name of iteration field.") + '-i', '--iter', + required=True, + help="Name of iteration field.") parser.add_argument( - '-n', '--size', - help="Optional name of size field.") + '-n', '--size', + help="Optional name of size field.") parser.add_argument( - '-f', '--field', - dest='fields', - action='append', - type=lambda x: ( - lambda k, vs=None: ( - k.strip(), - tuple(v.strip() for v in vs.split(',')) - if vs is not None else ()) - )(*x.split('=', 1)), - help="Field to amortize. Can rename fields with new_name=old_name.") + '-f', '--field', + dest='fields', + action='append', + type=lambda x: ( + lambda k, vs=None: ( + k.strip(), + tuple(v.strip() for v in vs.split(',')) + if vs is not None else ()) + )(*x.split('=', 1)), + help="Field to amortize. Can rename fields with " + "new_name=old_name.") parser.add_argument( - '-D', '--define', - dest='defines', - action='append', - type=lambda x: ( - lambda k, vs: ( - k.strip(), - {v.strip() for v in vs.split(',')}) - )(*x.split('=', 1)), - help="Only include results where this field is this value. May include " - "comma-separated options.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: ( + lambda k, vs: ( + k.strip(), + {v.strip() for v in vs.split(',')}) + )(*x.split('=', 1)), + help="Only include results where this field is this value. May " + "include comma-separated options.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/avg.py b/scripts/avg.py index 92cbe0d9..f6e01c1b 100755 --- a/scripts/avg.py +++ b/scripts/avg.py @@ -53,8 +53,8 @@ def collect(csv_paths, renames=[], defines=[]): with openio(path) as f: reader = csv.DictReader(f, restval='') fields.extend( - k for k in reader.fieldnames - if k not in fields) + k for k in reader.fieldnames + if k not in fields) for r in reader: # apply any renames if renames: @@ -108,8 +108,8 @@ def main(csv_paths, output, *, # separate out renames renames = list(it.chain.from_iterable( - ((k, v) for v in vs) - for k, vs in it.chain(by or [], seeds or [], fields or []))) + ((k, v) for v in vs) + for k, vs in it.chain(by or [], seeds or [], fields or []))) if by is not None: by = [k for k, _ in by] if seeds is not None: @@ -119,7 +119,7 @@ def main(csv_paths, output, *, if by is None and fields is None: print("error: needs --by or --fields to figure out fields", - file=sys.stderr) + file=sys.stderr) sys.exit(-1) # collect results from csv files @@ -128,22 +128,20 @@ def main(csv_paths, output, *, # if by not specified, guess it's anything not in # seeds/fields/renames/defines if by is None: - by = [ - k for k in fields_ - if k not in (seeds or []) - and k not in (fields or []) - and not any(k == old_k for _, old_k in renames) - and not any(k == k_ for k_, _ in defines)] + by = [k for k in fields_ + if k not in (seeds or []) + and k not in (fields or []) + and not any(k == old_k for _, old_k in renames) + and not any(k == k_ for k_, _ in defines)] # if fields not specified, guess it's anything not in # by/seeds/renames/defines if fields is None: - fields = [ - k for k in fields_ - if k not in (by or []) - and k not in (seeds or []) - and not any(k == old_k for _, old_k in renames) - and not any(k == k_ for k_, _ in defines)] + fields = [k for k in fields_ + if k not in (by or []) + and k not in (seeds or []) + and not any(k == old_k for _, old_k in renames) + and not any(k == k_ for k_, _ in defines)] # add meas to by if it isn't already present if meas is not None and meas not in by: @@ -174,12 +172,11 @@ def main(csv_paths, output, *, meas__ = r[meas] def append(meas_, f_): - avgs.append( - {k: v for k, v in zip(by, key)} - | {f: f_(vs_) for f, vs_ in vs.items()} - | ({} if meas is None - else {meas: meas_} if meas__ is None - else {meas: meas__+'+'+meas_})) + avgs.append({k: v for k, v in zip(by, key)} + | {f: f_(vs_) for f, vs_ in vs.items()} + | ({} if meas is None + else {meas: meas_} if meas__ is None + else {meas: meas__+'+'+meas_})) if sum_: append('sum', lambda vs: sum(vs)) if prod: append('prod', lambda vs: mt.prod(vs)) @@ -189,16 +186,16 @@ def main(csv_paths, output, *, if bnd: append('bnd', lambda vs: max(vs, default=0)) if avg: append('avg', lambda vs: sum(vs) / max(len(vs), 1)) if stddev: append('stddev', lambda vs: ( - lambda avg: mt.sqrt( - sum((v - avg)**2 for v in vs) / max(len(vs), 1)) - )(sum(vs) / max(len(vs), 1))) + lambda avg: mt.sqrt( + sum((v - avg)**2 for v in vs) / max(len(vs), 1)) + )(sum(vs) / max(len(vs), 1))) if gmean: append('gmean', lambda vs: - mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1))) + mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1))) if gstddev: append('gstddev', lambda vs: ( - lambda gmean: mt.exp(mt.sqrt( - sum(mt.log(v/gmean)**2 for v in vs) / max(len(vs), 1))) - if gmean else mt.inf - )(mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1)))) + lambda gmean: mt.exp(mt.sqrt( + sum(mt.log(v/gmean)**2 for v in vs) / max(len(vs), 1))) + if gmean else mt.inf + )(mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1)))) # write results to CSVS with openio(output, 'w') as f: @@ -212,101 +209,103 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Compute averages/etc of benchmark measurements.", - allow_abbrev=False) + description="Compute averages/etc of benchmark measurements.", + allow_abbrev=False) parser.add_argument( - 'csv_paths', - nargs='*', - help="Input *.csv files.") + 'csv_paths', + nargs='*', + help="Input *.csv files.") parser.add_argument( - '-o', '--output', - required=True, - help="*.csv file to write amortized measurements to.") + '-o', '--output', + required=True, + help="*.csv file to write amortized measurements to.") parser.add_argument( - '--sum', - action='store_true', - help="Compute the sum.") + '--sum', + action='store_true', + help="Compute the sum.") parser.add_argument( - '--prod', - action='store_true', - help="Compute the product.") + '--prod', + action='store_true', + help="Compute the product.") parser.add_argument( - '--min', - action='store_true', - help="Compute the min.") + '--min', + action='store_true', + help="Compute the min.") parser.add_argument( - '--max', - action='store_true', - help="Compute the max.") + '--max', + action='store_true', + help="Compute the max.") parser.add_argument( - '--bnd', - action='store_true', - help="Compute the bounds (min+max concatenated).") + '--bnd', + action='store_true', + help="Compute the bounds (min+max concatenated).") parser.add_argument( - '--avg', '--mean', - action='store_true', - help="Compute the average (the default).") + '--avg', '--mean', + action='store_true', + help="Compute the average (the default).") parser.add_argument( - '--stddev', - action='store_true', - help="Compute the standard deviation.") + '--stddev', + action='store_true', + help="Compute the standard deviation.") parser.add_argument( - '--gmean', - action='store_true', - help="Compute the geometric mean.") + '--gmean', + action='store_true', + help="Compute the geometric mean.") parser.add_argument( - '--gstddev', - action='store_true', - help="Compute the geometric standard deviation.") + '--gstddev', + action='store_true', + help="Compute the geometric standard deviation.") parser.add_argument( - '-b', '--by', - action='append', - type=lambda x: ( - lambda k, vs=None: ( - k.strip(), - tuple(v.strip() for v in vs.split(',')) - if vs is not None else ()) - )(*x.split('=', 1)), - help="Group by this field. Can rename fields with new_name=old_name.") + '-b', '--by', + action='append', + type=lambda x: ( + lambda k, vs=None: ( + k.strip(), + tuple(v.strip() for v in vs.split(',')) + if vs is not None else ()) + )(*x.split('=', 1)), + help="Group by this field. Can rename fields with " + "new_name=old_name.") parser.add_argument( - '-m', '--meas', - help="Optional name of measurement name field. If provided, the name " - "will be modified with +amor or +per.") + '-m', '--meas', + help="Optional name of measurement name field. If provided, the " + "name will be modified with +amor or +per.") parser.add_argument( - '-s', '--seed', - dest='seeds', - action='append', - type=lambda x: ( - lambda k, vs=None: ( - k.strip(), - tuple(v.strip() for v in vs.split(',')) - if vs is not None else ()) - )(*x.split('=', 1)), - help="Field to ignore when averaging. Can rename fields with " - "new_name=old_name.") + '-s', '--seed', + dest='seeds', + action='append', + type=lambda x: ( + lambda k, vs=None: ( + k.strip(), + tuple(v.strip() for v in vs.split(',')) + if vs is not None else ()) + )(*x.split('=', 1)), + help="Field to ignore when averaging. Can rename fields with " + "new_name=old_name.") parser.add_argument( - '-f', '--field', - dest='fields', - action='append', - type=lambda x: ( - lambda k, vs=None: ( - k.strip(), - tuple(v.strip() for v in vs.split(',')) - if vs is not None else ()) - )(*x.split('=', 1)), - help="Field to amortize. Can rename fields with new_name=old_name.") + '-f', '--field', + dest='fields', + action='append', + type=lambda x: ( + lambda k, vs=None: ( + k.strip(), + tuple(v.strip() for v in vs.split(',')) + if vs is not None else ()) + )(*x.split('=', 1)), + help="Field to amortize. Can rename fields with " + "new_name=old_name.") parser.add_argument( - '-D', '--define', - dest='defines', - action='append', - type=lambda x: ( - lambda k, vs: ( - k.strip(), - {v.strip() for v in vs.split(',')}) - )(*x.split('=', 1)), - help="Only include results where this field is this value. May include " - "comma-separated options.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: ( + lambda k, vs: ( + k.strip(), + {v.strip() for v in vs.split(',')}) + )(*x.split('=', 1)), + help="Only include results where this field is this value. May " + "include comma-separated options.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/bench.py b/scripts/bench.py index a19adf5a..fb829739 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -60,7 +60,7 @@ class BenchCase: self.code = config.pop('code') self.code_lineno = config.pop('code_lineno', None) self.in_ = config.pop('in', - config.pop('suite_in', None)) + config.pop('suite_in', None)) self.internal = bool(self.in_) @@ -103,17 +103,17 @@ class BenchCase: vs = [] for v_ in csplit(v): m = re.search(r'\brange\b\s*\(' - '(?P[^,\s]*)' - '\s*(?:,\s*(?P[^,\s]*)' - '\s*(?:,\s*(?P[^,\s]*)\s*)?)?\)', - v_) + '(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)\s*)?)?\)', + v_) if m: start = (int(m.group('start'), 0) - if m.group('start') else 0) + if m.group('start') else 0) stop = (int(m.group('stop'), 0) - if m.group('stop') else None) + if m.group('stop') else None) step = (int(m.group('step'), 0) - if m.group('step') else 1) + if m.group('step') else 1) if m.lastindex <= 1: start, stop = 0, start vs.append(range(start, stop, step)) @@ -132,16 +132,16 @@ class BenchCase: for defines_ in defines: self.defines |= defines_.keys() self.permutations.append({ - k: parse_define(v) - for k, v in (suite_defines_ | defines_).items()}) + k: parse_define(v) + for k, v in (suite_defines_ | defines_).items()}) for k in config.keys(): print('%swarning:%s in %s, found unused key %r' % ( - '\x1b[01;33m' if args['color'] else '', - '\x1b[m' if args['color'] else '', - self.name, - k), - file=sys.stderr) + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + self.name, + k), + file=sys.stderr) def __repr__(self): return '' % self.name @@ -149,12 +149,12 @@ class BenchCase: def __lt__(self, other): # sort by suite, lineno, and name return ((self.suite, self.lineno, self.name) - < (other.suite, other.lineno, other.name)) + < (other.suite, other.lineno, other.name)) def isin(self, path): return (self.in_ is not None - and os.path.normpath(self.in_) - == os.path.normpath(path)) + and os.path.normpath(self.in_) + == os.path.normpath(path)) class BenchSuite: @@ -176,9 +176,9 @@ class BenchSuite: code_linenos = [] for i, line in enumerate(f): match = re.match( - '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' - '|' '(?Pcode\s*=)', - line) + '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + '|' '(?Pcode\s*=)', + line) if match and match.group('case'): case_linenos.append((i+1, match.group('name'))) elif match and match.group('code'): @@ -192,8 +192,9 @@ class BenchSuite: case_linenos, case_linenos[1:], fillvalue=(float('inf'), None)): code_lineno = min( - (l for l in code_linenos if l >= lineno and l < nlineno), - default=None) + (l for l in code_linenos + if l >= lineno and l < nlineno), + default=None) cases[name]['lineno'] = lineno cases[name]['code_lineno'] = code_lineno @@ -207,9 +208,9 @@ class BenchSuite: self.code = config.pop('code', None) self.code_lineno = min( - (l for l in code_linenos - if not case_linenos or l < case_linenos[0][0]), - default=None) + (l for l in code_linenos + if not case_linenos or l < case_linenos[0][0]), + default=None) self.in_ = config.pop('in', None) self.after = config.pop('after', []) @@ -221,33 +222,34 @@ class BenchSuite: self.cases = [] for name, case in cases.items(): - self.cases.append(BenchCase(config={ - 'name': name, - 'path': path + (':%d' % case['lineno'] - if 'lineno' in case else ''), - 'suite': self.name, - 'suite_defines': defines, - 'suite_in': self.in_, - **case}, - args=args)) + self.cases.append(BenchCase( + config={ + 'name': name, + 'path': path + (':%d' % case['lineno'] + if 'lineno' in case else ''), + 'suite': self.name, + 'suite_defines': defines, + 'suite_in': self.in_, + **case}, + args=args)) # sort for consistency self.cases.sort() # combine per-case defines self.defines = set.union(set(), *( - set(case.defines) for case in self.cases)) + set(case.defines) for case in self.cases)) # combine other per-case things self.internal = any(case.internal for case in self.cases) for k in config.keys(): print('%swarning:%s in %s, found unused key %r' % ( - '\x1b[01;33m' if args['color'] else '', - '\x1b[m' if args['color'] else '', - self.name, - k), - file=sys.stderr) + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + self.name, + k), + file=sys.stderr) def __repr__(self): return '' % self.name @@ -260,8 +262,8 @@ class BenchSuite: def isin(self, path): return (self.in_ is not None - and os.path.normpath(self.in_) - == os.path.normpath(path)) + and os.path.normpath(self.in_) + == os.path.normpath(path)) def compile(bench_paths, **args): @@ -284,10 +286,10 @@ def compile(bench_paths, **args): if len(pending_) == len(pending): print('%serror:%s cycle detected in suite ordering: {%s}' % ( - '\x1b[01;31m' if args['color'] else '', - '\x1b[m' if args['color'] else '', - ', '.join(suite.name for suite in pending.values())), - file=sys.stderr) + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + ', '.join(suite.name for suite in pending.values())), + file=sys.stderr) sys.exit(-1) pending = pending_ @@ -298,36 +300,36 @@ def compile(bench_paths, **args): for suite in suites: if suite.name in seen: print('%swarning:%s conflicting suite %r, %s and %s' % ( - '\x1b[01;33m' if args['color'] else '', - '\x1b[m' if args['color'] else '', - suite.name, - suite.path, - seen[suite.name].path), - file=sys.stderr) + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + suite.name, + suite.path, + seen[suite.name].path), + file=sys.stderr) seen[suite.name] = suite for case in suite.cases: # only allow conflicts if a case and its suite share a name if case.name in seen and not ( isinstance(seen[case.name], BenchSuite) - and seen[case.name].cases == [case]): + and seen[case.name].cases == [case]): print('%swarning:%s conflicting case %r, %s and %s' % ( - '\x1b[01;33m' if args['color'] else '', - '\x1b[m' if args['color'] else '', - case.name, - case.path, - seen[case.name].path), - file=sys.stderr) + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + case.name, + case.path, + seen[case.name].path), + file=sys.stderr) seen[case.name] = case # we can only compile one bench suite at a time if not args.get('source'): if len(suites) > 1: print('%serror:%s compiling more than one bench suite? (%r)' % ( - '\x1b[01;31m' if args['color'] else '', - '\x1b[m' if args['color'] else '', - bench_paths), - file=sys.stderr) + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + bench_paths), + file=sys.stderr) sys.exit(-1) suite = suites[0] @@ -373,24 +375,22 @@ def compile(bench_paths, **args): for i, permutation in enumerate(case.permutations): for k, vs in sorted(permutation.items()): f.writeln('intmax_t __bench__%s__%s__%d(' - '__attribute__((unused)) void *data, ' - 'size_t i) {' - % (case.name, k, i)) + '__attribute__((unused)) void *data, ' + 'size_t i) {' % ( + case.name, k, i)) j = 0 for v in vs: # generate range if isinstance(v, range): - f.writeln( - 4*' '+'if (i < %d) ' - 'return (i-%d)*%d + %d;' - % (j+len(v), j, v.step, v.start)) + f.writeln(4*' '+'if (i < %d) ' + 'return (i-%d)*%d + %d;' % ( + j+len(v), j, v.step, v.start)) j += len(v) # translate index to define else: - f.writeln( - 4*' '+'if (i == %d) ' - 'return %s;' - % (j, v)) + f.writeln(4*' '+'if (i == %d) ' + 'return %s;' % ( + j, v)) j += 1; f.writeln(4*' '+'__builtin_unreachable();') @@ -399,29 +399,30 @@ def compile(bench_paths, **args): # create case if function if suite.if_ or case.if_: - f.writeln('bool __bench__%s__if(void) {' - % (case.name)) + f.writeln('bool __bench__%s__if(void) {' % ( + case.name)) for if_ in it.chain(suite.if_, case.if_): f.writeln(4*' '+'if (!(%s)) return false;' % ( - 'true' if if_ is True - else 'false' if if_ is False - else if_)) + 'true' if if_ is True + else 'false' if if_ is False + else if_)) f.writeln(4*' '+'return true;') f.writeln('}') f.writeln() # create case run function f.writeln('void __bench__%s__run(' - '__attribute__((unused)) struct lfs_config *CFG) {' - % (case.name)) + '__attribute__((unused)) ' + 'struct lfs_config *CFG) {' % ( + case.name)) f.writeln(4*' '+'// bench case %s' % case.name) if case.code_lineno is not None: - f.writeln(4*' '+'#line %d "%s"' - % (case.code_lineno, suite.path)) + f.writeln(4*' '+'#line %d "%s"' % ( + case.code_lineno, suite.path)) f.write(case.code) if case.code_lineno is not None: - f.writeln(4*' '+'#line %d "%s"' - % (f.lineno+1, args['output'])) + f.writeln(4*' '+'#line %d "%s"' % ( + f.lineno+1, args['output'])) f.writeln('}') f.writeln() @@ -441,19 +442,19 @@ def compile(bench_paths, **args): # write any suite defines if suite.defines: for define in sorted(suite.defines): - f.writeln('__attribute__((weak)) intmax_t %s;' - % define) + f.writeln('__attribute__((weak)) intmax_t %s;' % ( + define)) f.writeln() # write any suite code if suite.code is not None and suite.in_ is None: if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (suite.code_lineno, suite.path)) + f.writeln('#line %d "%s"' % ( + suite.code_lineno, suite.path)) f.write(suite.code) if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (f.lineno+1, args['output'])) + f.writeln('#line %d "%s"' % ( + f.lineno+1, args['output'])) f.writeln() # create case functions @@ -464,15 +465,15 @@ def compile(bench_paths, **args): for i, permutation in enumerate(case.permutations): for k, vs in sorted(permutation.items()): f.writeln('extern intmax_t __bench__%s__%s__%d(' - 'void *data, size_t i);' - % (case.name, k, i)) + 'void *data, size_t i);' % ( + case.name, k, i)) if suite.if_ or case.if_: f.writeln('extern bool __bench__%s__if(' - 'void);' - % (case.name)) + 'void);' % ( + case.name)) f.writeln('extern void __bench__%s__run(' - 'struct lfs_config *CFG);' - % (case.name)) + 'struct lfs_config *CFG);' % ( + case.name)) f.writeln() # write any ifdef epilogues @@ -482,22 +483,22 @@ def compile(bench_paths, **args): f.writeln() # create suite struct - f.writeln('const struct bench_suite __bench__%s__suite = {' - % suite.name) + f.writeln('const struct bench_suite __bench__%s__suite = {' % ( + suite.name)) f.writeln(4*' '+'.name = "%s",' % suite.name) f.writeln(4*' '+'.path = "%s",' % suite.path) - f.writeln(4*' '+'.flags = %s,' - % (' | '.join(filter(None, [ - 'BENCH_INTERNAL' if suite.internal else None])) - or 0)) + f.writeln(4*' '+'.flags = %s,' % ( + ' | '.join(filter(None, [ + 'BENCH_INTERNAL' if suite.internal else None])) + or 0)) for ifdef in suite.ifdef: f.writeln(4*' '+'#ifdef %s' % ifdef) # create suite defines if suite.defines: f.writeln(4*' '+'.defines = (const bench_define_t[]){') for k in sorted(suite.defines): - f.writeln(8*' '+'{"%s", &%s, NULL, NULL, 0},' - % (k, k)) + f.writeln(8*' '+'{"%s", &%s, NULL, NULL, 0},' % ( + k, k)) f.writeln(4*' '+'},') f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) for ifdef in suite.ifdef: @@ -509,39 +510,41 @@ def compile(bench_paths, **args): f.writeln(8*' '+'{') f.writeln(12*' '+'.name = "%s",' % case.name) f.writeln(12*' '+'.path = "%s",' % case.path) - f.writeln(12*' '+'.flags = %s,' - % (' | '.join(filter(None, [ - 'BENCH_INTERNAL' if suite.internal else None])) - or 0)) + f.writeln(12*' '+'.flags = %s,' % ( + ' | '.join(filter(None, [ + 'BENCH_INTERNAL' if suite.internal + else None])) + or 0)) for ifdef in it.chain(suite.ifdef, case.ifdef): f.writeln(12*' '+'#ifdef %s' % ifdef) # create case defines if case.defines: f.writeln(12*' '+'.defines' - ' = (const bench_define_t*)' - '(const bench_define_t[][%d]){' - % (len(suite.defines))) + ' = (const bench_define_t*)' + '(const bench_define_t[][%d]){' % ( + len(suite.defines))) for i, permutation in enumerate(case.permutations): f.writeln(16*' '+'{') for k, vs in sorted(permutation.items()): f.writeln(20*' '+'[%d] = {' '"%s", &%s, ' - '__bench__%s__%s__%d, NULL, %d},' - % (sorted(suite.defines).index(k), - k, k, case.name, k, i, - sum(len(v) - if isinstance(v, range) - else 1 - for v in vs))) + '__bench__%s__%s__%d, ' + 'NULL, %d},' % ( + sorted(suite.defines).index(k), + k, k, case.name, k, i, + sum(len(v) + if isinstance(v, range) + else 1 + for v in vs))) f.writeln(16*' '+'},') f.writeln(12*' '+'},') - f.writeln(12*' '+'.permutations = %d,' - % len(case.permutations)) + f.writeln(12*' '+'.permutations = %d,' % ( + len(case.permutations))) if suite.if_ or case.if_: - f.writeln(12*' '+'.if_ = __bench__%s__if,' - % (case.name)) - f.writeln(12*' '+'.run = __bench__%s__run,' - % (case.name)) + f.writeln(12*' '+'.if_ = __bench__%s__if,' % ( + case.name)) + f.writeln(12*' '+'.run = __bench__%s__run,' % ( + case.name)) for ifdef in it.chain(suite.ifdef, case.ifdef): f.writeln(12*' '+'#endif') f.writeln(8*' '+'},') @@ -560,18 +563,18 @@ def compile(bench_paths, **args): # merge all defines we need, otherwise we will run into # redefinition errors defines = ({define - for suite in suites - if suite.isin(args['source']) - for define in suite.defines} - | {define - for suite in suites - for case in suite.cases - if case.isin(args['source']) - for define in case.defines}) + for suite in suites + if suite.isin(args['source']) + for define in suite.defines} + | {define + for suite in suites + for case in suite.cases + if case.isin(args['source']) + for define in case.defines}) if defines: for define in sorted(defines): - f.writeln('__attribute__((weak)) intmax_t %s;' - % define) + f.writeln('__attribute__((weak)) intmax_t %s;' % ( + define)) f.writeln() # write any internal benches @@ -585,12 +588,12 @@ def compile(bench_paths, **args): # any suite code if suite.isin(args['source']): if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (suite.code_lineno, suite.path)) + f.writeln('#line %d "%s"' % ( + suite.code_lineno, suite.path)) f.write(suite.code) if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (f.lineno+1, args['output'])) + f.writeln('#line %d "%s"' % ( + f.lineno+1, args['output'])) f.writeln() # any case functions @@ -611,11 +614,12 @@ def compile(bench_paths, **args): # will be linked for suite in suites: f.writeln('extern const struct bench_suite ' - '__bench__%s__suite;' % suite.name) + '__bench__%s__suite;' % ( + suite.name)) f.writeln() f.writeln('__attribute__((weak))') - f.writeln('const struct bench_suite *const bench_suites[] = {'); + f.writeln('const struct bench_suite *const bench_suites[] = {') for suite in suites: f.writeln(4*' '+'&__bench__%s__suite,' % suite.name) if len(suites) == 0: @@ -636,24 +640,24 @@ def find_runner(runner, id=None, **args): # run under valgrind? if args.get('valgrind'): cmd[:0] = args['valgrind_path'] + [ - '--leak-check=full', - '--track-origins=yes', - '--error-exitcode=4', - '-q'] + '--leak-check=full', + '--track-origins=yes', + '--error-exitcode=4', + '-q'] # run under perf? if args.get('perf'): cmd[:0] = args['perf_script'] + list(filter(None, [ - '-R', - '--perf-freq=%s' % args['perf_freq'] - if args.get('perf_freq') else None, - '--perf-period=%s' % args['perf_period'] - if args.get('perf_period') else None, - '--perf-events=%s' % args['perf_events'] - if args.get('perf_events') else None, - '--perf-path=%s' % args['perf_path'] - if args.get('perf_path') else None, - '-o%s' % args['perf']])) + '-R', + '--perf-freq=%s' % args['perf_freq'] + if args.get('perf_freq') else None, + '--perf-period=%s' % args['perf_period'] + if args.get('perf_period') else None, + '--perf-events=%s' % args['perf_events'] + if args.get('perf_events') else None, + '--perf-path=%s' % args['perf_path'] + if args.get('perf_path') else None, + '-o%s' % args['perf']])) # other context if args.get('define_depth'): @@ -704,15 +708,15 @@ def find_perms(runner, bench_ids=[], **args): if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) pattern = re.compile( - '^(?P[^\s]+)' - '\s+(?P[^\s]+)' - '\s+(?P\d+)/(?P\d+)') + '^(?P[^\s]+)' + '\s+(?P[^\s]+)' + '\s+(?P\d+)/(?P\d+)') # skip the first line for line in it.islice(proc.stdout, 1, None): m = pattern.match(line) @@ -720,8 +724,8 @@ def find_perms(runner, bench_ids=[], **args): filtered = int(m.group('filtered')) perms = int(m.group('perms')) expected_case_perms[m.group('case')] = ( - expected_case_perms.get(m.group('case'), 0) - + filtered) + expected_case_perms.get(m.group('case'), 0) + + filtered) expected_perms += filtered total_perms += perms proc.wait() @@ -736,14 +740,14 @@ def find_perms(runner, bench_ids=[], **args): if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) pattern = re.compile( - '^(?P[^\s]+)' - '\s+(?P[^:]+):(?P\d+)') + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') # skip the first line for line in it.islice(proc.stdout, 1, None): m = pattern.match(line) @@ -765,15 +769,14 @@ def find_perms(runner, bench_ids=[], **args): expected_suite_perms = co.OrderedDict() for case, suite in case_suites.items(): expected_suite_perms[suite] = ( - expected_suite_perms.get(suite, 0) - + expected_case_perms.get(case, 0)) + expected_suite_perms.get(suite, 0) + + expected_case_perms.get(case, 0)) - return ( - case_suites, - expected_suite_perms, - expected_case_perms, - expected_perms, - total_perms) + return (case_suites, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) def find_path(runner, id, **args): runner_ = find_runner(runner, id, **args) @@ -783,14 +786,14 @@ def find_path(runner, id, **args): if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) pattern = re.compile( - '^(?P[^\s]+)' - '\s+(?P[^:]+):(?P\d+)') + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') # skip the first line for line in it.islice(proc.stdout, 1, None): m = pattern.match(line) @@ -814,11 +817,11 @@ def find_defines(runner, id, **args): if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) defines = co.OrderedDict() pattern = re.compile('^(?P\w+)=(?P.+)') for line in proc.stdout: @@ -846,10 +849,10 @@ def find_ids(runner, bench_ids=[], **args): # lookup suites/cases (suite_cases, - expected_suite_perms, - expected_case_perms, - _, - _) = find_perms(runner, **args) + expected_suite_perms, + expected_case_perms, + _, + _) = find_perms(runner, **args) # no ids => all ids, before we evaluate globs if not bench_ids and args.get('by_cases'): @@ -866,12 +869,12 @@ def find_ids(runner, bench_ids=[], **args): # resolve globs if '*' in name: bench_ids__.extend(suite - for suite in expected_suite_perms.keys() - if fnmatch.fnmatch(suite, name)) + for suite in expected_suite_perms.keys() + if fnmatch.fnmatch(suite, name)) if not bench_ids__: bench_ids__.extend(case_ - for case_ in expected_case_perms.keys() - if fnmatch.fnmatch(case_, name)) + for case_ in expected_case_perms.keys() + if fnmatch.fnmatch(case_, name)) # literal suite elif name in expected_suite_perms: bench_ids__.append(id) @@ -882,10 +885,10 @@ def find_ids(runner, bench_ids=[], **args): # no suite/case found? error if not bench_ids__: print('%serror:%s no benches match id %r?' % ( - '\x1b[01;31m' if args['color'] else '', - '\x1b[m' if args['color'] else '', - id), - file=sys.stderr) + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + id), + file=sys.stderr) sys.exit(-1) bench_ids_.extend(bench_ids__) @@ -975,10 +978,10 @@ class BenchFailure(Exception): def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): # get expected suite/case/perm counts (case_suites, - expected_suite_perms, - expected_case_perms, - expected_perms, - total_perms) = find_perms(runner, bench_ids, **args) + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner, bench_ids, **args) passed_suite_perms = co.defaultdict(lambda: 0) passed_case_perms = co.defaultdict(lambda: 0) @@ -991,18 +994,18 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): killed = False pattern = re.compile('^(?:' - '(?Prunning|finished|skipped)' - ' (?P(?P[^:]+)[^\s]*)' - '|' '(?P[^:]+):(?P\d+):(?Passert):' - ' *(?P.*)' - '|' '(?Pbenched)' - ' (?P[^\s]+)' - ' (?P\d+)' - ' (?P\d+)' - '(?: (?P[\d\.]+))?' - '(?: (?P[\d\.]+))?' - '(?: (?P[\d\.]+))?' - ')$') + '(?Prunning|finished|skipped)' + ' (?P(?P[^:]+)[^\s]*)' + '|' '(?P[^:]+):(?P\d+):(?Passert):' + ' *(?P.*)' + '|' '(?Pbenched)' + ' (?P[^\s]+)' + ' (?P\d+)' + ' (?P\d+)' + '(?: (?P[\d\.]+))?' + '(?: (?P[\d\.]+))?' + '(?: (?P[\d\.]+))?' + ')$') locals = th.local() children = set() @@ -1077,9 +1080,9 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): locals.seen_perms += 1 elif op == 'assert': last_assert = ( - m.group('path'), - int(m.group('lineno')), - m.group('message')) + m.group('path'), + int(m.group('lineno')), + m.group('message')) # go ahead and kill the process, aborting takes a while if args.get('keep_going'): proc.kill() @@ -1103,19 +1106,19 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): # once per perm if last_defines is None: last_defines = find_defines( - runner, last_id, **args) + runner, last_id, **args) # write measurements immediately, this allows # analysis of partial results output_.writerow({ - 'suite': last_suite, - 'case': last_case, - **last_defines, - 'meas': meas, - 'iter': iter, - 'size': size, - 'readed': readed_, - 'proged': proged_, - 'erased': erased_}) + 'suite': last_suite, + 'case': last_case, + **last_defines, + 'meas': meas, + 'iter': iter, + 'size': size, + 'readed': readed_, + 'proged': proged_, + 'erased': erased_}) # keep track of total for summary readed += readed_ proged += proged_ @@ -1130,10 +1133,10 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): proc.wait() if proc.returncode != 0: raise BenchFailure( - last_id, - proc.returncode, - list(last_stdout), - last_assert) + last_id, + proc.returncode, + list(last_stdout), + last_assert) def run_job(start=None, step=None): nonlocal failed_perms @@ -1189,44 +1192,44 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): if 'jobs' in args: for job in range(args['jobs']): runners.append(th.Thread( - target=run_job, args=(job, args['jobs']), - daemon=True)) + target=run_job, args=(job, args['jobs']), + daemon=True)) else: runners.append(th.Thread( - target=run_job, args=(None, None), - daemon=True)) + target=run_job, args=(None, None), + daemon=True)) def print_update(done): if (not args.get('verbose') and not args.get('stdout') == '-' and (args['color'] or done)): sys.stdout.write('%s%srunning %s%s:%s %s%s' % ( - '\r\x1b[K' if args['color'] else '', - '\x1b[?7l' if not done else '', - ('\x1b[34m' if not failed_perms else '\x1b[31m') - if args['color'] else '', - name, - '\x1b[m' if args['color'] else '', - ', '.join(filter(None, [ - '%d/%d suites' % ( - sum(passed_suite_perms[k] == v - for k, v in expected_suite_perms.items()), - len(expected_suite_perms)) - if (not args.get('by_suites') - and not args.get('by_cases')) else None, - '%d/%d cases' % ( - sum(passed_case_perms[k] == v - for k, v in expected_case_perms.items()), - len(expected_case_perms)) - if not args.get('by_cases') else None, - '%d/%d perms' % (passed_perms, expected_perms), - '%s%d/%d failures%s' % ( - '\x1b[31m' if args['color'] else '', - failed_perms, - expected_perms, - '\x1b[m' if args['color'] else '') - if failed_perms else None])), - '\x1b[?7h' if not done else '\n')) + '\r\x1b[K' if args['color'] else '', + '\x1b[?7l' if not done else '', + ('\x1b[34m' if not failed_perms else '\x1b[31m') + if args['color'] else '', + name, + '\x1b[m' if args['color'] else '', + ', '.join(filter(None, [ + '%d/%d suites' % ( + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms)) + if (not args.get('by_suites') + and not args.get('by_cases')) else None, + '%d/%d cases' % ( + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms)) + if not args.get('by_cases') else None, + '%d/%d perms' % (passed_perms, expected_perms), + '%s%d/%d failures%s' % ( + '\x1b[31m' if args['color'] else '', + failed_perms, + expected_perms, + '\x1b[m' if args['color'] else '') + if failed_perms else None])), + '\x1b[?7h' if not done else '\n')) sys.stdout.flush() for r in runners: @@ -1246,34 +1249,33 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): for r in runners: r.join() - return ( - expected_perms, - passed_perms, - failed_perms, - readed, - proged, - erased, - failures, - killed) + return (expected_perms, + passed_perms, + failed_perms, + readed, + proged, + erased, + failures, + killed) def run(runner, bench_ids=[], **args): # query runner for benches print('using runner: %s' % ' '.join( - shlex.quote(c) for c in find_runner(runner, **args))) + shlex.quote(c) for c in find_runner(runner, **args))) # query ids, perms, etc bench_ids = find_ids(runner, bench_ids, **args) (_, - expected_suite_perms, - expected_case_perms, - expected_perms, - total_perms) = find_perms(runner, bench_ids, **args) + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner, bench_ids, **args) print('found %d suites, %d cases, %d/%d permutations' % ( - len(expected_suite_perms), - len(expected_case_perms), - expected_perms, - total_perms)) + len(expected_suite_perms), + len(expected_case_perms), + expected_perms, + total_perms)) print() # automatic job detection? @@ -1290,8 +1292,8 @@ def run(runner, bench_ids=[], **args): output = None if args.get('output'): output = BenchOutput(args['output'], - ['suite', 'case'], - ['meas', 'iter', 'size', 'readed', 'proged', 'erased']) + ['suite', 'case'], + ['meas', 'iter', 'size', 'readed', 'proged', 'erased']) # measure runtime start = time.time() @@ -1307,20 +1309,20 @@ def run(runner, bench_ids=[], **args): for by in (bench_ids if bench_ids else [None]): # spawn jobs for stage (expected_, - passed_, - failed_, - readed_, - proged_, - erased_, - failures_, - killed) = run_stage( - by or 'benches', - runner, - [by] if by is not None else [], - stdout, - trace, - output, - **args) + passed_, + failed_, + readed_, + proged_, + erased_, + failures_, + killed) = run_stage( + by or 'benches', + runner, + [by] if by is not None else [], + stdout, + trace, + output, + **args) # collect passes/failures expected += expected_ passed += passed_ @@ -1331,8 +1333,8 @@ def run(runner, bench_ids=[], **args): # do not store more failures than we need to, otherwise we # quickly explode RAM when a common bug fails a bunch of cases failures.extend(failures_[:max( - args.get('failures', 3) - len(failures), - 0)]) + args.get('failures', 3) - len(failures), + 0)]) if (failed and not args.get('keep_going')) or killed: break @@ -1354,21 +1356,21 @@ def run(runner, bench_ids=[], **args): # show summary print() print('%sdone:%s %s' % ( - ('\x1b[34m' if not failed else '\x1b[31m') - if args['color'] else '', - '\x1b[m' if args['color'] else '', - ', '.join(filter(None, [ - '%d readed' % readed, - '%d proged' % proged, - '%d erased' % erased, - 'in %.2fs' % (stop-start)])))) + ('\x1b[34m' if not failed else '\x1b[31m') + if args['color'] else '', + '\x1b[m' if args['color'] else '', + ', '.join(filter(None, [ + '%d readed' % readed, + '%d proged' % proged, + '%d erased' % erased, + 'in %.2fs' % (stop-start)])))) print() # print each failure for failure in failures[:args.get('failures', 3)]: assert failure.id is not None, '%s broken? %r' % ( - ' '.join(shlex.quote(c) for c in find_runner(runner, **args)), - failure) + ' '.join(shlex.quote(c) for c in find_runner(runner, **args)), + failure) # get some extra info from runner path, lineno = find_path(runner, failure.id, **args) @@ -1376,13 +1378,14 @@ def run(runner, bench_ids=[], **args): # show summary of failure print('%s%s:%d:%sfailure:%s %s%s failed' % ( - '\x1b[01m' if args['color'] else '', - path, lineno, - '\x1b[01;31m' if args['color'] else '', - '\x1b[m' if args['color'] else '', - failure.id, - ' (%s)' % ', '.join('%s=%s' % (k,v) for k,v in defines.items()) - if defines else '')) + '\x1b[01m' if args['color'] else '', + path, lineno, + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + failure.id, + ' (%s)' % ', '.join('%s=%s' % (k,v) + for k,v in defines.items()) + if defines else '')) if failure.stdout: stdout = failure.stdout @@ -1394,11 +1397,11 @@ def run(runner, bench_ids=[], **args): if failure.assert_ is not None: path, lineno, message = failure.assert_ print('%s%s:%d:%sassert:%s %s' % ( - '\x1b[01m' if args['color'] else '', - path, lineno, - '\x1b[01;31m' if args['color'] else '', - '\x1b[m' if args['color'] else '', - message)) + '\x1b[01m' if args['color'] else '', + path, lineno, + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + message)) with open(path) as f: line = next(it.islice(f, lineno-1, None)).strip('\n') print(line) @@ -1416,23 +1419,23 @@ def run(runner, bench_ids=[], **args): # can be helpful path, lineno = find_path(runner, failure.id, **args) cmd[:0] = args['gdb_path'] + [ - '-q', - '-ex', 'break main', - '-ex', 'break %s:%d' % (path, lineno), - '-ex', 'run', - '--args'] + '-q', + '-ex', 'break main', + '-ex', 'break %s:%d' % (path, lineno), + '-ex', 'run', + '--args'] elif args.get('gdb_perm'): path, lineno = find_path(runner, failure.id, **args) cmd[:0] = args['gdb_path'] + [ - '-q', - '-ex', 'break %s:%d' % (path, lineno), - '-ex', 'run', - '--args'] + '-q', + '-ex', 'break %s:%d' % (path, lineno), + '-ex', 'run', + '--args'] else: cmd[:0] = args['gdb_path'] + [ - '-q', - '-ex', 'run', - '--args'] + '-q', + '-ex', 'run', + '--args'] # exec gdb interactively if args.get('verbose'): @@ -1472,228 +1475,232 @@ if __name__ == "__main__": argparse.ArgumentParser._handle_conflict_ignore = lambda *_: None argparse._ArgumentGroup._handle_conflict_ignore = lambda *_: None parser = argparse.ArgumentParser( - description="Build and run benches.", - allow_abbrev=False, - conflict_handler='ignore') + description="Build and run benches.", + allow_abbrev=False, + conflict_handler='ignore') parser.add_argument( - '-v', '--verbose', - action='store_true', - help="Output commands that run behind the scenes.") + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") parser.add_argument( - '--color', - choices=['never', 'always', 'auto'], - default='auto', - help="When to use terminal colors. Defaults to 'auto'.") + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") # bench flags bench_parser = parser.add_argument_group('bench options') bench_parser.add_argument( - 'bench_ids', - nargs='*', - help="Description of benches to run.") + 'bench_ids', + nargs='*', + help="Description of benches to run.") bench_parser.add_argument( - '-R', '--runner', - type=lambda x: x.split(), - default=RUNNER_PATH, - help="Bench runner to use for benching. Defaults to %r." % RUNNER_PATH) + '-R', '--runner', + type=lambda x: x.split(), + default=RUNNER_PATH, + help="Bench runner to use for benching. Defaults to " + "%r." % RUNNER_PATH) bench_parser.add_argument( - '-Y', '--summary', - action='store_true', - help="Show quick summary.") + '-Y', '--summary', + action='store_true', + help="Show quick summary.") bench_parser.add_argument( - '-l', '--list-suites', - action='store_true', - help="List bench suites.") + '-l', '--list-suites', + action='store_true', + help="List bench suites.") bench_parser.add_argument( - '-L', '--list-cases', - action='store_true', - help="List bench cases.") + '-L', '--list-cases', + action='store_true', + help="List bench cases.") bench_parser.add_argument( - '--list-suite-paths', - action='store_true', - help="List the path for each bench suite.") + '--list-suite-paths', + action='store_true', + help="List the path for each bench suite.") bench_parser.add_argument( - '--list-case-paths', - action='store_true', - help="List the path and line number for each bench case.") + '--list-case-paths', + action='store_true', + help="List the path and line number for each bench case.") bench_parser.add_argument( - '--list-defines', - action='store_true', - help="List all defines in this bench-runner.") + '--list-defines', + action='store_true', + help="List all defines in this bench-runner.") bench_parser.add_argument( - '--list-permutation-defines', - action='store_true', - help="List explicit defines in this bench-runner.") + '--list-permutation-defines', + action='store_true', + help="List explicit defines in this bench-runner.") bench_parser.add_argument( - '--list-implicit-defines', - action='store_true', - help="List implicit defines in this bench-runner.") + '--list-implicit-defines', + action='store_true', + help="List implicit defines in this bench-runner.") bench_parser.add_argument( - '-D', '--define', - action='append', - help="Override a bench define.") + '-D', '--define', + action='append', + help="Override a bench define.") bench_parser.add_argument( - '--define-depth', - help="How deep to evaluate recursive defines before erroring.") + '--define-depth', + help="How deep to evaluate recursive defines before erroring.") bench_parser.add_argument( - '-a', '--all', - action='store_true', - help="Ignore bench filters.") + '-a', '--all', + action='store_true', + help="Ignore bench filters.") bench_parser.add_argument( - '-d', '--disk', - help="Direct block device operations to this file.") + '-d', '--disk', + help="Direct block device operations to this file.") bench_parser.add_argument( - '-t', '--trace', - help="Direct trace output to this file.") + '-t', '--trace', + help="Direct trace output to this file.") bench_parser.add_argument( - '--trace-backtrace', - action='store_true', - help="Include a backtrace with every trace statement.") + '--trace-backtrace', + action='store_true', + help="Include a backtrace with every trace statement.") bench_parser.add_argument( - '--trace-period', - help="Sample trace output at this period in cycles.") + '--trace-period', + help="Sample trace output at this period in cycles.") bench_parser.add_argument( - '--trace-freq', - help="Sample trace output at this frequency in hz.") + '--trace-freq', + help="Sample trace output at this frequency in hz.") bench_parser.add_argument( - '-O', '--stdout', - help="Direct stdout to this file. Note stderr is already merged here.") + '-O', '--stdout', + help="Direct stdout to this file. Note stderr is already merged " + "here.") bench_parser.add_argument( - '-o', '--output', - help="CSV file to store results.") + '-o', '--output', + help="CSV file to store results.") bench_parser.add_argument( - '--read-sleep', - help="Artificial read delay in seconds.") + '--read-sleep', + help="Artificial read delay in seconds.") bench_parser.add_argument( - '--prog-sleep', - help="Artificial prog delay in seconds.") + '--prog-sleep', + help="Artificial prog delay in seconds.") bench_parser.add_argument( - '--erase-sleep', - help="Artificial erase delay in seconds.") + '--erase-sleep', + help="Artificial erase delay in seconds.") bench_parser.add_argument( - '-j', '--jobs', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Number of parallel runners to run. 0 runs one runner per core.") + '-j', '--jobs', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Number of parallel runners to run. 0 runs one runner per " + "core.") bench_parser.add_argument( - '-k', '--keep-going', - action='store_true', - help="Don't stop on first failure.") + '-k', '--keep-going', + action='store_true', + help="Don't stop on first failure.") bench_parser.add_argument( - '-f', '--fail', - action='store_true', - help="Force a failure.") + '-f', '--fail', + action='store_true', + help="Force a failure.") bench_parser.add_argument( - '-i', '--isolate', - action='store_true', - help="Run each bench permutation in a separate process.") + '-i', '--isolate', + action='store_true', + help="Run each bench permutation in a separate process.") bench_parser.add_argument( - '-b', '--by-suites', - action='store_true', - help="Step through benches by suite.") + '-b', '--by-suites', + action='store_true', + help="Step through benches by suite.") bench_parser.add_argument( - '-B', '--by-cases', - action='store_true', - help="Step through benches by case.") + '-B', '--by-cases', + action='store_true', + help="Step through benches by case.") bench_parser.add_argument( - '-F', '--failures', - type=lambda x: int(x, 0), - default=3, - help="Show this many bench failures. Defaults to 3.") + '-F', '--failures', + type=lambda x: int(x, 0), + default=3, + help="Show this many bench failures. Defaults to 3.") bench_parser.add_argument( - '-C', '--context', - type=lambda x: int(x, 0), - default=5, - help="Show this many lines of stdout on bench failure. " - "Defaults to 5.") + '-C', '--context', + type=lambda x: int(x, 0), + default=5, + help="Show this many lines of stdout on bench failure. " + "Defaults to 5.") bench_parser.add_argument( - '--gdb', - action='store_true', - help="Drop into gdb on bench failure.") + '--gdb', + action='store_true', + help="Drop into gdb on bench failure.") bench_parser.add_argument( - '--gdb-perm', '--gdb-permutation', - action='store_true', - help="Drop into gdb on bench failure but stop at the beginning " - "of the failing bench case.") + '--gdb-perm', '--gdb-permutation', + action='store_true', + help="Drop into gdb on bench failure but stop at the beginning " + "of the failing bench case.") bench_parser.add_argument( - '--gdb-main', - action='store_true', - help="Drop into gdb on bench failure but stop at the beginning " - "of main.") + '--gdb-main', + action='store_true', + help="Drop into gdb on bench failure but stop at the beginning " + "of main.") bench_parser.add_argument( - '--gdb-path', - type=lambda x: x.split(), - default=GDB_PATH, - help="Path to the gdb executable, may include flags. " - "Defaults to %r." % GDB_PATH) + '--gdb-path', + type=lambda x: x.split(), + default=GDB_PATH, + help="Path to the gdb executable, may include flags. " + "Defaults to %r." % GDB_PATH) bench_parser.add_argument( - '--exec', - type=lambda e: e.split(), - help="Run under another executable.") + '--exec', + type=lambda e: e.split(), + help="Run under another executable.") bench_parser.add_argument( - '--valgrind', - action='store_true', - help="Run under Valgrind to find memory errors. Implicitly sets " - "--isolate.") + '--valgrind', + action='store_true', + help="Run under Valgrind to find memory errors. Implicitly sets " + "--isolate.") bench_parser.add_argument( - '--valgrind-path', - type=lambda x: x.split(), - default=VALGRIND_PATH, - help="Path to the Valgrind executable, may include flags. " - "Defaults to %r." % VALGRIND_PATH) + '--valgrind-path', + type=lambda x: x.split(), + default=VALGRIND_PATH, + help="Path to the Valgrind executable, may include flags. " + "Defaults to %r." % VALGRIND_PATH) bench_parser.add_argument( - '-p', '--perf', - help="Run under Linux's perf to sample performance counters, writing " - "samples to this file.") + '-p', '--perf', + help="Run under Linux's perf to sample performance counters, " + "writing samples to this file.") bench_parser.add_argument( - '--perf-freq', - help="perf sampling frequency. This is passed directly to the perf " - "script.") + '--perf-freq', + help="perf sampling frequency. This is passed directly to the " + "perf script.") bench_parser.add_argument( - '--perf-period', - help="perf sampling period. This is passed directly to the perf " - "script.") + '--perf-period', + help="perf sampling period. This is passed directly to the perf " + "script.") bench_parser.add_argument( - '--perf-events', - help="perf events to record. This is passed directly to the perf " - "script.") + '--perf-events', + help="perf events to record. This is passed directly to the perf " + "script.") bench_parser.add_argument( - '--perf-script', - type=lambda x: x.split(), - default=PERF_SCRIPT, - help="Path to the perf script to use. Defaults to %r." % PERF_SCRIPT) + '--perf-script', + type=lambda x: x.split(), + default=PERF_SCRIPT, + help="Path to the perf script to use. Defaults to " + "%r." % PERF_SCRIPT) bench_parser.add_argument( - '--perf-path', - type=lambda x: x.split(), - help="Path to the perf executable, may include flags. This is passed " - "directly to the perf script") + '--perf-path', + type=lambda x: x.split(), + help="Path to the perf executable, may include flags. This is " + "passed directly to the perf script") # compilation flags comp_parser = parser.add_argument_group('compilation options') comp_parser.add_argument( - 'bench_paths', - nargs='*', - help="Set of *.toml files to compile.") + 'bench_paths', + nargs='*', + help="Set of *.toml files to compile.") comp_parser.add_argument( - '-c', '--compile', - action='store_true', - help="Compile a bench suite or source file.") + '-c', '--compile', + action='store_true', + help="Compile a bench suite or source file.") comp_parser.add_argument( - '-s', '--source', - help="Source file to compile, possibly injecting internal benches.") + '-s', '--source', + help="Source file to compile, possibly injecting internal benches.") comp_parser.add_argument( - '--include', - default=HEADER_PATH, - help="Inject this header file into every compiled bench file. " - "Defaults to %r." % HEADER_PATH) + '--include', + default=HEADER_PATH, + help="Inject this header file into every compiled bench file. " + "Defaults to %r." % HEADER_PATH) comp_parser.add_argument( - '-o', '--output', - help="Output file.") + '-o', '--output', + help="Output file.") # do the thing args = parser.parse_intermixed_args() args.bench_paths = args.bench_ids sys.exit(main(**{k: v - for k, v in vars(args).items() - if v is not None})) + for k, v in vars(args).items() + if v is not None})) diff --git a/scripts/changeprefix.py b/scripts/changeprefix.py index f59deb31..d20a7a62 100755 --- a/scripts/changeprefix.py +++ b/scripts/changeprefix.py @@ -21,6 +21,7 @@ import shutil import subprocess import tempfile + GIT_PATH = ['git'] @@ -36,17 +37,17 @@ def openio(path, mode='r', buffering=-1): def changeprefix(from_prefix, to_prefix, line): line, count1 = re.subn( - '\\b'+from_prefix, - to_prefix, - line) + '\\b'+from_prefix, + to_prefix, + line) line, count2 = re.subn( - '\\b'+from_prefix.upper(), - to_prefix.upper(), - line) + '\\b'+from_prefix.upper(), + to_prefix.upper(), + line) line, count3 = re.subn( - '\\B-D'+from_prefix.upper(), - '-D'+to_prefix.upper(), - line) + '\\B-D'+from_prefix.upper(), + '-D'+to_prefix.upper(), + line) return line, count1+count2+count3 def changefile(from_prefix, to_prefix, from_path, to_path, *, @@ -79,8 +80,9 @@ def changefile(from_prefix, to_prefix, from_path, to_path, *, # Summary print('%s: %d replacements' % ( - '%s -> %s' % (from_path, to_path) if not to_path_temp else from_path, - count)) + '%s -> %s' % (from_path, to_path) if not to_path_temp + else from_path, + count)) def main(from_prefix, to_prefix, paths=[], *, verbose=False, @@ -111,7 +113,7 @@ def main(from_prefix, to_prefix, paths=[], *, # rename contents changefile(from_prefix, to_prefix, from_path, to_path, - no_replacements=no_replacements) + no_replacements=no_replacements) # stage? if git and not no_stage: @@ -130,49 +132,49 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Change prefixes in files/filenames. Useful for creating " - "different versions of a codebase that don't conflict at compile " - "time.", - allow_abbrev=False) + description="Change prefixes in files/filenames. Useful for " + "creating different versions of a codebase that don't " + "conflict at compile time.", + allow_abbrev=False) parser.add_argument( - 'from_prefix', - help="Prefix to replace.") + 'from_prefix', + help="Prefix to replace.") parser.add_argument( - 'to_prefix', - help="Prefix to replace with.") + 'to_prefix', + help="Prefix to replace with.") parser.add_argument( - 'paths', - nargs='*', - help="Files to operate on.") + 'paths', + nargs='*', + help="Files to operate on.") parser.add_argument( - '-v', '--verbose', - action='store_true', - help="Output commands that run behind the scenes.") + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") parser.add_argument( - '-o', '--output', - help="Output file.") + '-o', '--output', + help="Output file.") parser.add_argument( - '-N', '--no-replacements', - action='store_true', - help="Don't change prefixes in files") + '-N', '--no-replacements', + action='store_true', + help="Don't change prefixes in files") parser.add_argument( - '-R', '--no-renames', - action='store_true', - help="Don't rename files") + '-R', '--no-renames', + action='store_true', + help="Don't rename files") parser.add_argument( - '--git', - action='store_true', - help="Use git to find/update files.") + '--git', + action='store_true', + help="Use git to find/update files.") parser.add_argument( - '--no-stage', - action='store_true', - help="Don't stage changes with git.") + '--no-stage', + action='store_true', + help="Don't stage changes with git.") parser.add_argument( - '--git-path', - type=lambda x: x.split(), - default=GIT_PATH, - help="Path to git executable, may include flags. " - "Defaults to %r." % GIT_PATH) + '--git-path', + type=lambda x: x.split(), + default=GIT_PATH, + help="Path to git executable, may include flags. " + "Defaults to %r." % GIT_PATH) sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/code.py b/scripts/code.py index 1660bc6d..9e2dde07 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -115,11 +115,11 @@ class CodeResult(co.namedtuple('CodeResult', [ __slots__ = () def __new__(cls, file='', function='', size=0): return super().__new__(cls, file, function, - RInt(size)) + RInt(size)) def __add__(self, other): return CodeResult(self.file, self.function, - self.size + other.size) + self.size + other.size) def openio(path, mode='r', buffering=-1): @@ -140,18 +140,18 @@ def collect(obj_paths, *, everything=False, **args): size_pattern = re.compile( - '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(nm_types) + - ' (?P.+?)$') + '^(?P[0-9a-fA-F]+)' + + ' (?P[%s])' % re.escape(nm_types) + + ' (?P.+?)$') line_pattern = re.compile( - '^\s+(?P[0-9]+)' - '(?:\s+(?P[0-9]+))?' - '\s+.*' - '\s+(?P[^\s]+)$') + '^\s+(?P[0-9]+)' + '(?:\s+(?P[0-9]+))?' + '\s+.*' + '\s+(?P[^\s]+)$') info_pattern = re.compile( - '^(?:.*(?PDW_TAG_[a-z_]+).*' - '|.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' - '|.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') + '^(?:.*(?PDW_TAG_[a-z_]+).*' + '|.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' + '|.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') results = [] for path in obj_paths: @@ -165,11 +165,11 @@ def collect(obj_paths, *, if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) for line in proc.stdout: m = size_pattern.match(line) if m: @@ -178,8 +178,8 @@ def collect(obj_paths, *, if not everything and func.startswith('__'): continue results_.append(CodeResult( - file, func, - int(m.group('size'), 16))) + file, func, + int(m.group('size'), 16))) proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -196,11 +196,11 @@ def collect(obj_paths, *, if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) for line in proc.stdout: # note that files contain references to dirs, which we # dereference as soon as we see them as each file table follows a @@ -215,8 +215,8 @@ def collect(obj_paths, *, dir = int(m.group('dir')) if dir in dirs: files[int(m.group('no'))] = os.path.join( - dirs[dir], - m.group('path')) + dirs[dir], + m.group('path')) else: files[int(m.group('no'))] = m.group('path') proc.wait() @@ -241,11 +241,11 @@ def collect(obj_paths, *, if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) for line in proc.stdout: # state machine here to find definitions m = info_pattern.match(line) @@ -279,17 +279,16 @@ def collect(obj_paths, *, file = defs[r.function] else: _, file = max( - defs.items(), - key=lambda d: difflib.SequenceMatcher(None, - d[0], - r.function, False).ratio()) + defs.items(), + key=lambda d: difflib.SequenceMatcher(None, + d[0], + r.function, False).ratio()) else: file = r.file # ignore filtered sources if sources is not None: - if not any( - os.path.abspath(file) == os.path.abspath(s) + if not any(os.path.abspath(file) == os.path.abspath(s) for s in sources): continue else: @@ -319,7 +318,7 @@ def fold(Result, results, by=None, defines=[]): for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k, - file=sys.stderr) + file=sys.stderr) sys.exit(-1) # filter by matching defines @@ -368,52 +367,55 @@ def table(Result, results, diff_results=None, *, # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r - for r in results} + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r - for r in diff_results or []} + ','.join(str(getattr(r, k) or '') for k in by): r + for r in diff_results or []} names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] + for name in table.keys() | diff_table.keys() + if diff_results is None + or all_ + or any( + types[k].ratio( + getattr(table.get(name), k, None), + getattr(diff_table.get(name), k, None)) + for k in fields)] # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) + names.sort( + key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), + reverse=True) if sort: for k, reverse in reversed(sort): names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None else () - for k in ([k] if k else [ - k for k in Result._sort if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + key=lambda n: tuple( + (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None + else () + for k in ( + [k] if k else [ + k for k in Result._sort + if k in fields])), + reverse=reverse ^ (not k or k in Result._fields)) # build up our lines lines = [] # header - header = [ - '%s%s' % ( - ','.join(by), - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') + header = ['%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') if not summary else ''] if diff_results is None: for k in fields: @@ -436,43 +438,43 @@ def table(Result, results, diff_results=None, *, if diff_results is None: for k in fields: entry.append( - (getattr(r, k).table(), - getattr(getattr(r, k), 'notes', lambda: [])()) - if getattr(r, k, None) is not None - else types[k].none) + (getattr(r, k).table(), + getattr(getattr(r, k), 'notes', lambda: [])()) + if getattr(r, k, None) is not None + else types[k].none) elif percent: for k in fields: entry.append( - (getattr(r, k).table() - if getattr(r, k, None) is not None - else types[k].none, - (lambda t: ['+∞%'] if t == +mt.inf - else ['-∞%'] if t == -mt.inf - else ['%+.1f%%' % (100*t)])( - types[k].ratio( - getattr(r, k, None), - getattr(diff_r, k, None))))) + (getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none, + (lambda t: ['+∞%'] if t == +mt.inf + else ['-∞%'] if t == -mt.inf + else ['%+.1f%%' % (100*t)])( + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None))))) else: for k in fields: entry.append(getattr(diff_r, k).table() - if getattr(diff_r, k, None) is not None - else types[k].none) + if getattr(diff_r, k, None) is not None + else types[k].none) for k in fields: entry.append(getattr(r, k).table() - if getattr(r, k, None) is not None - else types[k].none) + if getattr(r, k, None) is not None + else types[k].none) for k in fields: entry.append( - (types[k].diff( - getattr(r, k, None), - getattr(diff_r, k, None)), - (lambda t: ['+∞%'] if t == +mt.inf - else ['-∞%'] if t == -mt.inf - else ['%+.1f%%' % (100*t)] if t - else [])( - types[k].ratio( + (types[k].diff( getattr(r, k, None), - getattr(diff_r, k, None))))) + getattr(diff_r, k, None)), + (lambda t: ['+∞%'] if t == +mt.inf + else ['-∞%'] if t == -mt.inf + else ['%+.1f%%' % (100*t)] if t + else [])( + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None))))) return entry # entries @@ -495,8 +497,8 @@ def table(Result, results, diff_results=None, *, # homogenize lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + [x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -510,11 +512,11 @@ def table(Result, results, diff_results=None, *, # print our table for line in lines: print('%-*s %s' % ( - widths[0], line[0][0], - ' '.join('%*s%-*s' % ( - widths[i], x[0], - notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') - for i, x in enumerate(line[1:], 1)))) + widths[0], line[0][0], + ' '.join('%*s%-*s' % ( + widths[i], x[0], + notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') + for i, x in enumerate(line[1:], 1)))) def main(obj_paths, *, @@ -540,10 +542,10 @@ def main(obj_paths, *, continue try: results.append(CodeResult( - **{k: r[k] for k in CodeResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in CodeResult._fields - if k in r and r[k].strip()})) + **{k: r[k] for k in CodeResult._by + if k in r and r[k].strip()}, + **{k: r[k] for k in CodeResult._fields + if k in r and r[k].strip()})) except TypeError: pass @@ -555,25 +557,27 @@ def main(obj_paths, *, if sort: for k, reverse in reversed(sort): results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else CodeResult._sort)), - reverse=reverse ^ (not k or k in CodeResult._fields)) + key=lambda r: tuple( + (getattr(r, k),) if getattr(r, k) is not None else () + for k in ([k] if k else CodeResult._sort)), + reverse=reverse ^ (not k or k in CodeResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: writer = csv.DictWriter(f, - (by if by is not None else CodeResult._by) - + [k for k in ( - fields if fields is not None else CodeResult._fields)]) + (by if by is not None else CodeResult._by) + + [k for k in ( + fields if fields is not None + else CodeResult._fields)]) writer.writeheader() for r in results: writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else CodeResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None else CodeResult._fields)}) + {k: getattr(r, k) for k in ( + by if by is not None else CodeResult._by)} + | {k: getattr(r, k) for k in ( + fields if fields is not None + else CodeResult._fields)}) # find previous results? if args.get('diff'): @@ -591,10 +595,10 @@ def main(obj_paths, *, continue try: diff_results.append(CodeResult( - **{k: r[k] for k in CodeResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in CodeResult._fields - if k in r and r[k].strip()})) + **{k: r[k] for k in CodeResult._by + if k in r and r[k].strip()}, + **{k: r[k] for k in CodeResult._fields + if k in r and r[k].strip()})) except TypeError: pass except FileNotFoundError: @@ -606,115 +610,116 @@ def main(obj_paths, *, # print table if not args.get('quiet'): table(CodeResult, results, - diff_results if args.get('diff') else None, - by=by if by is not None else ['function'], - fields=fields, - sort=sort, - **args) + diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, + **args) if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find code size at the function level.", - allow_abbrev=False) + description="Find code size at the function level.", + allow_abbrev=False) parser.add_argument( - 'obj_paths', - nargs='*', - help="Input *.o files.") + 'obj_paths', + nargs='*', + help="Input *.o files.") parser.add_argument( - '-v', '--verbose', - action='store_true', - help="Output commands that run behind the scenes.") + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") parser.add_argument( - '-q', '--quiet', - action='store_true', - help="Don't show anything, useful with -o.") + '-q', '--quiet', + action='store_true', + help="Don't show anything, useful with -o.") parser.add_argument( - '-o', '--output', - help="Specify CSV file to store results.") + '-o', '--output', + help="Specify CSV file to store results.") parser.add_argument( - '-u', '--use', - help="Don't parse anything, use this CSV file.") + '-u', '--use', + help="Don't parse anything, use this CSV file.") parser.add_argument( - '-d', '--diff', - help="Specify CSV file to diff against.") + '-d', '--diff', + help="Specify CSV file to diff against.") parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( - '-p', '--percent', - action='store_true', - help="Only show percentage change, not a full diff.") + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by', - action='append', - choices=CodeResult._by, - help="Group by this field.") + '-b', '--by', + action='append', + choices=CodeResult._by, + help="Group by this field.") parser.add_argument( - '-f', '--field', - dest='fields', - action='append', - choices=CodeResult._fields, - help="Show this field.") + '-f', '--field', + dest='fields', + action='append', + choices=CodeResult._fields, + help="Show this field.") parser.add_argument( - '-D', '--define', - dest='defines', - action='append', - type=lambda x: ( - lambda k, vs: ( - k.strip(), - {v.strip() for v in vs.split(',')}) - )(*x.split('=', 1)), - help="Only include results where this field is this value.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: ( + lambda k, vs: ( + k.strip(), + {v.strip() for v in vs.split(',')}) + )(*x.split('=', 1)), + help="Only include results where this field is this value.") class AppendSort(argparse.Action): def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] namespace.sort.append((value, True if option == '-S' else False)) parser.add_argument( - '-s', '--sort', - nargs='?', - action=AppendSort, - help="Sort by this field.") + '-s', '--sort', + nargs='?', + action=AppendSort, + help="Sort by this field.") parser.add_argument( - '-S', '--reverse-sort', - nargs='?', - action=AppendSort, - help="Sort by this field, but backwards.") + '-S', '--reverse-sort', + nargs='?', + action=AppendSort, + help="Sort by this field, but backwards.") parser.add_argument( - '-Y', '--summary', - action='store_true', - help="Only show the total.") + '-Y', '--summary', + action='store_true', + help="Only show the total.") parser.add_argument( - '-F', '--source', - dest='sources', - action='append', - help="Only consider definitions in this file. Defaults to anything " - "in the current directory.") + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to " + "anything in the current directory.") parser.add_argument( - '--everything', - action='store_true', - help="Include builtin and libc specific symbols.") + '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") parser.add_argument( - '--nm-types', - default=NM_TYPES, - help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %r." % NM_TYPES) + '--nm-types', + default=NM_TYPES, + help="Type of symbols to report, this uses the same " + "single-character type-names emitted by nm. Defaults to " + "%r." % NM_TYPES) parser.add_argument( - '--nm-path', - type=lambda x: x.split(), - default=NM_PATH, - help="Path to the nm executable, may include flags. " - "Defaults to %r." % NM_PATH) + '--nm-path', + type=lambda x: x.split(), + default=NM_PATH, + help="Path to the nm executable, may include flags. " + "Defaults to %r." % NM_PATH) parser.add_argument( - '--objdump-path', - type=lambda x: x.split(), - default=OBJDUMP_PATH, - help="Path to the objdump executable, may include flags. " - "Defaults to %r." % OBJDUMP_PATH) + '--objdump-path', + type=lambda x: x.split(), + default=OBJDUMP_PATH, + help="Path to the objdump executable, may include flags. " + "Defaults to %r." % OBJDUMP_PATH) sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/cov.py b/scripts/cov.py index 7c0f4cd5..dbe369d7 100755 --- a/scripts/cov.py +++ b/scripts/cov.py @@ -4,8 +4,8 @@ # # Example: # ./scripts/cov.py \ -# lfs.t.a.gcda lfs_util.t.a.gcda \ -# -Flfs.c -Flfs_util.c -slines +# lfs.t.a.gcda lfs_util.t.a.gcda \ +# -Flfs.c -Flfs_util.c -slines # # Copyright (c) 2022, The littlefs authors. # Copyright (c) 2020, Arm Limited. All rights reserved. @@ -22,6 +22,7 @@ import re import shlex import subprocess as sp + # TODO use explode_asserts to avoid counting assert branches? # TODO use dwarf=info to find functions for inline functions? @@ -128,15 +129,15 @@ class RFrac(co.namedtuple('RFrac', 'a,b')): def notes(self): t = self.a.x/self.b.x if self.b.x else 1.0 return ['∞%' if t == +mt.inf - else '-∞%' if t == -mt.inf - else '%.1f%%' % (100*t)] + else '-∞%' if t == -mt.inf + else '%.1f%%' % (100*t)] def diff(self, other): new_a, new_b = self if self else (RInt(0), RInt(0)) old_a, old_b = other if other else (RInt(0), RInt(0)) return '%11s' % ('%s/%s' % ( - new_a.diff(old_a).strip(), - new_b.diff(old_b).strip())) + new_a.diff(old_a).strip(), + new_b.diff(old_b).strip())) def ratio(self, other): new_a, new_b = self if self else (RInt(0), RInt(0)) @@ -184,23 +185,23 @@ class CovResult(co.namedtuple('CovResult', [ _fields = ['calls', 'hits', 'funcs', 'lines', 'branches'] _sort = ['funcs', 'lines', 'branches', 'hits', 'calls'] _types = { - 'calls': RInt, 'hits': RInt, - 'funcs': RFrac, 'lines': RFrac, 'branches': RFrac} + 'calls': RInt, 'hits': RInt, + 'funcs': RFrac, 'lines': RFrac, 'branches': RFrac} __slots__ = () def __new__(cls, file='', function='', line=0, calls=0, hits=0, funcs=0, lines=0, branches=0): return super().__new__(cls, file, function, int(RInt(line)), - RInt(calls), RInt(hits), - RFrac(funcs), RFrac(lines), RFrac(branches)) + RInt(calls), RInt(hits), + RFrac(funcs), RFrac(lines), RFrac(branches)) def __add__(self, other): return CovResult(self.file, self.function, self.line, - max(self.calls, other.calls), - max(self.hits, other.hits), - self.funcs + other.funcs, - self.lines + other.lines, - self.branches + other.branches) + max(self.calls, other.calls), + max(self.hits, other.hits), + self.funcs + other.funcs, + self.lines + other.lines, + self.branches + other.branches) def openio(path, mode='r', buffering=-1): @@ -226,11 +227,11 @@ def collect(gcda_paths, *, if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) data = json.load(proc.stdout) proc.wait() if proc.returncode != 0: @@ -243,8 +244,7 @@ def collect(gcda_paths, *, for file in data['files']: # ignore filtered sources if sources is not None: - if not any( - os.path.abspath(file['file']) == os.path.abspath(s) + if not any(os.path.abspath(file['file']) == os.path.abspath(s) for s in sources): continue else: @@ -272,11 +272,11 @@ def collect(gcda_paths, *, # go ahead and add functions, later folding will merge this if # there are other hits on this line results.append(CovResult( - file_name, func_name, func['start_line'], - func['execution_count'], 0, - RFrac(1 if func['execution_count'] > 0 else 0, 1), - 0, - 0)) + file_name, func_name, func['start_line'], + func['execution_count'], 0, + RFrac(1 if func['execution_count'] > 0 else 0, 1), + 0, + 0)) for line in file['lines']: func_name = line.get('function_name', '(inlined)') @@ -288,14 +288,14 @@ def collect(gcda_paths, *, # go ahead and add lines, later folding will merge this if # there are other hits on this line results.append(CovResult( - file_name, func_name, line['line_number'], - 0, line['count'], - 0, - RFrac(1 if line['count'] > 0 else 0, 1), - RFrac( - sum(1 if branch['count'] > 0 else 0 - for branch in line['branches']), - len(line['branches'])))) + file_name, func_name, line['line_number'], + 0, line['count'], + 0, + RFrac(1 if line['count'] > 0 else 0, 1), + RFrac( + sum(1 if branch['count'] > 0 else 0 + for branch in line['branches']), + len(line['branches'])))) return results @@ -307,7 +307,7 @@ def fold(Result, results, by=None, defines=[]): for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k, - file=sys.stderr) + file=sys.stderr) sys.exit(-1) # filter by matching defines @@ -356,52 +356,55 @@ def table(Result, results, diff_results=None, *, # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r - for r in results} + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r - for r in diff_results or []} + ','.join(str(getattr(r, k) or '') for k in by): r + for r in diff_results or []} names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] + for name in table.keys() | diff_table.keys() + if diff_results is None + or all_ + or any( + types[k].ratio( + getattr(table.get(name), k, None), + getattr(diff_table.get(name), k, None)) + for k in fields)] # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) + names.sort( + key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), + reverse=True) if sort: for k, reverse in reversed(sort): names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None else () - for k in ([k] if k else [ - k for k in Result._sort if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + key=lambda n: tuple( + (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None + else () + for k in ( + [k] if k else [ + k for k in Result._sort + if k in fields])), + reverse=reverse ^ (not k or k in Result._fields)) # build up our lines lines = [] # header - header = [ - '%s%s' % ( - ','.join(by), - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') + header = ['%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') if not summary else ''] if diff_results is None: for k in fields: @@ -424,43 +427,43 @@ def table(Result, results, diff_results=None, *, if diff_results is None: for k in fields: entry.append( - (getattr(r, k).table(), - getattr(getattr(r, k), 'notes', lambda: [])()) - if getattr(r, k, None) is not None - else types[k].none) + (getattr(r, k).table(), + getattr(getattr(r, k), 'notes', lambda: [])()) + if getattr(r, k, None) is not None + else types[k].none) elif percent: for k in fields: entry.append( - (getattr(r, k).table() - if getattr(r, k, None) is not None - else types[k].none, - (lambda t: ['+∞%'] if t == +mt.inf - else ['-∞%'] if t == -mt.inf - else ['%+.1f%%' % (100*t)])( - types[k].ratio( - getattr(r, k, None), - getattr(diff_r, k, None))))) + (getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none, + (lambda t: ['+∞%'] if t == +mt.inf + else ['-∞%'] if t == -mt.inf + else ['%+.1f%%' % (100*t)])( + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None))))) else: for k in fields: entry.append(getattr(diff_r, k).table() - if getattr(diff_r, k, None) is not None - else types[k].none) + if getattr(diff_r, k, None) is not None + else types[k].none) for k in fields: entry.append(getattr(r, k).table() - if getattr(r, k, None) is not None - else types[k].none) + if getattr(r, k, None) is not None + else types[k].none) for k in fields: entry.append( - (types[k].diff( - getattr(r, k, None), - getattr(diff_r, k, None)), - (lambda t: ['+∞%'] if t == +mt.inf - else ['-∞%'] if t == -mt.inf - else ['%+.1f%%' % (100*t)] if t - else [])( - types[k].ratio( + (types[k].diff( getattr(r, k, None), - getattr(diff_r, k, None))))) + getattr(diff_r, k, None)), + (lambda t: ['+∞%'] if t == +mt.inf + else ['-∞%'] if t == -mt.inf + else ['%+.1f%%' % (100*t)] if t + else [])( + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None))))) return entry # entries @@ -483,8 +486,8 @@ def table(Result, results, diff_results=None, *, # homogenize lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + [x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -498,11 +501,11 @@ def table(Result, results, diff_results=None, *, # print our table for line in lines: print('%-*s %s' % ( - widths[0], line[0][0], - ' '.join('%*s%-*s' % ( - widths[i], x[0], - notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') - for i, x in enumerate(line[1:], 1)))) + widths[0], line[0][0], + ' '.join('%*s%-*s' % ( + widths[i], x[0], + notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') + for i, x in enumerate(line[1:], 1)))) def annotate(Result, results, *, @@ -529,14 +532,14 @@ def annotate(Result, results, *, or (branches and r.branches.a < r.branches.b)): if last is not None and line - last.stop <= args['context']: last = range( - last.start, - line+1+args['context']) + last.start, + line+1+args['context']) else: if last is not None: spans.append((last, func)) last = range( - line-args['context'], - line+1+args['context']) + line-args['context'], + line+1+args['context']) func = r.function if last is not None: spans.append((last, func)) @@ -552,11 +555,11 @@ def annotate(Result, results, *, if skipped: skipped = False print('%s@@ %s:%d: %s @@%s' % ( - '\x1b[36m' if args['color'] else '', - path, - i+1, - next(iter(f for _, f in spans)), - '\x1b[m' if args['color'] else '')) + '\x1b[36m' if args['color'] else '', + path, + i+1, + next(iter(f for _, f in spans)), + '\x1b[m' if args['color'] else '')) # build line if line.endswith('\n'): @@ -565,11 +568,11 @@ def annotate(Result, results, *, if i+1 in table: r = table[i+1] line = '%-*s // %s hits%s' % ( - args['width'], - line, - r.hits, - ', %s branches' % (r.branches,) - if int(r.branches.b) else '') + args['width'], + line, + r.hits, + ', %s branches' % (r.branches,) + if int(r.branches.b) else '') if args['color']: if lines and int(r.hits) == 0: @@ -612,11 +615,11 @@ def main(gcda_paths, *, continue try: results.append(CovResult( - **{k: r[k] for k in CovResult._by - if k in r and r[k].strip()}, - **{k: r[k] - for k in CovResult._fields - if k in r and r[k].strip()})) + **{k: r[k] for k in CovResult._by + if k in r and r[k].strip()}, + **{k: r[k] + for k in CovResult._fields + if k in r and r[k].strip()})) except TypeError: pass @@ -628,25 +631,27 @@ def main(gcda_paths, *, if sort: for k, reverse in reversed(sort): results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else CovResult._sort)), - reverse=reverse ^ (not k or k in CovResult._fields)) + key=lambda r: tuple( + (getattr(r, k),) if getattr(r, k) is not None else () + for k in ([k] if k else CovResult._sort)), + reverse=reverse ^ (not k or k in CovResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: writer = csv.DictWriter(f, - (by if by is not None else CovResult._by) - + [k for k in ( - fields if fields is not None else CovResult._fields)]) + (by if by is not None else CovResult._by) + + [k for k in ( + fields if fields is not None + else CovResult._fields)]) writer.writeheader() for r in results: writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else CovResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None else CovResult._fields)}) + {k: getattr(r, k) for k in ( + by if by is not None else CovResult._by)} + | {k: getattr(r, k) for k in ( + fields if fields is not None + else CovResult._fields)}) # find previous results? if args.get('diff'): @@ -664,19 +669,17 @@ def main(gcda_paths, *, continue try: diff_results.append(CovResult( - **{k: r[k] for k in CovResult._by - if k in r and r[k].strip()}, - **{k: r[k] - for k in CovResult._fields - if k in r and r[k].strip()})) + **{k: r[k] for k in CovResult._by + if k in r and r[k].strip()}, + **{k: r[k] for k in CovResult._fields + if k in r and r[k].strip()})) except TypeError: pass except FileNotFoundError: pass # fold - diff_results = fold(CovResult, diff_results, - by=by, defines=defines) + diff_results = fold(CovResult, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): @@ -688,13 +691,13 @@ def main(gcda_paths, *, else: # print table table(CovResult, results, - diff_results if args.get('diff') else None, - by=by if by is not None else ['function'], - fields=fields if fields is not None - else ['lines', 'branches'] if not hits - else ['calls', 'hits'], - sort=sort, - **args) + diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields if fields is not None + else ['lines', 'branches'] if not hits + else ['calls', 'hits'], + sort=sort, + **args) # catch lack of coverage if args.get('error_on_lines') and any( @@ -709,132 +712,133 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find coverage info after running tests.", - allow_abbrev=False) + description="Find coverage info after running tests.", + allow_abbrev=False) parser.add_argument( - 'gcda_paths', - nargs='*', - help="Input *.gcda files.") + 'gcda_paths', + nargs='*', + help="Input *.gcda files.") parser.add_argument( - '-v', '--verbose', - action='store_true', - help="Output commands that run behind the scenes.") + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") parser.add_argument( - '-q', '--quiet', - action='store_true', - help="Don't show anything, useful with -o.") + '-q', '--quiet', + action='store_true', + help="Don't show anything, useful with -o.") parser.add_argument( - '-o', '--output', - help="Specify CSV file to store results.") + '-o', '--output', + help="Specify CSV file to store results.") parser.add_argument( - '-u', '--use', - help="Don't parse anything, use this CSV file.") + '-u', '--use', + help="Don't parse anything, use this CSV file.") parser.add_argument( - '-d', '--diff', - help="Specify CSV file to diff against.") + '-d', '--diff', + help="Specify CSV file to diff against.") parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( - '-p', '--percent', - action='store_true', - help="Only show percentage change, not a full diff.") + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by', - action='append', - choices=CovResult._by, - help="Group by this field.") + '-b', '--by', + action='append', + choices=CovResult._by, + help="Group by this field.") parser.add_argument( - '-f', '--field', - dest='fields', - action='append', - choices=CovResult._fields, - help="Show this field.") + '-f', '--field', + dest='fields', + action='append', + choices=CovResult._fields, + help="Show this field.") parser.add_argument( - '-D', '--define', - dest='defines', - action='append', - type=lambda x: ( - lambda k, vs: ( - k.strip(), - {v.strip() for v in vs.split(',')}) - )(*x.split('=', 1)), - help="Only include results where this field is this value.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: ( + lambda k, vs: ( + k.strip(), + {v.strip() for v in vs.split(',')}) + )(*x.split('=', 1)), + help="Only include results where this field is this value.") class AppendSort(argparse.Action): def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] namespace.sort.append((value, True if option == '-S' else False)) parser.add_argument( - '-s', '--sort', - nargs='?', - action=AppendSort, - help="Sort by this field.") + '-s', '--sort', + nargs='?', + action=AppendSort, + help="Sort by this field.") parser.add_argument( - '-S', '--reverse-sort', - nargs='?', - action=AppendSort, - help="Sort by this field, but backwards.") + '-S', '--reverse-sort', + nargs='?', + action=AppendSort, + help="Sort by this field, but backwards.") parser.add_argument( - '-Y', '--summary', - action='store_true', - help="Only show the total.") + '-Y', '--summary', + action='store_true', + help="Only show the total.") parser.add_argument( - '-F', '--source', - dest='sources', - action='append', - help="Only consider definitions in this file. Defaults to anything " - "in the current directory.") + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to " + "anything in the current directory.") parser.add_argument( - '--everything', - action='store_true', - help="Include builtin and libc specific symbols.") + '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") parser.add_argument( - '--hits', - action='store_true', - help="Show total hits instead of coverage.") + '--hits', + action='store_true', + help="Show total hits instead of coverage.") parser.add_argument( - '-A', '--annotate', - action='store_true', - help="Show source files annotated with coverage info.") + '-A', '--annotate', + action='store_true', + help="Show source files annotated with coverage info.") parser.add_argument( - '-L', '--lines', - action='store_true', - help="Show uncovered lines.") + '-L', '--lines', + action='store_true', + help="Show uncovered lines.") parser.add_argument( - '-B', '--branches', - action='store_true', - help="Show uncovered branches.") + '-B', '--branches', + action='store_true', + help="Show uncovered branches.") parser.add_argument( - '-C', '--context', - type=lambda x: int(x, 0), - default=3, - help="Show n additional lines of context. Defaults to 3.") + '-C', '--context', + type=lambda x: int(x, 0), + default=3, + help="Show n additional lines of context. Defaults to 3.") parser.add_argument( - '-W', '--width', - type=lambda x: int(x, 0), - default=80, - help="Assume source is styled with this many columns. Defaults to 80.") + '-W', '--width', + type=lambda x: int(x, 0), + default=80, + help="Assume source is styled with this many columns. Defaults " + "to 80.") parser.add_argument( - '--color', - choices=['never', 'always', 'auto'], - default='auto', - help="When to use terminal colors. Defaults to 'auto'.") + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( - '-e', '--error-on-lines', - action='store_true', - help="Error if any lines are not covered.") + '-e', '--error-on-lines', + action='store_true', + help="Error if any lines are not covered.") parser.add_argument( - '-E', '--error-on-branches', - action='store_true', - help="Error if any branches are not covered.") + '-E', '--error-on-branches', + action='store_true', + help="Error if any branches are not covered.") parser.add_argument( - '--gcov-path', - default=GCOV_PATH, - type=lambda x: x.split(), - help="Path to the gcov executable, may include paths. " - "Defaults to %r." % GCOV_PATH) + '--gcov-path', + default=GCOV_PATH, + type=lambda x: x.split(), + help="Path to the gcov executable, may include paths. " + "Defaults to %r." % GCOV_PATH) sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/crc32c.py b/scripts/crc32c.py index 4301d1e8..3b99227e 100755 --- a/scripts/crc32c.py +++ b/scripts/crc32c.py @@ -57,24 +57,25 @@ def main(paths, **args): else: print('%08x' % crc) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Calculates crc32cs.", - allow_abbrev=False) + description="Calculates crc32cs.", + allow_abbrev=False) parser.add_argument( - 'paths', - nargs='*', - help="Paths to read. Reads stdin by default.") + 'paths', + nargs='*', + help="Paths to read. Reads stdin by default.") parser.add_argument( - '-x', '--hex', - action='store_true', - help="Interpret as a sequence of hex bytes.") + '-x', '--hex', + action='store_true', + help="Interpret as a sequence of hex bytes.") parser.add_argument( - '-s', '--string', - action='store_true', - help="Interpret as strings.") + '-s', '--string', + action='store_true', + help="Interpret as strings.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/data.py b/scripts/data.py index 26848851..f541b58e 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -115,11 +115,11 @@ class DataResult(co.namedtuple('DataResult', [ __slots__ = () def __new__(cls, file='', function='', size=0): return super().__new__(cls, file, function, - RInt(size)) + RInt(size)) def __add__(self, other): return DataResult(self.file, self.function, - self.size + other.size) + self.size + other.size) def openio(path, mode='r', buffering=-1): @@ -140,18 +140,18 @@ def collect(obj_paths, *, everything=False, **args): size_pattern = re.compile( - '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(nm_types) + - ' (?P.+?)$') + '^(?P[0-9a-fA-F]+)' + + ' (?P[%s])' % re.escape(nm_types) + + ' (?P.+?)$') line_pattern = re.compile( - '^\s+(?P[0-9]+)' - '(?:\s+(?P[0-9]+))?' - '\s+.*' - '\s+(?P[^\s]+)$') + '^\s+(?P[0-9]+)' + '(?:\s+(?P[0-9]+))?' + '\s+.*' + '\s+(?P[^\s]+)$') info_pattern = re.compile( - '^(?:.*(?PDW_TAG_[a-z_]+).*' - '|.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' - '|.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') + '^(?:.*(?PDW_TAG_[a-z_]+).*' + '|.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' + '|.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') results = [] for path in obj_paths: @@ -165,11 +165,11 @@ def collect(obj_paths, *, if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) for line in proc.stdout: m = size_pattern.match(line) if m: @@ -178,8 +178,8 @@ def collect(obj_paths, *, if not everything and func.startswith('__'): continue results_.append(DataResult( - file, func, - int(m.group('size'), 16))) + file, func, + int(m.group('size'), 16))) proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -196,11 +196,11 @@ def collect(obj_paths, *, if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) for line in proc.stdout: # note that files contain references to dirs, which we # dereference as soon as we see them as each file table follows a @@ -215,8 +215,8 @@ def collect(obj_paths, *, dir = int(m.group('dir')) if dir in dirs: files[int(m.group('no'))] = os.path.join( - dirs[dir], - m.group('path')) + dirs[dir], + m.group('path')) else: files[int(m.group('no'))] = m.group('path') proc.wait() @@ -241,11 +241,11 @@ def collect(obj_paths, *, if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) for line in proc.stdout: # state machine here to find definitions m = info_pattern.match(line) @@ -279,17 +279,16 @@ def collect(obj_paths, *, file = defs[r.function] else: _, file = max( - defs.items(), - key=lambda d: difflib.SequenceMatcher(None, - d[0], - r.function, False).ratio()) + defs.items(), + key=lambda d: difflib.SequenceMatcher(None, + d[0], + r.function, False).ratio()) else: file = r.file # ignore filtered sources if sources is not None: - if not any( - os.path.abspath(file) == os.path.abspath(s) + if not any(os.path.abspath(file) == os.path.abspath(s) for s in sources): continue else: @@ -319,7 +318,7 @@ def fold(Result, results, by=None, defines=[]): for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k, - file=sys.stderr) + file=sys.stderr) sys.exit(-1) # filter by matching defines @@ -368,52 +367,55 @@ def table(Result, results, diff_results=None, *, # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r - for r in results} + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r - for r in diff_results or []} + ','.join(str(getattr(r, k) or '') for k in by): r + for r in diff_results or []} names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] + for name in table.keys() | diff_table.keys() + if diff_results is None + or all_ + or any( + types[k].ratio( + getattr(table.get(name), k, None), + getattr(diff_table.get(name), k, None)) + for k in fields)] # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) + names.sort( + key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), + reverse=True) if sort: for k, reverse in reversed(sort): names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None else () - for k in ([k] if k else [ - k for k in Result._sort if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + key=lambda n: tuple( + (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None + else () + for k in ( + [k] if k else [ + k for k in Result._sort + if k in fields])), + reverse=reverse ^ (not k or k in Result._fields)) # build up our lines lines = [] # header - header = [ - '%s%s' % ( - ','.join(by), - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') + header = ['%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') if not summary else ''] if diff_results is None: for k in fields: @@ -436,43 +438,43 @@ def table(Result, results, diff_results=None, *, if diff_results is None: for k in fields: entry.append( - (getattr(r, k).table(), - getattr(getattr(r, k), 'notes', lambda: [])()) - if getattr(r, k, None) is not None - else types[k].none) + (getattr(r, k).table(), + getattr(getattr(r, k), 'notes', lambda: [])()) + if getattr(r, k, None) is not None + else types[k].none) elif percent: for k in fields: entry.append( - (getattr(r, k).table() - if getattr(r, k, None) is not None - else types[k].none, - (lambda t: ['+∞%'] if t == +mt.inf - else ['-∞%'] if t == -mt.inf - else ['%+.1f%%' % (100*t)])( - types[k].ratio( - getattr(r, k, None), - getattr(diff_r, k, None))))) + (getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none, + (lambda t: ['+∞%'] if t == +mt.inf + else ['-∞%'] if t == -mt.inf + else ['%+.1f%%' % (100*t)])( + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None))))) else: for k in fields: entry.append(getattr(diff_r, k).table() - if getattr(diff_r, k, None) is not None - else types[k].none) + if getattr(diff_r, k, None) is not None + else types[k].none) for k in fields: entry.append(getattr(r, k).table() - if getattr(r, k, None) is not None - else types[k].none) + if getattr(r, k, None) is not None + else types[k].none) for k in fields: entry.append( - (types[k].diff( - getattr(r, k, None), - getattr(diff_r, k, None)), - (lambda t: ['+∞%'] if t == +mt.inf - else ['-∞%'] if t == -mt.inf - else ['%+.1f%%' % (100*t)] if t - else [])( - types[k].ratio( + (types[k].diff( getattr(r, k, None), - getattr(diff_r, k, None))))) + getattr(diff_r, k, None)), + (lambda t: ['+∞%'] if t == +mt.inf + else ['-∞%'] if t == -mt.inf + else ['%+.1f%%' % (100*t)] if t + else [])( + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None))))) return entry # entries @@ -495,8 +497,8 @@ def table(Result, results, diff_results=None, *, # homogenize lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + [x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -510,11 +512,11 @@ def table(Result, results, diff_results=None, *, # print our table for line in lines: print('%-*s %s' % ( - widths[0], line[0][0], - ' '.join('%*s%-*s' % ( - widths[i], x[0], - notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') - for i, x in enumerate(line[1:], 1)))) + widths[0], line[0][0], + ' '.join('%*s%-*s' % ( + widths[i], x[0], + notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') + for i, x in enumerate(line[1:], 1)))) def main(obj_paths, *, @@ -537,10 +539,10 @@ def main(obj_paths, *, try: results.append(DataResult( - **{k: r[k] for k in DataResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in DataResult._fields - if k in r and r[k].strip()})) + **{k: r[k] for k in DataResult._by + if k in r and r[k].strip()}, + **{k: r[k] for k in DataResult._fields + if k in r and r[k].strip()})) except TypeError: pass @@ -552,25 +554,27 @@ def main(obj_paths, *, if sort: for k, reverse in reversed(sort): results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else DataResult._sort)), - reverse=reverse ^ (not k or k in DataResult._fields)) + key=lambda r: tuple( + (getattr(r, k),) if getattr(r, k) is not None else () + for k in ([k] if k else DataResult._sort)), + reverse=reverse ^ (not k or k in DataResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: writer = csv.DictWriter(f, - (by if by is not None else DataResult._by) - + [k for k in ( - fields if fields is not None else DataResult._fields)]) + (by if by is not None else DataResult._by) + + [k for k in ( + fields if fields is not None + else DataResult._fields)]) writer.writeheader() for r in results: writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else DataResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None else DataResult._fields)}) + {k: getattr(r, k) for k in ( + by if by is not None else DataResult._by)} + | {k: getattr(r, k) for k in ( + fields if fields is not None + else DataResult._fields)}) # find previous results? if args.get('diff'): @@ -588,10 +592,10 @@ def main(obj_paths, *, continue try: diff_results.append(DataResult( - **{k: r[k] for k in DataResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in DataResult._fields - if k in r and r[k].strip()})) + **{k: r[k] for k in DataResult._by + if k in r and r[k].strip()}, + **{k: r[k] for k in DataResult._fields + if k in r and r[k].strip()})) except TypeError: pass except FileNotFoundError: @@ -603,115 +607,116 @@ def main(obj_paths, *, # print table if not args.get('quiet'): table(DataResult, results, - diff_results if args.get('diff') else None, - by=by if by is not None else ['function'], - fields=fields, - sort=sort, - **args) + diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, + **args) if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find data size at the function level.", - allow_abbrev=False) + description="Find data size at the function level.", + allow_abbrev=False) parser.add_argument( - 'obj_paths', - nargs='*', - help="Input *.o files.") + 'obj_paths', + nargs='*', + help="Input *.o files.") parser.add_argument( - '-v', '--verbose', - action='store_true', - help="Output commands that run behind the scenes.") + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") parser.add_argument( - '-q', '--quiet', - action='store_true', - help="Don't show anything, useful with -o.") + '-q', '--quiet', + action='store_true', + help="Don't show anything, useful with -o.") parser.add_argument( - '-o', '--output', - help="Specify CSV file to store results.") + '-o', '--output', + help="Specify CSV file to store results.") parser.add_argument( - '-u', '--use', - help="Don't parse anything, use this CSV file.") + '-u', '--use', + help="Don't parse anything, use this CSV file.") parser.add_argument( - '-d', '--diff', - help="Specify CSV file to diff against.") + '-d', '--diff', + help="Specify CSV file to diff against.") parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( - '-p', '--percent', - action='store_true', - help="Only show percentage change, not a full diff.") + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by', - action='append', - choices=DataResult._by, - help="Group by this field.") + '-b', '--by', + action='append', + choices=DataResult._by, + help="Group by this field.") parser.add_argument( - '-f', '--field', - dest='fields', - action='append', - choices=DataResult._fields, - help="Show this field.") + '-f', '--field', + dest='fields', + action='append', + choices=DataResult._fields, + help="Show this field.") parser.add_argument( - '-D', '--define', - dest='defines', - action='append', - type=lambda x: ( - lambda k, vs: ( - k.strip(), - {v.strip() for v in vs.split(',')}) - )(*x.split('=', 1)), - help="Only include results where this field is this value.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: ( + lambda k, vs: ( + k.strip(), + {v.strip() for v in vs.split(',')}) + )(*x.split('=', 1)), + help="Only include results where this field is this value.") class AppendSort(argparse.Action): def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] namespace.sort.append((value, True if option == '-S' else False)) parser.add_argument( - '-s', '--sort', - nargs='?', - action=AppendSort, - help="Sort by this field.") + '-s', '--sort', + nargs='?', + action=AppendSort, + help="Sort by this field.") parser.add_argument( - '-S', '--reverse-sort', - nargs='?', - action=AppendSort, - help="Sort by this field, but backwards.") + '-S', '--reverse-sort', + nargs='?', + action=AppendSort, + help="Sort by this field, but backwards.") parser.add_argument( - '-Y', '--summary', - action='store_true', - help="Only show the total.") + '-Y', '--summary', + action='store_true', + help="Only show the total.") parser.add_argument( - '-F', '--source', - dest='sources', - action='append', - help="Only consider definitions in this file. Defaults to anything " - "in the current directory.") + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to " + "anything in the current directory.") parser.add_argument( - '--everything', - action='store_true', - help="Include builtin and libc specific symbols.") + '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") parser.add_argument( - '--nm-types', - default=NM_TYPES, - help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %r." % NM_TYPES) + '--nm-types', + default=NM_TYPES, + help="Type of symbols to report, this uses the same " + "single-character type-names emitted by nm. Defaults to " + "%r." % NM_TYPES) parser.add_argument( - '--nm-path', - type=lambda x: x.split(), - default=NM_PATH, - help="Path to the nm executable, may include flags. " - "Defaults to %r." % NM_PATH) + '--nm-path', + type=lambda x: x.split(), + default=NM_PATH, + help="Path to the nm executable, may include flags. " + "Defaults to %r." % NM_PATH) parser.add_argument( - '--objdump-path', - type=lambda x: x.split(), - default=OBJDUMP_PATH, - help="Path to the objdump executable, may include flags. " - "Defaults to %r." % OBJDUMP_PATH) + '--objdump-path', + type=lambda x: x.split(), + default=OBJDUMP_PATH, + help="Path to the objdump executable, may include flags. " + "Defaults to %r." % OBJDUMP_PATH) sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/dbgblock.py b/scripts/dbgblock.py index a6358345..460d7b6f 100755 --- a/scripts/dbgblock.py +++ b/scripts/dbgblock.py @@ -66,12 +66,12 @@ def rbydaddr(s): def xxd(data, width=16): for i in range(0, len(data), width): yield '%-*s %-*s' % ( - 3*width, - ' '.join('%02x' % b for b in data[i:i+width]), - width, - ''.join( - b if b >= ' ' and b <= '~' else '.' - for b in map(chr, data[i:i+width]))) + 3*width, + ' '.join('%02x' % b for b in data[i:i+width]), + width, + ''.join( + b if b >= ' ' and b <= '~' else '.' + for b in map(chr, data[i:i+width]))) def crc32c(data, crc=0): crc ^= 0xffffffff @@ -99,7 +99,7 @@ def main(disk, block=None, *, if len(block) > 1: print("error: more than one block address?", - file=sys.stderr) + file=sys.stderr) sys.exit(-1) block = block[0] @@ -112,17 +112,18 @@ def main(disk, block=None, *, # block may also encode an offset block, off, size = ( - block[0] if isinstance(block, tuple) else block, - off[0] if isinstance(off, tuple) - else off if off is not None - else size[0] if isinstance(size, tuple) and len(size) > 1 - else block[1] if isinstance(block, tuple) - else None, - size[1] - size[0] if isinstance(size, tuple) and len(size) > 1 - else size[0] if isinstance(size, tuple) - else size if size is not None - else off[1] - off[0] if isinstance(off, tuple) and len(off) > 1 - else block_size) + block[0] if isinstance(block, tuple) else block, + off[0] if isinstance(off, tuple) + else off if off is not None + else size[0] if isinstance(size, tuple) and len(size) > 1 + else block[1] if isinstance(block, tuple) + else None, + size[1] - size[0] if isinstance(size, tuple) and len(size) > 1 + else size[0] if isinstance(size, tuple) + else size if size is not None + else off[1] - off[0] + if isinstance(off, tuple) and len(off) > 1 + else block_size) # read the block f.seek((block * block_size) + (off or 0)) @@ -133,50 +134,51 @@ def main(disk, block=None, *, # print the header print('block %s, size %d, cksum %08x' % ( - '0x%x.%x' % (block, off) - if off is not None - else '0x%x' % block, - size, - cksum)) + '0x%x.%x' % (block, off) + if off is not None + else '0x%x' % block, + size, + cksum)) # render the hex view for o, line in enumerate(xxd(data)): print('%08x: %s' % ((off or 0) + 16*o, line)) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Debug block devices.", - allow_abbrev=False) + description="Debug block devices.", + allow_abbrev=False) parser.add_argument( - 'disk', - help="File containing the block device.") + 'disk', + help="File containing the block device.") parser.add_argument( - 'block', - nargs='?', - type=rbydaddr, - help="Block address.") + 'block', + nargs='?', + type=rbydaddr, + help="Block address.") parser.add_argument( - '-b', '--block-size', - type=bdgeom, - help="Block size/geometry in bytes.") + '-b', '--block-size', + type=bdgeom, + help="Block size/geometry in bytes.") parser.add_argument( - '--block-count', - type=lambda x: int(x, 0), - help="Block count in blocks.") + '--block-count', + type=lambda x: int(x, 0), + help="Block count in blocks.") parser.add_argument( - '--off', - type=lambda x: tuple( - int(x, 0) if x.strip() else None - for x in x.split(',')), - help="Show a specific offset, may be a range.") + '--off', + type=lambda x: tuple( + int(x, 0) if x.strip() else None + for x in x.split(',')), + help="Show a specific offset, may be a range.") parser.add_argument( - '-n', '--size', - type=lambda x: tuple( - int(x, 0) if x.strip() else None - for x in x.split(',')), - help="Show this many bytes, may be a range.") + '-n', '--size', + type=lambda x: tuple( + int(x, 0) if x.strip() else None + for x in x.split(',')), + help="Show this many bytes, may be a range.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/dbgbmap.py b/scripts/dbgbmap.py index 79d10a84..7328d54a 100755 --- a/scripts/dbgbmap.py +++ b/scripts/dbgbmap.py @@ -58,14 +58,14 @@ COLORS = ['33', '34', '32', '90'] CHARS_DOTS = " .':" CHARS_BRAILLE = ( - '⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴' - '⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶' - '⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼' - '⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾' - '⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵' - '⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷' - '⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽' - '⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿') + '⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴' + '⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶' + '⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼' + '⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾' + '⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵' + '⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷' + '⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽' + '⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿') # some ways of block geometry representations @@ -250,8 +250,8 @@ def hilbert_curve(width, height): yield from hilbert_(x, y, b_x_, b_y_, a_x_, a_y_) yield from hilbert_(x+b_x_, y+b_y_, a_x, a_y, b_x-b_x_, b_y-b_y_) yield from hilbert_( - x+(a_x-a_dx)+(b_x_-b_dx), y+(a_y-a_dy)+(b_y_-b_dy), - -b_x_, -b_y_, -(a_x-a_x_), -(a_y-a_y_)) + x+(a_x-a_dx)+(b_x_-b_dx), y+(a_y-a_dy)+(b_y_-b_dy), + -b_x_, -b_y_, -(a_x-a_x_), -(a_y-a_y_)) if width >= height: curve = hilbert_(0, 0, +width, 0, 0, +height) @@ -293,10 +293,10 @@ class Pixel(int): btree=False, data=False): return super().__new__(cls, - state - | (1 if mdir else 0) - | (2 if btree else 0) - | (4 if data else 0)) + state + | (1 if mdir else 0) + | (2 if btree else 0) + | (4 if data else 0)) @property def is_mdir(self): @@ -367,8 +367,8 @@ class Pixel(int): # apply colors if f and color: c = '%s%s\x1b[m' % ( - ''.join('\x1b[%sm' % f_ for f_ in f), - c) + ''.join('\x1b[%sm' % f_ for f_ in f), + c) return c @@ -434,25 +434,25 @@ class Bmap: block -= self._block_window.start size = (max(self._off_window.start, - min(self._off_window.stop, off+size)) - - max(self._off_window.start, - min(self._off_window.stop, off))) + min(self._off_window.stop, off+size)) + - max(self._off_window.start, + min(self._off_window.stop, off))) off = (max(self._off_window.start, - min(self._off_window.stop, off)) - - self._off_window.start) + min(self._off_window.stop, off)) + - self._off_window.start) if size == 0: return # map to our block space range_ = range( - block*len(self._off_window) + off, - block*len(self._off_window) + off+size) + block*len(self._off_window) + off, + block*len(self._off_window) + off+size) range_ = range( - (range_.start*len(self.pixels)) // self._window, - (range_.stop*len(self.pixels)) // self._window) + (range_.start*len(self.pixels)) // self._window, + (range_.stop*len(self.pixels)) // self._window) range_ = range( - range_.start, - max(range_.stop, range_.start+1)) + range_.start, + max(range_.stop, range_.start+1)) # apply the op for i in range_: @@ -476,9 +476,9 @@ class Bmap: width=None, height=None): block_size = (block_size if block_size is not None - else self.block_size) + else self.block_size) block_count = (block_count if block_count is not None - else self.block_count) + else self.block_count) width = width if width is not None else self.width height = height if height is not None else self.height @@ -496,17 +496,17 @@ class Bmap: for x in range(width*height): # map into our old bd space range_ = range( - (x*self._window) // (width*height), - ((x+1)*self._window) // (width*height)) + (x*self._window) // (width*height), + ((x+1)*self._window) // (width*height)) range_ = range( - range_.start, - max(range_.stop, range_.start+1)) + range_.start, + max(range_.stop, range_.start+1)) # aggregate state pixels.append(ft.reduce( - Pixel.__or__, - self.pixels[range_.start:range_.stop], - Pixel())) + Pixel.__or__, + self.pixels[range_.start:range_.stop], + Pixel())) self.width = width self.height = height @@ -552,12 +552,12 @@ class Bmap: byte_p |= 1 << i line.append(best_p.draw( - CHARS_BRAILLE[byte_p], - braille=True, - mdirs=mdirs, - btrees=btrees, - datas=datas, - **args)) + CHARS_BRAILLE[byte_p], + braille=True, + mdirs=mdirs, + btrees=btrees, + datas=datas, + **args)) elif dots: # encode into a byte for x in range(self.width): @@ -572,19 +572,19 @@ class Bmap: byte_p |= 1 << i line.append(best_p.draw( - CHARS_DOTS[byte_p], - dots=True, - mdirs=mdirs, - btrees=btrees, - datas=datas, - **args)) + CHARS_DOTS[byte_p], + dots=True, + mdirs=mdirs, + btrees=btrees, + datas=datas, + **args)) else: for x in range(self.width): line.append(grid[x + row*self.width].draw( - mdirs=mdirs, - btrees=btrees, - datas=datas, - **args)) + mdirs=mdirs, + btrees=btrees, + datas=datas, + **args)) return ''.join(line) @@ -610,9 +610,9 @@ class Rbyd: return '0x%x.%x' % (self.block, self.trunk) else: return '0x{%x,%s}.%x' % ( - self.block, - ','.join('%x' % block for block in self.redund_blocks), - self.trunk) + self.block, + ','.join('%x' % block for block in self.redund_blocks), + self.trunk) @classmethod def fetch(cls, f, block_size, blocks, trunk=None): @@ -621,21 +621,23 @@ class Rbyd: if len(blocks) > 1: # fetch all blocks - rbyds = [cls.fetch(f, block_size, block, trunk) for block in blocks] + rbyds = [cls.fetch(f, block_size, block, trunk) + for block in blocks] # determine most recent revision i = 0 for i_, rbyd in enumerate(rbyds): # compare with sequence arithmetic if rbyd and ( not rbyds[i] - or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) - or (rbyd.rev == rbyds[i].rev - and rbyd.trunk > rbyds[i].trunk)): + or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) + or (rbyd.rev == rbyds[i].rev + and rbyd.trunk > rbyds[i].trunk)): i = i_ # keep track of the other blocks rbyd = rbyds[i] - rbyd.redund_blocks = [rbyds[(i+1+j) % len(rbyds)].block - for j in range(len(rbyds)-1)] + rbyd.redund_blocks = [ + rbyds[(i+1+j) % len(rbyds)].block + for j in range(len(rbyds)-1)] return rbyd else: # block may encode a trunk @@ -789,7 +791,9 @@ class Rbyd: done = not tag_ or (rid_, tag_) < (rid, tag) - return done, rid_, tag_, w_, j, d, self.data[j+d:j+d+jump], path + return (done, rid_, tag_, w_, j, d, + self.data[j+d:j+d+jump], + path) def __bool__(self): return bool(self.trunk) @@ -834,7 +838,7 @@ class Rbyd: w = 0 for i in it.count(): done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( - rid_, tag+0x1) + rid_, tag+0x1) if done or (i != 0 and rid__ != rid_): break @@ -880,14 +884,15 @@ class Rbyd: # lookup our mbid done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid) + f, block_size, mbid) if done: return True, -1, 0, None - mdir = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) + mdir = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) if not mdir: return True, -1, 0, None @@ -979,7 +984,7 @@ def main(disk, mroots=None, *, if any(isinstance(b, list) and len(b) > 1 for b in block): print("error: more than one block address?", - file=sys.stderr) + file=sys.stderr) sys.exit(-1) if isinstance(block[0], list): block = (block[0][0], *block[1:]) @@ -1034,16 +1039,17 @@ def main(disk, mroots=None, *, # create our block device representation bmap = Bmap( - block_size=block_size, - block_count=block_count, - block_window=block_window, - off_window=off_window, - # scale if we're printing with dots or braille - width=2*width_ if braille else width_, - height=max(1, - 4*height_ if braille - else 2*height_ if dots - else height_)) + block_size=block_size, + block_count=block_count, + block_window=block_window, + off_window=off_window, + # scale if we're printing with dots or braille + width=2*width_ if braille else width_, + height=max( + 1, + 4*height_ if braille + else 2*height_ if dots + else height_)) # keep track of how many blocks are in use mdirs_ = 0 @@ -1063,16 +1069,16 @@ def main(disk, mroots=None, *, block_size = f.tell() block_count = 1 bmap.resize( - block_size=block_size, - block_count=block_count) + block_size=block_size, + block_count=block_count) # if block_count is omitted, derive the block_count from our file size if block_count is None: f.seek(0, os.SEEK_END) block_count = f.tell() // block_size bmap.resize( - block_size=block_size, - block_count=block_count) + block_size=block_size, + block_count=block_count) #### traverse the filesystem @@ -1090,7 +1096,7 @@ def main(disk, mroots=None, *, # mark mroots in our bmap for block in mroot.blocks: bmap.mdir(block, - mroot.eoff if args.get('in_use') else block_size) + mroot.eoff if args.get('in_use') else block_size) mdirs_ += 1; # find any file btrees in our mroot @@ -1129,7 +1135,8 @@ def main(disk, mroots=None, *, # mark mdir in our bmap for block in mdir.blocks: bmap.mdir(block, - mdir.eoff if args.get('in_use') else block_size) + mdir.eoff if args.get('in_use') + else block_size) mdirs_ += 1 # find any file btrees in our mdir @@ -1153,8 +1160,8 @@ def main(disk, mroots=None, *, ppath = [] while True: done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth) + f, block_size, mbid+1, + depth=args.get('depth', mdepth)-mdepth) if done: break @@ -1176,8 +1183,8 @@ def main(disk, mroots=None, *, d, (mid_, w_, rbyd_, rid_, tags_) = x for block in rbyd_.blocks: bmap.btree(block, - rbyd_.eoff if args.get('in_use') - else block_size) + rbyd_.eoff if args.get('in_use') + else block_size) btrees_ += 1 ppath = path @@ -1190,10 +1197,11 @@ def main(disk, mroots=None, *, mdir__ = None if (not args.get('depth') or mdepth+len(path) < args.get('depth')): - mdir__ = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) + mdir__ = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) if mdir__: # fetch the mdir @@ -1208,8 +1216,8 @@ def main(disk, mroots=None, *, # mark mdir in our bmap for block in mdir_.blocks: bmap.mdir(block, 0, - mdir_.eoff if args.get('in_use') - else block_size) + mdir_.eoff if args.get('in_use') + else block_size) mdirs_ += 1 # find any file btrees in our mdir @@ -1233,8 +1241,8 @@ def main(disk, mroots=None, *, size, block, off = frombptr(data) # mark block in our bmap bmap.data(block, - off if args.get('in_use') else 0, - size if args.get('in_use') else block_size) + off if args.get('in_use') else 0, + size if args.get('in_use') else block_size) datas_ += 1 continue @@ -1258,9 +1266,9 @@ def main(disk, mroots=None, *, ppath = [] while True: (done, bid, w, rbyd, rid, tags, path - ) = btree.btree_lookup( - f, block_size, bid+1, - depth=args.get('depth', mdepth)-mdepth) + ) = btree.btree_lookup( + f, block_size, bid+1, + depth=args.get('depth', mdepth)-mdepth) if done: break @@ -1285,8 +1293,8 @@ def main(disk, mroots=None, *, continue for block in rbyd_.blocks: bmap.btree(block, - rbyd_.eoff if args.get('in_use') - else block_size) + rbyd_.eoff if args.get('in_use') + else block_size) btrees_ += 1 ppath = path @@ -1299,10 +1307,11 @@ def main(disk, mroots=None, *, bptr__ = None if (not args.get('depth') or mdepth+len(path) < args.get('depth')): - bptr__ = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag & 0xfff == TAG_BLOCK), - None) + bptr__ = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag & 0xfff == TAG_BLOCK), + None) if bptr__: # fetch the block @@ -1311,8 +1320,8 @@ def main(disk, mroots=None, *, # mark blocks in our bmap bmap.data(block, - off if args.get('in_use') else 0, - size if args.get('in_use') else block_size) + off if args.get('in_use') else 0, + size if args.get('in_use') else block_size) datas_ += 1 #### actual rendering begins here @@ -1320,29 +1329,29 @@ def main(disk, mroots=None, *, # print some information about the bmap if not no_header: print('bd %dx%d%s%s%s' % ( - block_size, block_count, - ', %6s mdir' % ('%.1f%%' % (100*mdirs_ / block_count)) - if mdirs else '', - ', %6s btree' % ('%.1f%%' % (100*btrees_ / block_count)) - if btrees else '', - ', %6s data' % ('%.1f%%' % (100*datas_ / block_count)) - if datas else '')) + block_size, block_count, + ', %6s mdir' % ('%.1f%%' % (100*mdirs_ / block_count)) + if mdirs else '', + ', %6s btree' % ('%.1f%%' % (100*btrees_ / block_count)) + if btrees else '', + ', %6s data' % ('%.1f%%' % (100*datas_ / block_count)) + if datas else '')) # and then print the bmap for row in range( mt.ceil(bmap.height/4) if braille - else mt.ceil(bmap.height/2) if dots - else bmap.height): + else mt.ceil(bmap.height/2) if dots + else bmap.height): line = bmap.draw(row, - mdirs=mdirs, - btrees=btrees, - datas=datas, - color=color, - dots=dots, - braille=braille, - hilbert=hilbert, - lebesgue=lebesgue, - **args) + mdirs=mdirs, + btrees=btrees, + datas=datas, + color=color, + dots=dots, + braille=braille, + hilbert=hilbert, + lebesgue=lebesgue, + **args) print(line) if args.get('error_on_corrupt') and corrupted: @@ -1353,122 +1362,122 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Render currently used blocks in a littlefs image.", - allow_abbrev=False) + description="Render currently used blocks in a littlefs image.", + allow_abbrev=False) parser.add_argument( - 'disk', - help="File containing the block device.") + 'disk', + help="File containing the block device.") parser.add_argument( - 'mroots', - nargs='*', - type=rbydaddr, - help="Block address of the mroots. Defaults to 0x{0,1}.") + 'mroots', + nargs='*', + type=rbydaddr, + help="Block address of the mroots. Defaults to 0x{0,1}.") parser.add_argument( - '-b', '--block-size', - type=bdgeom, - help="Block size/geometry in bytes.") + '-b', '--block-size', + type=bdgeom, + help="Block size/geometry in bytes.") parser.add_argument( - '--block-count', - type=lambda x: int(x, 0), - help="Block count in blocks.") + '--block-count', + type=lambda x: int(x, 0), + help="Block count in blocks.") parser.add_argument( - '-@', '--block', - nargs='?', - type=lambda x: tuple( - rbydaddr(x) if x.strip() else None - for x in x.split(',')), - help="Optional block to show, may be a range.") + '-@', '--block', + nargs='?', + type=lambda x: tuple( + rbydaddr(x) if x.strip() else None + for x in x.split(',')), + help="Optional block to show, may be a range.") parser.add_argument( - '--off', - type=lambda x: tuple( - int(x, 0) if x.strip() else None - for x in x.split(',')), - help="Show a specific offset, may be a range.") + '--off', + type=lambda x: tuple( + int(x, 0) if x.strip() else None + for x in x.split(',')), + help="Show a specific offset, may be a range.") parser.add_argument( - '--size', - type=lambda x: tuple( - int(x, 0) if x.strip() else None - for x in x.split(',')), - help="Show this many bytes, may be a range.") + '--size', + type=lambda x: tuple( + int(x, 0) if x.strip() else None + for x in x.split(',')), + help="Show this many bytes, may be a range.") parser.add_argument( - '-M', '--mdirs', - action='store_true', - help="Render mdir blocks.") + '-M', '--mdirs', + action='store_true', + help="Render mdir blocks.") parser.add_argument( - '-B', '--btrees', - action='store_true', - help="Render btree blocks.") + '-B', '--btrees', + action='store_true', + help="Render btree blocks.") parser.add_argument( - '-D', '--datas', - action='store_true', - help="Render data blocks.") + '-D', '--datas', + action='store_true', + help="Render data blocks.") parser.add_argument( - '-N', '--no-header', - action='store_true', - help="Don't show the header.") + '-N', '--no-header', + action='store_true', + help="Don't show the header.") parser.add_argument( - '--color', - choices=['never', 'always', 'auto'], - default='auto', - help="When to use terminal colors. Defaults to 'auto'.") + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( - '-:', '--dots', - action='store_true', - help="Use 1x2 ascii dot characters.") + '-:', '--dots', + action='store_true', + help="Use 1x2 ascii dot characters.") parser.add_argument( - '-⣿', '--braille', - action='store_true', - help="Use 2x4 unicode braille characters. Note that braille characters " - "sometimes suffer from inconsistent widths.") + '-⣿', '--braille', + action='store_true', + help="Use 2x4 unicode braille characters. Note that braille " + "characters sometimes suffer from inconsistent widths.") parser.add_argument( - '--chars', - help="Characters to use for mdir, btree, data, unused blocks.") + '--chars', + help="Characters to use for mdir, btree, data, unused blocks.") parser.add_argument( - '--colors', - type=lambda x: [x.strip() for x in x.split(',')], - help="Colors to use for mdir, btree, data, unused blocks.") + '--colors', + type=lambda x: [x.strip() for x in x.split(',')], + help="Colors to use for mdir, btree, data, unused blocks.") parser.add_argument( - '-W', '--width', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Width in columns. 0 uses the terminal width. Defaults to " - "min(terminal, 80).") + '-W', '--width', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Width in columns. 0 uses the terminal width. Defaults to " + "min(terminal, 80).") parser.add_argument( - '-H', '--height', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Height in rows. 0 uses the terminal height. Defaults to 1.") + '-H', '--height', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Height in rows. 0 uses the terminal height. Defaults to 1.") parser.add_argument( - '-n', '--lines', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Show this many lines of history. 0 uses the terminal height. " - "Defaults to 5.") + '-n', '--lines', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Show this many lines of history. 0 uses the terminal " + "height. Defaults to 5.") parser.add_argument( - '-U', '--hilbert', - action='store_true', - help="Render as a space-filling Hilbert curve.") + '-U', '--hilbert', + action='store_true', + help="Render as a space-filling Hilbert curve.") parser.add_argument( - '-Z', '--lebesgue', - action='store_true', - help="Render as a space-filling Z-curve.") + '-Z', '--lebesgue', + action='store_true', + help="Render as a space-filling Z-curve.") parser.add_argument( - '-i', '--in-use', - action='store_true', - help="Show how much of each block is in use.") + '-i', '--in-use', + action='store_true', + help="Show how much of each block is in use.") parser.add_argument( - '-z', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of the filesystem tree to parse.") + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of the filesystem tree to parse.") parser.add_argument( - '-e', '--error-on-corrupt', - action='store_true', - help="Error if the filesystem is corrupt.") + '-e', '--error-on-corrupt', + action='store_true', + help="Error if the filesystem is corrupt.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 117f8295..cb79a7d4 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -154,108 +154,108 @@ def frombranch(data): def xxd(data, width=16): for i in range(0, len(data), width): yield '%-*s %-*s' % ( - 3*width, - ' '.join('%02x' % b for b in data[i:i+width]), - width, - ''.join( - b if b >= ' ' and b <= '~' else '.' - for b in map(chr, data[i:i+width]))) + 3*width, + ' '.join('%02x' % b for b in data[i:i+width]), + width, + ''.join( + b if b >= ' ' and b <= '~' else '.' + for b in map(chr, data[i:i+width]))) def tagrepr(tag, w=None, size=None, off=None): if (tag & 0x6fff) == TAG_NULL: return '%snull%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - ' w%d' % w if w else '', - ' %d' % size if size else '') + 'shrub' if tag & TAG_SHRUB else '', + ' w%d' % w if w else '', + ' %d' % size if size else '') elif (tag & 0x6f00) == TAG_CONFIG: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'magic' if (tag & 0xfff) == TAG_MAGIC - else 'version' if (tag & 0xfff) == TAG_VERSION - else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT - else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT - else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT - else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY - else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT - else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT - else 'config 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'magic' if (tag & 0xfff) == TAG_MAGIC + else 'version' if (tag & 0xfff) == TAG_VERSION + else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT + else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT + else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT + else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY + else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT + else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT + else 'config 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_GDELTA: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA - else 'gdelta 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gdelta 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_NAME: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'name' if (tag & 0xfff) == TAG_NAME - else 'reg' if (tag & 0xfff) == TAG_REG - else 'dir' if (tag & 0xfff) == TAG_DIR - else 'orphan' if (tag & 0xfff) == TAG_ORPHAN - else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK - else 'name 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'name' if (tag & 0xfff) == TAG_NAME + else 'reg' if (tag & 0xfff) == TAG_REG + else 'dir' if (tag & 0xfff) == TAG_DIR + else 'orphan' if (tag & 0xfff) == TAG_ORPHAN + else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK + else 'name 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_STRUCT: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'data' if (tag & 0xfff) == TAG_DATA - else 'block' if (tag & 0xfff) == TAG_BLOCK - else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB - else 'btree' if (tag & 0xfff) == TAG_BTREE - else 'mroot' if (tag & 0xfff) == TAG_MROOT - else 'mdir' if (tag & 0xfff) == TAG_MDIR - else 'mtree' if (tag & 0xfff) == TAG_MTREE - else 'did' if (tag & 0xfff) == TAG_DID - else 'branch' if (tag & 0xfff) == TAG_BRANCH - else 'struct 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'data' if (tag & 0xfff) == TAG_DATA + else 'block' if (tag & 0xfff) == TAG_BLOCK + else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB + else 'btree' if (tag & 0xfff) == TAG_BTREE + else 'mroot' if (tag & 0xfff) == TAG_MROOT + else 'mdir' if (tag & 0xfff) == TAG_MDIR + else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'did' if (tag & 0xfff) == TAG_DID + else 'branch' if (tag & 0xfff) == TAG_BRANCH + else 'struct 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6e00) == TAG_ATTR: return '%s%sattr 0x%02x%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 's' if tag & 0x100 else 'u', - ((tag & 0x100) >> 1) ^ (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 's' if tag & 0x100 else 'u', + ((tag & 0x100) >> 1) ^ (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( - 'r' if tag & TAG_R else 'b', - 'a' if tag & 0x0fff == 0 and tag & TAG_GT - else 'n' if tag & 0x0fff == 0 - else 'gt' if tag & TAG_GT - else 'le', - ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', - ' w%d' % w if w is not None else '', - ' 0x%x' % (0xffffffff & (off-size)) - if size and off is not None - else ' -%d' % size if size - else '') + 'r' if tag & TAG_R else 'b', + 'a' if tag & 0x0fff == 0 and tag & TAG_GT + else 'n' if tag & 0x0fff == 0 + else 'gt' if tag & TAG_GT + else 'le', + ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', + ' w%d' % w if w is not None else '', + ' 0x%x' % (0xffffffff & (off-size)) + if size and off is not None + else ' -%d' % size if size + else '') elif (tag & 0x7f00) == TAG_CKSUM: return 'cksum%s%s%s%s%s' % ( - 'q' if not tag & 0xfc and tag & TAG_Q else '', - 'p' if not tag & 0xfc and tag & TAG_P else '', - ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'q' if not tag & 0xfc and tag & TAG_Q else '', + 'p' if not tag & 0xfc and tag & TAG_P else '', + ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_NOTE: return 'note%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_ECKSUM: return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( - tag, - ' w%d' % w if w is not None else '', - ' %d' % size if size is not None else '') + tag, + ' w%d' % w if w is not None else '', + ' %d' % size if size is not None else '') # this type is used for tree representations @@ -278,9 +278,9 @@ class Rbyd: return '0x%x.%x' % (self.block, self.trunk) else: return '0x{%x,%s}.%x' % ( - self.block, - ','.join('%x' % block for block in self.redund_blocks), - self.trunk) + self.block, + ','.join('%x' % block for block in self.redund_blocks), + self.trunk) @classmethod def fetch(cls, f, block_size, blocks, trunk=None): @@ -289,21 +289,23 @@ class Rbyd: if len(blocks) > 1: # fetch all blocks - rbyds = [cls.fetch(f, block_size, block, trunk) for block in blocks] + rbyds = [cls.fetch(f, block_size, block, trunk) + for block in blocks] # determine most recent revision i = 0 for i_, rbyd in enumerate(rbyds): # compare with sequence arithmetic if rbyd and ( not rbyds[i] - or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) - or (rbyd.rev == rbyds[i].rev - and rbyd.trunk > rbyds[i].trunk)): + or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) + or (rbyd.rev == rbyds[i].rev + and rbyd.trunk > rbyds[i].trunk)): i = i_ # keep track of the other blocks rbyd = rbyds[i] - rbyd.redund_blocks = [rbyds[(i+1+j) % len(rbyds)].block - for j in range(len(rbyds)-1)] + rbyd.redund_blocks = [ + rbyds[(i+1+j) % len(rbyds)].block + for j in range(len(rbyds)-1)] return rbyd else: # block may encode a trunk @@ -457,7 +459,9 @@ class Rbyd: done = not tag_ or (rid_, tag_) < (rid, tag) - return done, rid_, tag_, w_, j, d, self.data[j+d:j+d+jump], path + return (done, rid_, tag_, w_, j, d, + self.data[j+d:j+d+jump], + path) def __bool__(self): return bool(self.trunk) @@ -549,8 +553,8 @@ class Rbyd: else: if 'h' not in alts[j_]: alts[j_]['h'] = max( - rec_height(alts[j_]['f']), - rec_height(alts[j_]['nf'])) + 1 + rec_height(alts[j_]['f']), + rec_height(alts[j_]['nf'])) + 1 return alts[j_]['h'] for j_ in alts.keys(): @@ -614,10 +618,10 @@ def main(disk, roots=None, *, # fetch the root btree = Rbyd.fetch(f, block_size, roots, trunk) print('btree %s w%d, rev %08x, cksum %08x' % ( - btree.addr(), - btree.weight, - btree.rev, - btree.cksum)) + btree.addr(), + btree.weight, + btree.rev, + btree.cksum)) # look up a bid, while keeping track of the search path def btree_lookup(bid, *, @@ -642,7 +646,7 @@ def main(disk, roots=None, *, w = 0 for i in it.count(): done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( - rid_, tag+0x1) + rid_, tag+0x1) if done or (i != 0 and rid__ != rid_): break @@ -683,7 +687,7 @@ def main(disk, roots=None, *, bid = -1 while True: done, bid, w, rbyd, rid, tags, path = btree_lookup( - bid+1, depth=args.get('depth')) + bid+1, depth=args.get('depth')) if done: break @@ -698,7 +702,7 @@ def main(disk, roots=None, *, bid = -1 while True: done, bid, w, rbyd, rid, tags, path = btree_lookup( - bid+1, depth=args.get('depth')) + bid+1, depth=args.get('depth')) if done: break @@ -731,8 +735,8 @@ def main(disk, roots=None, *, # connect our branch to the rbyd's root if leaf is not None: root = min(rtree, - key=lambda branch: branch.d, - default=None) + key=lambda branch: branch.d, + default=None) if root is not None: r_rid, r_tag = root.a @@ -758,9 +762,10 @@ def main(disk, roots=None, *, d_ += max(bdepths.get(d, 0), 1) leaf = (bid-(w-1), d, rid-(w-1), - next((tag for tag, _, _, _ in tags - if tag & 0xfff == TAG_BRANCH), - TAG_BRANCH)) + next( + (tag for tag, _, _, _ in tags + if tag & 0xfff == TAG_BRANCH), + TAG_BRANCH)) # remap branches to leaves if we aren't showing inner branches if not args.get('inner'): @@ -813,7 +818,7 @@ def main(disk, roots=None, *, bid = -1 while True: done, bid, w, rbyd, rid, tags, path = btree_lookup( - bid+1, depth=args.get('depth')) + bid+1, depth=args.get('depth')) if done: break @@ -836,7 +841,7 @@ def main(disk, roots=None, *, continue b = (bid-(w-1), d, rid-(w-1), - (name if name else tags[0])[0]) + (name if name else tags[0])[0]) # remap branches to leaves if we aren't showing # inner branches @@ -846,8 +851,8 @@ def main(disk, roots=None, *, if not tags: continue branches[b] = ( - bid-(w-1), len(path)-1, rid-(w-1), - (name if name else tags[0])[0]) + bid-(w-1), len(path)-1, rid-(w-1), + (name if name else tags[0])[0]) b = branches[b] # found entry point? @@ -905,16 +910,16 @@ def main(disk, roots=None, *, was = None for d in range(t_depth): t, c, was = branchrepr( - (bid-(w-1), bd, rid-(w-1), tag), d, was) + (bid-(w-1), bd, rid-(w-1), tag), d, was) trunk.append('%s%s%s%s' % ( - '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color and c == 'b' - else '', - t, - ('>' if was else ' ') if d == t_depth-1 else '', - '\x1b[m' if color and c else '')) + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == t_depth-1 else '', + '\x1b[m' if color and c else '')) return '%s ' % ''.join(trunk) @@ -931,39 +936,41 @@ def main(disk, roots=None, *, # show human-readable representation for i, (tag, j, d, data) in enumerate(tags): print('%10s %s%*s %-*s %s' % ( - '%04x.%04x:' % (rbyd.block, rbyd.trunk) - if prbyd is None or rbyd != prbyd - else '', - treerepr(bid, w, bd, rid, tag) - if args.get('tree') - or args.get('rbyd') - or args.get('btree') else '', - 2*w_width+1, '' if i != 0 - else '%d-%d' % (bid-(w-1), bid) if w > 1 - else bid if w > 0 - else '', - 21+w_width, tagrepr( - tag, w if i == 0 else 0, len(data), None), - next(xxd(data, 8), '') - if not args.get('raw') and not args.get('no_truncate') - else '')) + '%04x.%04x:' % (rbyd.block, rbyd.trunk) + if prbyd is None or rbyd != prbyd + else '', + treerepr(bid, w, bd, rid, tag) + if args.get('tree') + or args.get('rbyd') + or args.get('btree') + else '', + 2*w_width+1, '' if i != 0 + else '%d-%d' % (bid-(w-1), bid) if w > 1 + else bid if w > 0 + else '', + 21+w_width, tagrepr( + tag, w if i == 0 else 0, len(data), None), + next(xxd(data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else '')) prbyd = rbyd # show on-disk encoding of tags/data if args.get('raw'): for o, line in enumerate(xxd(rbyd.data[j:j+d])): print('%9s: %*s%*s %s' % ( - '%04x' % (j + o*16), - t_width, '', - 2*w_width+1, '', - line)) + '%04x' % (j + o*16), + t_width, '', + 2*w_width+1, '', + line)) if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data)): print('%9s: %*s%*s %s' % ( - '%04x' % (j+d + o*16), - t_width, '', - 2*w_width+1, '', - line)) + '%04x' % (j+d + o*16), + t_width, '', + 2*w_width+1, '', + line)) # traverse and print entries @@ -973,7 +980,7 @@ def main(disk, roots=None, *, corrupted = False while True: done, bid, w, rbyd, rid, tags, path = btree_lookup( - bid+1, depth=args.get('depth')) + bid+1, depth=args.get('depth')) if done: break @@ -997,11 +1004,11 @@ def main(disk, roots=None, *, # corrupted? try to keep printing the tree if not rbyd: print('%04x.%04x: %*s%s%s%s' % ( - rbyd.block, rbyd.trunk, - t_width, '', - '\x1b[31m' if color else '', - '(corrupted rbyd %s)' % rbyd.addr(), - '\x1b[m' if color else '')) + rbyd.block, rbyd.trunk, + t_width, '', + '\x1b[31m' if color else '', + '(corrupted rbyd %s)' % rbyd.addr(), + '\x1b[m' if color else '')) prbyd = rbyd corrupted = True continue @@ -1020,8 +1027,8 @@ def main(disk, roots=None, *, if name is not None: tags = [name] + [(tag, j, d, data) - for tag, j, d, data in tags - if tag & 0x7f00 != TAG_NAME] + for tag, j, d, data in tags + if tag & 0x7f00 != TAG_NAME] # show the branch dbg_branch(bid, w, rbyd, rid, tags, len(path)-1) @@ -1034,67 +1041,67 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Debug rbyd B-trees.", - allow_abbrev=False) + description="Debug rbyd B-trees.", + allow_abbrev=False) parser.add_argument( - 'disk', - help="File containing the block device.") + 'disk', + help="File containing the block device.") parser.add_argument( - 'roots', - nargs='*', - type=rbydaddr, - help="Block address of the roots of the tree.") + 'roots', + nargs='*', + type=rbydaddr, + help="Block address of the roots of the tree.") parser.add_argument( - '-b', '--block-size', - type=bdgeom, - help="Block size/geometry in bytes.") + '-b', '--block-size', + type=bdgeom, + help="Block size/geometry in bytes.") parser.add_argument( - '--block-count', - type=lambda x: int(x, 0), - help="Block count in blocks.") + '--block-count', + type=lambda x: int(x, 0), + help="Block count in blocks.") parser.add_argument( - '--trunk', - type=lambda x: int(x, 0), - help="Use this offset as the trunk of the tree.") + '--trunk', + type=lambda x: int(x, 0), + help="Use this offset as the trunk of the tree.") parser.add_argument( - '--color', - choices=['never', 'always', 'auto'], - default='auto', - help="When to use terminal colors. Defaults to 'auto'.") + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( - '-r', '--raw', - action='store_true', - help="Show the raw data including tag encodings.") + '-r', '--raw', + action='store_true', + help="Show the raw data including tag encodings.") parser.add_argument( - '-T', '--no-truncate', - action='store_true', - help="Don't truncate, show the full contents.") + '-T', '--no-truncate', + action='store_true', + help="Don't truncate, show the full contents.") parser.add_argument( - '-t', '--tree', - action='store_true', - help="Show the underlying rbyd trees.") + '-t', '--tree', + action='store_true', + help="Show the underlying rbyd trees.") parser.add_argument( - '-B', '--btree', - action='store_true', - help="Show the B-tree.") + '-B', '--btree', + action='store_true', + help="Show the B-tree.") parser.add_argument( - '-R', '--rbyd', - action='store_true', - help="Show the full underlying rbyd trees.") + '-R', '--rbyd', + action='store_true', + help="Show the full underlying rbyd trees.") parser.add_argument( - '-i', '--inner', - action='store_true', - help="Show inner branches.") + '-i', '--inner', + action='store_true', + help="Show inner branches.") parser.add_argument( - '-z', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of tree to show.") + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of tree to show.") parser.add_argument( - '-e', '--error-on-corrupt', - action='store_true', - help="Error if B-tree is corrupt.") + '-e', '--error-on-corrupt', + action='store_true', + help="Error if B-tree is corrupt.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/dbgcat.py b/scripts/dbgcat.py index abbeb1b9..d88fa8c3 100755 --- a/scripts/dbgcat.py +++ b/scripts/dbgcat.py @@ -66,12 +66,12 @@ def rbydaddr(s): def xxd(data, width=16): for i in range(0, len(data), width): yield '%-*s %-*s' % ( - 3*width, - ' '.join('%02x' % b for b in data[i:i+width]), - width, - ''.join( - b if b >= ' ' and b <= '~' else '.' - for b in map(chr, data[i:i+width]))) + 3*width, + ' '.join('%02x' % b for b in data[i:i+width]), + width, + ''.join( + b if b >= ' ' and b <= '~' else '.' + for b in map(chr, data[i:i+width]))) def crc32c(data, crc=0): crc ^= 0xffffffff @@ -105,19 +105,21 @@ def main(disk, blocks=None, *, # blocks may also encode offsets blocks, offs, size = ( - [block[0] if isinstance(block, tuple) else block - for block in blocks], - [off[0] if isinstance(off, tuple) - else off if off is not None - else size[0] if isinstance(size, tuple) and len(size) > 1 - else block[1] if isinstance(block, tuple) - else None - for block in blocks], - size[1] - size[0] if isinstance(size, tuple) and len(size) > 1 - else size[0] if isinstance(size, tuple) - else size if size is not None - else off[1] - off[0] if isinstance(off, tuple) and len(off) > 1 - else block_size) + [block[0] if isinstance(block, tuple) else block + for block in blocks], + [off[0] if isinstance(off, tuple) + else off if off is not None + else size[0] + if isinstance(size, tuple) and len(size) > 1 + else block[1] if isinstance(block, tuple) + else None + for block in blocks], + size[1] - size[0] if isinstance(size, tuple) and len(size) > 1 + else size[0] if isinstance(size, tuple) + else size if size is not None + else off[1] - off[0] + if isinstance(off, tuple) and len(off) > 1 + else block_size) # cat the blocks for block, off in zip(blocks, offs): @@ -126,40 +128,41 @@ def main(disk, blocks=None, *, sys.stdout.buffer.write(data) sys.stdout.flush() + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Cat data from a block device.", - allow_abbrev=False) + description="Cat data from a block device.", + allow_abbrev=False) parser.add_argument( - 'disk', - help="File containing the block device.") + 'disk', + help="File containing the block device.") parser.add_argument( - 'blocks', - nargs='*', - type=rbydaddr, - help="Block address.") + 'blocks', + nargs='*', + type=rbydaddr, + help="Block address.") parser.add_argument( - '-b', '--block-size', - type=bdgeom, - help="Block size/geometry in bytes.") + '-b', '--block-size', + type=bdgeom, + help="Block size/geometry in bytes.") parser.add_argument( - '--block-count', - type=lambda x: int(x, 0), - help="Block count in blocks.") + '--block-count', + type=lambda x: int(x, 0), + help="Block count in blocks.") parser.add_argument( - '--off', - type=lambda x: tuple( - int(x, 0) if x.strip() else None - for x in x.split(',')), - help="Show a specific offset, may be a range.") + '--off', + type=lambda x: tuple( + int(x, 0) if x.strip() else None + for x in x.split(',')), + help="Show a specific offset, may be a range.") parser.add_argument( - '-n', '--size', - type=lambda x: tuple( - int(x, 0) if x.strip() else None - for x in x.split(',')), - help="Show this many bytes, may be a range.") + '-n', '--size', + type=lambda x: tuple( + int(x, 0) if x.strip() else None + for x in x.split(',')), + help="Show this many bytes, may be a range.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/dbgerr.py b/scripts/dbgerr.py index 216222cb..3a645b9e 100755 --- a/scripts/dbgerr.py +++ b/scripts/dbgerr.py @@ -37,9 +37,9 @@ def main(errs, *, # print for n, e, h in ERRS: print('%-*s %-*s %s' % ( - w[0], 'LFS_ERR_'+n, - w[1], e, - h)) + w[0], 'LFS_ERR_'+n, + w[1], e, + h)) # find these errors else: @@ -77,20 +77,21 @@ def main(errs, *, except KeyError: print('%s ?' % err) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Decode littlefs error codes.", - allow_abbrev=False) + description="Decode littlefs error codes.", + allow_abbrev=False) parser.add_argument( - 'errs', - nargs='*', - help="Error codes or error names to decode.") + 'errs', + nargs='*', + help="Error codes or error names to decode.") parser.add_argument( - '-l', '--list', - action='store_true', - help="List all known error codes.") + '-l', '--list', + action='store_true', + help="List all known error codes.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/dbglfs.py b/scripts/dbglfs.py index 6e90077a..f277c2bc 100755 --- a/scripts/dbglfs.py +++ b/scripts/dbglfs.py @@ -185,108 +185,108 @@ def frombptr(data): def xxd(data, width=16): for i in range(0, len(data), width): yield '%-*s %-*s' % ( - 3*width, - ' '.join('%02x' % b for b in data[i:i+width]), - width, - ''.join( - b if b >= ' ' and b <= '~' else '.' - for b in map(chr, data[i:i+width]))) + 3*width, + ' '.join('%02x' % b for b in data[i:i+width]), + width, + ''.join( + b if b >= ' ' and b <= '~' else '.' + for b in map(chr, data[i:i+width]))) def tagrepr(tag, w=None, size=None, off=None): if (tag & 0x6fff) == TAG_NULL: return '%snull%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - ' w%d' % w if w else '', - ' %d' % size if size else '') + 'shrub' if tag & TAG_SHRUB else '', + ' w%d' % w if w else '', + ' %d' % size if size else '') elif (tag & 0x6f00) == TAG_CONFIG: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'magic' if (tag & 0xfff) == TAG_MAGIC - else 'version' if (tag & 0xfff) == TAG_VERSION - else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT - else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT - else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT - else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY - else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT - else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT - else 'config 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'magic' if (tag & 0xfff) == TAG_MAGIC + else 'version' if (tag & 0xfff) == TAG_VERSION + else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT + else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT + else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT + else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY + else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT + else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT + else 'config 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_GDELTA: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA - else 'gdelta 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gdelta 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_NAME: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'name' if (tag & 0xfff) == TAG_NAME - else 'reg' if (tag & 0xfff) == TAG_REG - else 'dir' if (tag & 0xfff) == TAG_DIR - else 'orphan' if (tag & 0xfff) == TAG_ORPHAN - else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK - else 'name 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'name' if (tag & 0xfff) == TAG_NAME + else 'reg' if (tag & 0xfff) == TAG_REG + else 'dir' if (tag & 0xfff) == TAG_DIR + else 'orphan' if (tag & 0xfff) == TAG_ORPHAN + else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK + else 'name 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_STRUCT: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'data' if (tag & 0xfff) == TAG_DATA - else 'block' if (tag & 0xfff) == TAG_BLOCK - else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB - else 'btree' if (tag & 0xfff) == TAG_BTREE - else 'mroot' if (tag & 0xfff) == TAG_MROOT - else 'mdir' if (tag & 0xfff) == TAG_MDIR - else 'mtree' if (tag & 0xfff) == TAG_MTREE - else 'did' if (tag & 0xfff) == TAG_DID - else 'branch' if (tag & 0xfff) == TAG_BRANCH - else 'struct 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'data' if (tag & 0xfff) == TAG_DATA + else 'block' if (tag & 0xfff) == TAG_BLOCK + else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB + else 'btree' if (tag & 0xfff) == TAG_BTREE + else 'mroot' if (tag & 0xfff) == TAG_MROOT + else 'mdir' if (tag & 0xfff) == TAG_MDIR + else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'did' if (tag & 0xfff) == TAG_DID + else 'branch' if (tag & 0xfff) == TAG_BRANCH + else 'struct 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6e00) == TAG_ATTR: return '%s%sattr 0x%02x%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 's' if tag & 0x100 else 'u', - ((tag & 0x100) >> 1) ^ (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 's' if tag & 0x100 else 'u', + ((tag & 0x100) >> 1) ^ (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( - 'r' if tag & TAG_R else 'b', - 'a' if tag & 0x0fff == 0 and tag & TAG_GT - else 'n' if tag & 0x0fff == 0 - else 'gt' if tag & TAG_GT - else 'le', - ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', - ' w%d' % w if w is not None else '', - ' 0x%x' % (0xffffffff & (off-size)) - if size and off is not None - else ' -%d' % size if size - else '') + 'r' if tag & TAG_R else 'b', + 'a' if tag & 0x0fff == 0 and tag & TAG_GT + else 'n' if tag & 0x0fff == 0 + else 'gt' if tag & TAG_GT + else 'le', + ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', + ' w%d' % w if w is not None else '', + ' 0x%x' % (0xffffffff & (off-size)) + if size and off is not None + else ' -%d' % size if size + else '') elif (tag & 0x7f00) == TAG_CKSUM: return 'cksum%s%s%s%s%s' % ( - 'q' if not tag & 0xfc and tag & TAG_Q else '', - 'p' if not tag & 0xfc and tag & TAG_P else '', - ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'q' if not tag & 0xfc and tag & TAG_Q else '', + 'p' if not tag & 0xfc and tag & TAG_P else '', + ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_NOTE: return 'note%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_ECKSUM: return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( - tag, - ' w%d' % w if w is not None else '', - ' %d' % size if size is not None else '') + tag, + ' w%d' % w if w is not None else '', + ' %d' % size if size is not None else '') # this type is used for tree representations @@ -309,9 +309,9 @@ class Rbyd: return '0x%x.%x' % (self.block, self.trunk) else: return '0x{%x,%s}.%x' % ( - self.block, - ','.join('%x' % block for block in self.redund_blocks), - self.trunk) + self.block, + ','.join('%x' % block for block in self.redund_blocks), + self.trunk) @classmethod def fetch(cls, f, block_size, blocks, trunk=None): @@ -320,21 +320,23 @@ class Rbyd: if len(blocks) > 1: # fetch all blocks - rbyds = [cls.fetch(f, block_size, block, trunk) for block in blocks] + rbyds = [cls.fetch(f, block_size, block, trunk) + for block in blocks] # determine most recent revision i = 0 for i_, rbyd in enumerate(rbyds): # compare with sequence arithmetic if rbyd and ( not rbyds[i] - or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) - or (rbyd.rev == rbyds[i].rev - and rbyd.trunk > rbyds[i].trunk)): + or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) + or (rbyd.rev == rbyds[i].rev + and rbyd.trunk > rbyds[i].trunk)): i = i_ # keep track of the other blocks rbyd = rbyds[i] - rbyd.redund_blocks = [rbyds[(i+1+j) % len(rbyds)].block - for j in range(len(rbyds)-1)] + rbyd.redund_blocks = [ + rbyds[(i+1+j) % len(rbyds)].block + for j in range(len(rbyds)-1)] return rbyd else: # block may encode a trunk @@ -488,7 +490,9 @@ class Rbyd: done = not tag_ or (rid_, tag_) < (rid, tag) - return done, rid_, tag_, w_, j, d, self.data[j+d:j+d+jump], path + return (done, rid_, tag_, w_, j, d, + self.data[j+d:j+d+jump], + path) def __bool__(self): return bool(self.trunk) @@ -580,8 +584,8 @@ class Rbyd: else: if 'h' not in alts[j_]: alts[j_]['h'] = max( - rec_height(alts[j_]['f']), - rec_height(alts[j_]['nf'])) + 1 + rec_height(alts[j_]['f']), + rec_height(alts[j_]['nf'])) + 1 return alts[j_]['h'] for j_ in alts.keys(): @@ -632,7 +636,7 @@ class Rbyd: w = 0 for i in it.count(): done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( - rid_, tag+0x1) + rid_, tag+0x1) if done or (i != 0 and rid__ != rid_): break @@ -675,7 +679,7 @@ class Rbyd: bid = -1 while True: done, bid, w, rbyd_, rid, tags, path = self.btree_lookup( - f, block_size, bid+1, depth=depth) + f, block_size, bid+1, depth=depth) if done: break @@ -690,7 +694,7 @@ class Rbyd: bid = -1 while True: done, bid, w, rbyd_, rid, tags, path = self.btree_lookup( - f, block_size, bid+1, depth=depth) + f, block_size, bid+1, depth=depth) if done: break @@ -723,8 +727,8 @@ class Rbyd: # connect our branch to the rbyd's root if leaf is not None: root = min(rtree, - key=lambda branch: branch.d, - default=None) + key=lambda branch: branch.d, + default=None) if root is not None: r_rid, r_tag = root.a @@ -750,9 +754,10 @@ class Rbyd: d_ += max(bdepths.get(d, 0), 1) leaf = (bid-(w-1), d, rid-(w-1), - next((tag for tag, _, _, _ in tags - if tag & 0xfff == TAG_BRANCH), - TAG_BRANCH)) + next( + (tag for tag, _, _, _ in tags + if tag & 0xfff == TAG_BRANCH), + TAG_BRANCH)) # remap branches to leaves if we aren't showing inner branches if not inner: @@ -809,7 +814,7 @@ class Rbyd: bid = -1 while True: done, bid, w, rbyd, rid, tags, path = self.btree_lookup( - f, block_size, bid+1, depth=depth) + f, block_size, bid+1, depth=depth) if done: break @@ -832,7 +837,7 @@ class Rbyd: continue b = (bid-(w-1), d, rid-(w-1), - (name if name else tags[0])[0]) + (name if name else tags[0])[0]) # remap branches to leaves if we aren't showing # inner branches @@ -842,8 +847,8 @@ class Rbyd: if not tags: continue branches[b] = ( - bid-(w-1), len(path)-1, rid-(w-1), - (name if name else tags[0])[0]) + bid-(w-1), len(path)-1, rid-(w-1), + (name if name else tags[0])[0]) b = branches[b] # found entry point? @@ -874,14 +879,15 @@ class Rbyd: # lookup our mbid done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid) + f, block_size, mbid) if done: return True, -1, 0, None - mdir = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) + mdir = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) if not mdir: return True, -1, 0, None @@ -912,7 +918,7 @@ class Rbyd: upper = self.weight while lower < upper: done, rid, tag, w, j, d, data, _ = self.lookup( - lower + (upper-1-lower)//2, TAG_NAME) + lower + (upper-1-lower)//2, TAG_NAME) if done: break @@ -973,7 +979,7 @@ class Rbyd: # lookup our name in the mtree mbid, tag_, mw, data = mtree.btree_namelookup( - f, block_size, did, name) + f, block_size, did, name) if tag_ != TAG_MDIR: return False, -1, 0, None, -1, 0, 0 @@ -1004,7 +1010,7 @@ class Rbyd: def mtree_dir(self, f, block_size, did): # lookup the bookmark found, mbid, mw, mdir, rid, tag, w = self.mtree_namelookup( - f, block_size, did, b'') + f, block_size, did, b'') # iterate through all files until the next bookmark while found: # lookup each rid @@ -1133,19 +1139,19 @@ class Config: def crepr(tag, data): if tag == TAG_MAGIC: return 'magic \"%s\"' % ''.join( - b if b >= ' ' and b <= '~' else '.' - for b in map(chr, self.magic)) + b if b >= ' ' and b <= '~' else '.' + for b in map(chr, self.magic)) elif tag == TAG_VERSION: return 'version v%d.%d' % self.version elif tag == TAG_RCOMPAT: return 'rcompat 0x%s' % ''.join( - '%x' % f for f in reversed(self.rcompat)) + '%x' % f for f in reversed(self.rcompat)) elif tag == TAG_WCOMPAT: return 'wcompat 0x%s' % ''.join( - '%x' % f for f in reversed(self.wcompat)) + '%x' % f for f in reversed(self.wcompat)) elif tag == TAG_OCOMPAT: return 'ocompat 0x%s' % ''.join( - '%x' % f for f in reversed(self.ocompat)) + '%x' % f for f in reversed(self.ocompat)) elif tag == TAG_GEOMETRY: return 'geometry %dx%d' % self.geometry elif tag == TAG_NAMELIMIT: @@ -1181,8 +1187,9 @@ class GState: # xor gstate if tag not in self.gstate: self.gstate[tag] = b'' - self.gstate[tag] = bytes(a^b for a,b in it.zip_longest( - self.gstate[tag], data, fillvalue=0)) + self.gstate[tag] = bytes( + a^b for a,b in it.zip_longest( + self.gstate[tag], data, fillvalue=0)) # parsers for some gstate @ft.cached_property @@ -1198,8 +1205,8 @@ class GState: for _ in range(count): mid, d_ = fromleb128(data[d:]); d += d_ rms.append(( - mid - (mid % self.mleaf_weight), - mid % self.mleaf_weight)) + mid - (mid % self.mleaf_weight), + mid % self.mleaf_weight)) return rms def repr(self): @@ -1207,11 +1214,12 @@ class GState: if tag == TAG_GRMDELTA: count, _ = fromleb128(data) return 'grm %s' % ( - 'none' if count == 0 - else ' '.join('%d.%d' % (mbid//self.mleaf_weight, rid) - for mbid, rid in self.grm) + 'none' if count == 0 + else ' '.join( + '%d.%d' % (mbid//self.mleaf_weight, rid) + for mbid, rid in self.grm) if count <= 2 - else '0x%x %d' % (count, len(data))) + else '0x%x %d' % (count, len(data))) else: return 'gstate 0x%02x %d' % (tag, len(data)) @@ -1246,8 +1254,8 @@ def frepr(mdir, rid, tag): size = max(size, weight) structs.append('btree 0x%x.%x' % (block, trunk)) return '%s %s' % ( - 'orphan' if tag == TAG_ORPHAN else 'reg', - ', '.join(it.chain(['%d' % size], structs))) + 'orphan' if tag == TAG_ORPHAN else 'reg', + ', '.join(it.chain(['%d' % size], structs))) elif tag == TAG_DIR: # read the did @@ -1280,24 +1288,24 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, # inlined data? if tag == TAG_DATA: btree = Rbyd( - mdir.block, - mdir.data, - mdir.rev, - mdir.eoff, - j, - 0, - 0) + mdir.block, + mdir.data, + mdir.rev, + mdir.eoff, + j, + 0, + 0) w = len(data) # direct block? elif tag == TAG_BLOCK: btree = Rbyd( - mdir.block, - mdir.data, - mdir.rev, - mdir.eoff, - j, - 0, - 0) + mdir.block, + mdir.data, + mdir.rev, + mdir.eoff, + j, + 0, + 0) size, block, off, cksize, cksum = frombptr(data) w = size # inlined bshrub? @@ -1315,17 +1323,17 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, t_width = 0 if args.get('tree') or args.get('rbyd'): tree, tdepth = btree.btree_tree( - f, block_size, - depth=args.get('struct_depth'), - inner=args.get('inner'), - rbyd=args.get('rbyd')) + f, block_size, + depth=args.get('struct_depth'), + inner=args.get('inner'), + rbyd=args.get('rbyd')) # precompute B-trees if requested elif args.get('btree'): tree, tdepth = btree.btree_btree( - f, block_size, - depth=args.get('struct_depth'), - inner=args.get('inner')) + f, block_size, + depth=args.get('struct_depth'), + inner=args.get('inner')) if args.get('tree') or args.get('rbyd') or args.get('btree'): # map the tree into our block space @@ -1422,8 +1430,8 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, for branch in tree): return '+-', branch.c, branch.c elif any(branch.d == d - and x > min(branch.a, branch.b) - and x < max(branch.a, branch.b) + and x > min(branch.a, branch.b) + and x < max(branch.a, branch.b) for branch in tree): return '|-', branch.c, branch.c elif branch.a < branch.b: @@ -1446,16 +1454,16 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, was = None for d in range(t_depth): t, c, was = branchrepr( - (bid-(w-1), bd, rid-(w-1), block, tag), d, was) + (bid-(w-1), bd, rid-(w-1), block, tag), d, was) trunk.append('%s%s%s%s' % ( - '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color and c == 'b' - else '', - t, - ('>' if was else ' ') if d == t_depth-1 else '', - '\x1b[m' if color and c else '')) + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == t_depth-1 else '', + '\x1b[m' if color and c else '')) return '%s ' % ''.join(trunk) @@ -1472,42 +1480,45 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, # show human-readable representation for i, (tag, j, d, data) in enumerate(tags): print('%12s %*s %s%s%-*s %s' % ( - '%04x.%04x:' % (rbyd.block, rbyd.trunk) - if prbyd is None or rbyd != prbyd - else '', - m_width, '', - treerepr(bid, w, bd, rid, False, tag) - if args.get('tree') - or args.get('rbyd') - or args.get('btree') else '', - '%*s ' % (2*w_width+1, '' if i != 0 - else '%d-%d' % (bid-(w-1), bid) if w > 1 - else bid if w > 0 - else ''), - 21+2*w_width+1, - tagrepr(tag, w if i == 0 else 0, len(data), None), - next(xxd(data, 8), '') - if not args.get('raw') and not args.get('no_truncate') - else '')) + '%04x.%04x:' % (rbyd.block, rbyd.trunk) + if prbyd is None or rbyd != prbyd + else '', + m_width, '', + treerepr(bid, w, bd, rid, False, tag) + if args.get('tree') + or args.get('rbyd') + or args.get('btree') + else '', + '%*s ' % ( + 2*w_width+1, '' if i != 0 + else '%d-%d' % (bid-(w-1), bid) if w > 1 + else bid if w > 0 + else ''), + 21+2*w_width+1, tagrepr( + tag, w if i == 0 else 0, len(data), None), + next(xxd(data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else '')) prbyd = rbyd # show on-disk encoding of tags/data if args.get('raw'): for o, line in enumerate(xxd(rbyd.data[j:j+d])): print('%11s: %*s %*s%s%s' % ( - '%04x' % (j + o*16), - m_width, '', - t_width, '', - '%*s ' % (2*w_width+1, ''), - line)) + '%04x' % (j + o*16), + m_width, '', + t_width, '', + '%*s ' % (2*w_width+1, ''), + line)) if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data)): print('%11s: %*s %*s%s%s' % ( - '%04x' % (j+d + o*16), - m_width, '', - t_width, '', - '%*s ' % (2*w_width+1, ''), - line)) + '%04x' % (j+d + o*16), + m_width, '', + t_width, '', + '%*s ' % (2*w_width+1, ''), + line)) def dbg_block(bid, w, rbyd, rid, bptr, block, off, size, cksize, cksum, data, notes, @@ -1517,31 +1528,34 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, # show human-readable representation print('%s%12s%s %*s %s%s%s%-*s%s%s' % ( - '\x1b[31m' if color and notes else '', - '%04x.%04x:' % (rbyd.block, rbyd.trunk) - if prbyd is None or rbyd != prbyd - else '', - '\x1b[0m' if color and notes else '', - m_width, '', - treerepr(bid, w, bd, rid, True, tag) - if args.get('tree') - or args.get('rbyd') - or args.get('btree') else '', - '\x1b[31m' if color and notes else '', - '%*s ' % (2*w_width+1, '%d-%d' % (bid-(w-1), bid) if w > 1 - else bid if w > 0 - else ''), - 56+2*w_width+1, '%-*s %s' % ( - 21+2*w_width+1, '%s%s%s 0x%x.%x %d' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'block', - ' w%d' % w if w else '', - block, off, size), - next(xxd(data, 8), '') - if not args.get('raw') and not args.get('no_truncate') - else ''), - ' (%s)' % ', '.join(notes) if notes else '', - '\x1b[m' if color and notes else '')) + '\x1b[31m' if color and notes else '', + '%04x.%04x:' % (rbyd.block, rbyd.trunk) + if prbyd is None or rbyd != prbyd + else '', + '\x1b[0m' if color and notes else '', + m_width, '', + treerepr(bid, w, bd, rid, True, tag) + if args.get('tree') + or args.get('rbyd') + or args.get('btree') + else '', + '\x1b[31m' if color and notes else '', + '%*s ' % ( + 2*w_width+1, '%d-%d' % (bid-(w-1), bid) if w > 1 + else bid if w > 0 + else ''), + 56+2*w_width+1, '%-*s %s' % ( + 21+2*w_width+1, '%s%s%s 0x%x.%x %d' % ( + 'shrub' if tag & TAG_SHRUB else '', + 'block', + ' w%d' % w if w else '', + block, off, size), + next(xxd(data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else ''), + ' (%s)' % ', '.join(notes) if notes else '', + '\x1b[m' if color and notes else '')) prbyd = rbyd # show on-disk encoding of tags/bptr/data @@ -1549,30 +1563,30 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, _, j, d, data_ = bptr for o, line in enumerate(xxd(rbyd.data[j:j+d])): print('%11s: %*s %*s%s%s' % ( - '%04x' % (j + o*16), - m_width, '', - t_width, '', - '%*s ' % (2*w_width+1, ''), - line)) + '%04x' % (j + o*16), + m_width, '', + t_width, '', + '%*s ' % (2*w_width+1, ''), + line)) if args.get('raw'): _, j, d, data_ = bptr for o, line in enumerate(xxd(data_)): print('%11s: %*s %*s%s%s' % ( - '%04x' % (j+d + o*16), - m_width, '', - t_width, '', - '%*s ' % (2*w_width+1, ''), - line)) + '%04x' % (j+d + o*16), + m_width, '', + t_width, '', + '%*s ' % (2*w_width+1, ''), + line)) if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data)): print('%11s: %*s %*s%s%s' % ( - '%04x.%04x' % (block, off + o*16) - if o == 0 and block != prbyd.block - else '%04x' % (off + o*16), - m_width, '', - t_width, '', - '%*s ' % (2*w_width+1, ''), - line)) + '%04x.%04x' % (block, off + o*16) + if o == 0 and block != prbyd.block + else '%04x' % (off + o*16), + m_width, '', + t_width, '', + '%*s ' % (2*w_width+1, ''), + line)) # if we show non-truncated file contents we need to # reset the rbyd address if block != prbyd.block: @@ -1582,13 +1596,13 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, # entry is a single-element btree if tag == TAG_DATA or tag == TAG_BLOCK or args.get('inner'): dbg_branch(w-1, w, Rbyd( - mdir.block, - mdir.data, - mdir.rev, - mdir.eoff, - j, - 0, - 0), w-1, [(tag, j, d, data)], -1) + mdir.block, + mdir.data, + mdir.rev, + mdir.eoff, + j, + 0, + 0), w-1, [(tag, j, d, data)], -1) # traverse and print entries bid = -1 @@ -1597,7 +1611,7 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, corrupted = False while True: done, bid, w, rbyd, rid, tags, path = btree.btree_lookup( - f, block_size, bid+1, depth=args.get('struct_depth')) + f, block_size, bid+1, depth=args.get('struct_depth')) if done: break @@ -1621,12 +1635,12 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, # corrupted? try to keep printing the tree if not rbyd: print(' %04x.%04x: %*s %*s%s%s%s' % ( - rbyd.block, rbyd.trunk, - m_width, '', - t_width, '', - '\x1b[31m' if color else '', - '(corrupted rbyd %s)' % rbyd.addr(), - '\x1b[m' if color else '')) + rbyd.block, rbyd.trunk, + m_width, '', + t_width, '', + '\x1b[31m' if color else '', + '(corrupted rbyd %s)' % rbyd.addr(), + '\x1b[m' if color else '')) prbyd = rbyd corrupted = True continue @@ -1645,17 +1659,18 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, if name is not None: tags = [name] + [(tag, j, d, data) - for tag, j, d, data in tags - if tag & 0x7f00 != TAG_NAME] + for tag, j, d, data in tags + if tag & 0x7f00 != TAG_NAME] # found a block in the tags? bptr = None if (not args.get('struct_depth') or len(path) < args.get('struct_depth')): - bptr = next(((tag, j, d, data) - for tag, j, d, data in tags - if (tag & 0xfff) == TAG_BLOCK), - None) + bptr = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if (tag & 0xfff) == TAG_BLOCK), + None) # show other btree entries if args.get('inner') or not bptr: @@ -1681,8 +1696,8 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *, # show the block dbg_block(bid, w, rbyd, rid, bptr, - block, off, size, cksize, cksum, data, notes, - len(path)-1) + block, off, size, cksize, cksum, data, notes, + len(path)-1) @@ -1754,12 +1769,12 @@ def main(disk, mroots=None, *, for rid, tag, w, j, d, data in mroot: if tag == TAG_DID: did, d = fromleb128(data) - dir_dids.append( - (did, data[d:], -1, 0, mroot, rid, tag, w)) + dir_dids.append(( + did, data[d:], -1, 0, mroot, rid, tag, w)) elif tag == TAG_BOOKMARK: did, d = fromleb128(data) - bookmark_dids.append( - (did, data[d:], -1, 0, mroot, rid, tag, w)) + bookmark_dids.append(( + did, data[d:], -1, 0, mroot, rid, tag, w)) # fetch the next mroot done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MROOT) @@ -1789,11 +1804,11 @@ def main(disk, mroots=None, *, if tag == TAG_DID: did, d = fromleb128(data) dir_dids.append(( - did, data[d:], 0, 0, mdir, rid, tag, w)) + did, data[d:], 0, 0, mdir, rid, tag, w)) elif tag == TAG_BOOKMARK: did, d = fromleb128(data) bookmark_dids.append(( - did, data[d:], 0, 0, mdir, rid, tag, w)) + did, data[d:], 0, 0, mdir, rid, tag, w)) # fetch the actual mtree, if there is one mtree = None @@ -1808,7 +1823,7 @@ def main(disk, mroots=None, *, mbid = -1 while True: done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1) + f, block_size, mbid+1) if done: break @@ -1817,10 +1832,11 @@ def main(disk, mroots=None, *, corrupted = True continue - mdir__ = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) + mdir__ = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) if mdir__: # fetch the mdir @@ -1840,23 +1856,23 @@ def main(disk, mroots=None, *, if tag == TAG_DID: did, d = fromleb128(data) dir_dids.append(( - did, data[d:], - mbid, mw, mdir_, rid, tag, w)) + did, data[d:], + mbid, mw, mdir_, rid, tag, w)) elif tag == TAG_BOOKMARK: did, d = fromleb128(data) bookmark_dids.append(( - did, data[d:], - mbid, mw, mdir_, rid, tag, w)) + did, data[d:], + mbid, mw, mdir_, rid, tag, w)) # remove grms from our found dids, we treat these as already deleted grmed_dir_dids = {did_ - for (did_, name_, mbid_, mw_, mdir_, rid_, tag_, w_) - in dir_dids - if (max(mbid_-max(mw_-1, 0), 0), rid_) not in gstate.grm} + for (did_, name_, mbid_, mw_, mdir_, rid_, tag_, w_) + in dir_dids + if (max(mbid_-max(mw_-1, 0), 0), rid_) not in gstate.grm} grmed_bookmark_dids = {did_ - for (did_, name_, mbid_, mw_, mdir_, rid_, tag_, w_) - in bookmark_dids - if (max(mbid_-max(mw_-1, 0), 0), rid_) not in gstate.grm} + for (did_, name_, mbid_, mw_, mdir_, rid_, tag_, w_) + in bookmark_dids + if (max(mbid_-max(mw_-1, 0), 0), rid_) not in gstate.grm} # treat the filesystem as corrupted if our dirs and bookmarks are # mismatched, this should never happen unless there's a bug @@ -1865,7 +1881,7 @@ def main(disk, mroots=None, *, # are we going to end up rendering the dtree? dtree = args.get('files') or not ( - args.get('config') or args.get('gstate')) + args.get('config') or args.get('gstate')) # do a pass to find the width that fits file names+tree, this # may not terminate! It's up to the user to use -Z in that case @@ -1880,7 +1896,7 @@ def main(disk, mroots=None, *, # recurse? if tag == TAG_DIR and depth > 1: done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( - rid, TAG_DID) + rid, TAG_DID) if not done and rid_ == rid and tag_ == TAG_DID: did_, _ = fromleb128(data) depth__, width__ = rec_f_width(did_, depth-1) @@ -1897,90 +1913,96 @@ def main(disk, mroots=None, *, # print some information about the filesystem print('littlefs v%s.%s %dx%d %s w%d.%d, rev %08x' % ( - config.version[0] if config.version[0] is not None else '?', - config.version[1] if config.version[1] is not None else '?', - (config.geometry[0] or 0), (config.geometry[1] or 0), - mroot.addr(), - bweight//mleaf_weight, 1*mleaf_weight, - mroot.rev)) + config.version[0] if config.version[0] is not None else '?', + config.version[1] if config.version[1] is not None else '?', + (config.geometry[0] or 0), (config.geometry[1] or 0), + mroot.addr(), + bweight//mleaf_weight, 1*mleaf_weight, + mroot.rev)) # dynamically size the id field w_width = max( - mt.ceil(mt.log10(max(1, bweight//mleaf_weight)+1)), - mt.ceil(mt.log10(max(1, rweight)+1)), - # in case of -1.-1 - 2) + mt.ceil(mt.log10(max(1, bweight//mleaf_weight)+1)), + mt.ceil(mt.log10(max(1, rweight)+1)), + # in case of -1.-1 + 2) # print config? if args.get('config'): for i, (repr_, tag, j, data) in enumerate(config.repr()): print('%12s %*s %-*s %s' % ( - '{%s}:' % ','.join('%04x' % block - for block in it.chain([mroot.block], - mroot.redund_blocks)) - if i == 0 else '', - 2*w_width+1, '%d.%d' % (-1, -1) - if i == 0 else '', - 21+w_width, repr_, - next(xxd(data, 8), '') - if not args.get('raw') - and not args.get('no_truncate') else '')) + '{%s}:' % ','.join('%04x' % block + for block in it.chain( + [mroot.block], + mroot.redund_blocks)) + if i == 0 else '', + 2*w_width+1, '%d.%d' % (-1, -1) + if i == 0 else '', + 21+w_width, repr_, + next(xxd(data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else '')) # show on-disk encoding if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data)): print('%11s: %*s %s' % ( - '%04x' % (j + o*16), - 2*w_width+1, '', - line)) + '%04x' % (j + o*16), + 2*w_width+1, '', + line)) # print gstate? if args.get('gstate'): for i, (repr_, tag, data) in enumerate(gstate.repr()): print('%12s %*s %-*s %s' % ( - 'gstate:' if i == 0 else '', - 2*w_width+1, 'g' if i == 0 else '', - 21+w_width, repr_, - next(xxd(data, 8), '') - if not args.get('raw') - and not args.get('no_truncate') else '')) + 'gstate:' if i == 0 else '', + 2*w_width+1, 'g' if i == 0 else '', + 21+w_width, repr_, + next(xxd(data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else '')) # show on-disk encoding if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data)): print('%11s: %*s %s' % ( - '%04x' % (o*16), - 2*w_width+1, '', - line)) + '%04x' % (o*16), + 2*w_width+1, '', + line)) # print gdeltas? if args.get('gdelta'): for mbid, mw, mdir, j, d, data in gstate.gdelta[tag]: print('%s%12s %*s %-*s %s%s' % ( - '\x1b[90m' if color else '', - '{%s}:' % ','.join('%04x' % block - for block in it.chain([mdir.block], - mdir.redund_blocks)), - 2*w_width+1, '%d.%d' % (mbid//mleaf_weight, -1), - 21+w_width, tagrepr(tag, 0, len(data)), - next(xxd(data, 8), '') - if not args.get('raw') - and not args.get('no_truncate') else '', - '\x1b[m' if color else '')) + '\x1b[90m' if color else '', + '{%s}:' % ','.join('%04x' % block + for block in it.chain( + [mdir.block], + mdir.redund_blocks)), + 2*w_width+1, '%d.%d' % ( + mbid//mleaf_weight, -1), + 21+w_width, tagrepr(tag, 0, len(data)), + next(xxd(data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else '', + '\x1b[m' if color else '')) # show on-disk encoding if args.get('raw'): for o, line in enumerate(xxd(mdir.data[j:j+d])): print('%11s: %*s %s' % ( - '%04x' % (j + o*16), - 2*w_width+1, '', - line)) + '%04x' % (j + o*16), + 2*w_width+1, '', + line)) if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data)): print('%11s: %*s %s' % ( - '%04x' % (j+d + o*16), - 2*w_width+1, '', - line)) + '%04x' % (j+d + o*16), + 2*w_width+1, '', + line)) # print dtree? if dtree: @@ -2028,7 +2050,7 @@ def main(disk, mroots=None, *, # missing bookmark? if tag == TAG_DIR: done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( - rid, TAG_DID) + rid, TAG_DID) if not done and rid_ == rid and tag_ == TAG_DID: did_, _ = fromleb128(data) if did_ not in grmed_bookmark_dids: @@ -2038,7 +2060,7 @@ def main(disk, mroots=None, *, # orphaned? if tag == TAG_BOOKMARK: done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( - rid, tag) + rid, tag) if not done and rid_ == rid and tag_ == tag: did_, _ = fromleb128(data) if did_ not in grmed_dir_dids: @@ -2046,31 +2068,34 @@ def main(disk, mroots=None, *, # print human readable dtree entry print('%s%12s %*s %-*s %s%s%s' % ( - '\x1b[31m' if color and not grmed and notes - else '\x1b[90m' - if color and (grmed - or tag == TAG_BOOKMARK - or tag == TAG_ORPHAN) - else '', - '{%s}:' % ','.join('%04x' % block - for block in it.chain([mdir.block], - mdir.redund_blocks)) - if mbid != pmbid else '', - 2*w_width+1, '%d.%d-%d' % ( - mbid//mleaf_weight, rid-(w-1), rid) - if w > 1 else '%d.%d' % (mbid//mleaf_weight, rid) - if w > 0 else '', - f_width, '%s%s' % ( - prefixes[0+(i==len(dir)-1)], - name.decode('utf8')), - frepr(mdir, rid, tag), - ' (%s)' % ', '.join(notes) if notes else '', - '\x1b[m' if color and ( - notes - or grmed - or tag == TAG_BOOKMARK - or tag == TAG_ORPHAN) - else '')) + '\x1b[31m' if color and not grmed and notes + else '\x1b[90m' + if color and (grmed + or tag == TAG_BOOKMARK + or tag == TAG_ORPHAN) + else '', + '{%s}:' % ','.join('%04x' % block + for block in it.chain( + [mdir.block], + mdir.redund_blocks)) + if mbid != pmbid else '', + 2*w_width+1, '%d.%d-%d' % ( + mbid//mleaf_weight, rid-(w-1), rid) + if w > 1 + else '%d.%d' % (mbid//mleaf_weight, rid) + if w > 0 + else '', + f_width, '%s%s' % ( + prefixes[0+(i==len(dir)-1)], + name.decode('utf8')), + frepr(mdir, rid, tag), + ' (%s)' % ', '.join(notes) if notes else '', + '\x1b[m' if color and ( + notes + or grmed + or tag == TAG_BOOKMARK + or tag == TAG_ORPHAN) + else '')) pmbid = mbid # print attrs associated with this file? @@ -2078,89 +2103,88 @@ def main(disk, mroots=None, *, tag_ = 0 while True: done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( - rid, tag_+0x1) + rid, tag_+0x1) if done or rid_ != rid: break print('%12s %*s %-*s %s' % ( - '', - 2*w_width+1, '', - 21+w_width, tagrepr(tag_, w_, len(data)), - next(xxd(data, 8), '') - if not args.get('raw') - and not args.get('no_truncate') - else '')) + '', + 2*w_width+1, '', + 21+w_width, tagrepr(tag_, w_, len(data)), + next(xxd(data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else '')) # show on-disk encoding if args.get('raw'): for o, line in enumerate(xxd(mdir.data[j:j+d])): print('%11s: %*s %s' % ( - '%04x' % (j + o*16), - 2*w_width+1, '', - line)) + '%04x' % (j + o*16), + 2*w_width+1, '', + line)) if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data)): print('%11s: %*s %s' % ( - '%04x' % (j+d + o*16), - 2*w_width+1, '', - line)) + '%04x' % (j+d + o*16), + 2*w_width+1, '', + line)) # print file contents? if ((tag == TAG_REG or tag == TAG_ORPHAN) and args.get('structs')): # inlined sprout? done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( - rid, TAG_DATA) + rid, TAG_DATA) if not done and rid_ == rid and tag_ == TAG_DATA: dbg_fstruct(f, block_size, - mdir, rid_, tag_, j, d, data, - m_width=2*w_width+1, - color=color, - args=args) + mdir, rid_, tag_, j, d, data, + m_width=2*w_width+1, + color=color, + args=args) # direct block? done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( - rid, TAG_BLOCK) + rid, TAG_BLOCK) if not done and rid_ == rid and tag_ == TAG_BLOCK: dbg_fstruct(f, block_size, - mdir, rid_, tag_, j, d, data, - m_width=2*w_width+1, - color=color, - args=args) + mdir, rid_, tag_, j, d, data, + m_width=2*w_width+1, + color=color, + args=args) # inlined bshrub? done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( - rid, TAG_BSHRUB) + rid, TAG_BSHRUB) if not done and rid_ == rid and tag_ == TAG_BSHRUB: dbg_fstruct(f, block_size, - mdir, rid_, tag_, j, d, data, - m_width=2*w_width+1, - color=color, - args=args) + mdir, rid_, tag_, j, d, data, + m_width=2*w_width+1, + color=color, + args=args) # indirect btree? done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( - rid, TAG_BTREE) + rid, TAG_BTREE) if not done and rid_ == rid and tag_ == TAG_BTREE: dbg_fstruct(f, block_size, - mdir, rid_, tag_, j, d, data, - m_width=2*w_width+1, - color=color, - args=args) + mdir, rid_, tag_, j, d, data, + m_width=2*w_width+1, + color=color, + args=args) # recurse? if tag == TAG_DIR and depth > 1: done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( - rid, TAG_DID) + rid, TAG_DID) if not done and rid_ == rid and tag_ == TAG_DID: did_, _ = fromleb128(data) - rec_dir( - did_, - depth-1, - (prefixes[2+(i==len(dir)-1)] + "|-> ", - prefixes[2+(i==len(dir)-1)] + "'-> ", - prefixes[2+(i==len(dir)-1)] + "| ", - prefixes[2+(i==len(dir)-1)] + " ")) + rec_dir(did_, + depth-1, + (prefixes[2+(i==len(dir)-1)] + "|-> ", + prefixes[2+(i==len(dir)-1)] + "'-> ", + prefixes[2+(i==len(dir)-1)] + "| ", + prefixes[2+(i==len(dir)-1)] + " ")) rec_dir(0, args.get('depth') or mt.inf) @@ -2172,97 +2196,97 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Debug littlefs stuff.", - allow_abbrev=False) + description="Debug littlefs stuff.", + allow_abbrev=False) parser.add_argument( - 'disk', - help="File containing the block device.") + 'disk', + help="File containing the block device.") parser.add_argument( - 'mroots', - nargs='*', - type=rbydaddr, - help="Block address of the mroots. Defaults to 0x{0,1}.") + 'mroots', + nargs='*', + type=rbydaddr, + help="Block address of the mroots. Defaults to 0x{0,1}.") parser.add_argument( - '-b', '--block-size', - type=bdgeom, - help="Block size/geometry in bytes.") + '-b', '--block-size', + type=bdgeom, + help="Block size/geometry in bytes.") parser.add_argument( - '--block-count', - type=lambda x: int(x, 0), - help="Block count in blocks.") + '--block-count', + type=lambda x: int(x, 0), + help="Block count in blocks.") parser.add_argument( - '--color', - choices=['never', 'always', 'auto'], - default='auto', - help="When to use terminal colors. Defaults to 'auto'.") + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( - '-c', '--config', - action='store_true', - help="Show the on-disk config.") + '-c', '--config', + action='store_true', + help="Show the on-disk config.") parser.add_argument( - '-g', '--gstate', - action='store_true', - help="Show the current global-state.") + '-g', '--gstate', + action='store_true', + help="Show the current global-state.") parser.add_argument( - '-d', '--gdelta', - action='store_true', - help="Show the gdelta that xors into the global-state.") + '-d', '--gdelta', + action='store_true', + help="Show the gdelta that xors into the global-state.") parser.add_argument( - '-f', '--files', - action='store_true', - help="Show the files and directory tree (default).") + '-f', '--files', + action='store_true', + help="Show the files and directory tree (default).") parser.add_argument( - '-A', '--attrs', - action='store_true', - help="Show all attributes belonging to each file.") + '-A', '--attrs', + action='store_true', + help="Show all attributes belonging to each file.") parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all files including bookmarks and grmed files.") + '-a', '--all', + action='store_true', + help="Show all files including bookmarks and grmed files.") parser.add_argument( - '-r', '--raw', - action='store_true', - help="Show the raw data including tag encodings.") + '-r', '--raw', + action='store_true', + help="Show the raw data including tag encodings.") parser.add_argument( - '-T', '--no-truncate', - action='store_true', - help="Don't truncate, show the full contents.") + '-T', '--no-truncate', + action='store_true', + help="Don't truncate, show the full contents.") parser.add_argument( - '-z', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of the filesystem tree to show.") + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of the filesystem tree to show.") parser.add_argument( - '-s', '--structs', - action='store_true', - help="Store file data structures and data.") + '-s', '--structs', + action='store_true', + help="Store file data structures and data.") parser.add_argument( - '-t', '--tree', - action='store_true', - help="Show the underlying rbyd trees.") + '-t', '--tree', + action='store_true', + help="Show the underlying rbyd trees.") parser.add_argument( - '-B', '--btree', - action='store_true', - help="Show the underlying B-trees.") + '-B', '--btree', + action='store_true', + help="Show the underlying B-trees.") parser.add_argument( - '-R', '--rbyd', - action='store_true', - help="Show the full underlying rbyd trees.") + '-R', '--rbyd', + action='store_true', + help="Show the full underlying rbyd trees.") parser.add_argument( - '-i', '--inner', - action='store_true', - help="Show inner branches.") + '-i', '--inner', + action='store_true', + help="Show inner branches.") parser.add_argument( - '-Z', '--struct-depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of struct trees to show.") + '-Z', '--struct-depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of struct trees to show.") parser.add_argument( - '-e', '--error-on-corrupt', - action='store_true', - help="Error if the filesystem is corrupt.") + '-e', '--error-on-corrupt', + action='store_true', + help="Error if the filesystem is corrupt.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 85182963..5b8f2b4a 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -169,108 +169,108 @@ def frombtree(data): def xxd(data, width=16): for i in range(0, len(data), width): yield '%-*s %-*s' % ( - 3*width, - ' '.join('%02x' % b for b in data[i:i+width]), - width, - ''.join( - b if b >= ' ' and b <= '~' else '.' - for b in map(chr, data[i:i+width]))) + 3*width, + ' '.join('%02x' % b for b in data[i:i+width]), + width, + ''.join( + b if b >= ' ' and b <= '~' else '.' + for b in map(chr, data[i:i+width]))) def tagrepr(tag, w=None, size=None, off=None): if (tag & 0x6fff) == TAG_NULL: return '%snull%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - ' w%d' % w if w else '', - ' %d' % size if size else '') + 'shrub' if tag & TAG_SHRUB else '', + ' w%d' % w if w else '', + ' %d' % size if size else '') elif (tag & 0x6f00) == TAG_CONFIG: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'magic' if (tag & 0xfff) == TAG_MAGIC - else 'version' if (tag & 0xfff) == TAG_VERSION - else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT - else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT - else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT - else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY - else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT - else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT - else 'config 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'magic' if (tag & 0xfff) == TAG_MAGIC + else 'version' if (tag & 0xfff) == TAG_VERSION + else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT + else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT + else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT + else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY + else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT + else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT + else 'config 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_GDELTA: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA - else 'gdelta 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gdelta 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_NAME: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'name' if (tag & 0xfff) == TAG_NAME - else 'reg' if (tag & 0xfff) == TAG_REG - else 'dir' if (tag & 0xfff) == TAG_DIR - else 'orphan' if (tag & 0xfff) == TAG_ORPHAN - else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK - else 'name 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'name' if (tag & 0xfff) == TAG_NAME + else 'reg' if (tag & 0xfff) == TAG_REG + else 'dir' if (tag & 0xfff) == TAG_DIR + else 'orphan' if (tag & 0xfff) == TAG_ORPHAN + else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK + else 'name 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_STRUCT: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'data' if (tag & 0xfff) == TAG_DATA - else 'block' if (tag & 0xfff) == TAG_BLOCK - else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB - else 'btree' if (tag & 0xfff) == TAG_BTREE - else 'mroot' if (tag & 0xfff) == TAG_MROOT - else 'mdir' if (tag & 0xfff) == TAG_MDIR - else 'mtree' if (tag & 0xfff) == TAG_MTREE - else 'did' if (tag & 0xfff) == TAG_DID - else 'branch' if (tag & 0xfff) == TAG_BRANCH - else 'struct 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'data' if (tag & 0xfff) == TAG_DATA + else 'block' if (tag & 0xfff) == TAG_BLOCK + else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB + else 'btree' if (tag & 0xfff) == TAG_BTREE + else 'mroot' if (tag & 0xfff) == TAG_MROOT + else 'mdir' if (tag & 0xfff) == TAG_MDIR + else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'did' if (tag & 0xfff) == TAG_DID + else 'branch' if (tag & 0xfff) == TAG_BRANCH + else 'struct 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6e00) == TAG_ATTR: return '%s%sattr 0x%02x%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 's' if tag & 0x100 else 'u', - ((tag & 0x100) >> 1) ^ (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 's' if tag & 0x100 else 'u', + ((tag & 0x100) >> 1) ^ (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( - 'r' if tag & TAG_R else 'b', - 'a' if tag & 0x0fff == 0 and tag & TAG_GT - else 'n' if tag & 0x0fff == 0 - else 'gt' if tag & TAG_GT - else 'le', - ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', - ' w%d' % w if w is not None else '', - ' 0x%x' % (0xffffffff & (off-size)) - if size and off is not None - else ' -%d' % size if size - else '') + 'r' if tag & TAG_R else 'b', + 'a' if tag & 0x0fff == 0 and tag & TAG_GT + else 'n' if tag & 0x0fff == 0 + else 'gt' if tag & TAG_GT + else 'le', + ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', + ' w%d' % w if w is not None else '', + ' 0x%x' % (0xffffffff & (off-size)) + if size and off is not None + else ' -%d' % size if size + else '') elif (tag & 0x7f00) == TAG_CKSUM: return 'cksum%s%s%s%s%s' % ( - 'q' if not tag & 0xfc and tag & TAG_Q else '', - 'p' if not tag & 0xfc and tag & TAG_P else '', - ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'q' if not tag & 0xfc and tag & TAG_Q else '', + 'p' if not tag & 0xfc and tag & TAG_P else '', + ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_NOTE: return 'note%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_ECKSUM: return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( - tag, - ' w%d' % w if w is not None else '', - ' %d' % size if size is not None else '') + tag, + ' w%d' % w if w is not None else '', + ' %d' % size if size is not None else '') # this type is used for tree representations @@ -293,9 +293,9 @@ class Rbyd: return '0x%x.%x' % (self.block, self.trunk) else: return '0x{%x,%s}.%x' % ( - self.block, - ','.join('%x' % block for block in self.redund_blocks), - self.trunk) + self.block, + ','.join('%x' % block for block in self.redund_blocks), + self.trunk) @classmethod def fetch(cls, f, block_size, blocks, trunk=None): @@ -304,21 +304,23 @@ class Rbyd: if len(blocks) > 1: # fetch all blocks - rbyds = [cls.fetch(f, block_size, block, trunk) for block in blocks] + rbyds = [cls.fetch(f, block_size, block, trunk) + for block in blocks] # determine most recent revision i = 0 for i_, rbyd in enumerate(rbyds): # compare with sequence arithmetic if rbyd and ( not rbyds[i] - or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) - or (rbyd.rev == rbyds[i].rev - and rbyd.trunk > rbyds[i].trunk)): + or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) + or (rbyd.rev == rbyds[i].rev + and rbyd.trunk > rbyds[i].trunk)): i = i_ # keep track of the other blocks rbyd = rbyds[i] - rbyd.redund_blocks = [rbyds[(i+1+j) % len(rbyds)].block - for j in range(len(rbyds)-1)] + rbyd.redund_blocks = [ + rbyds[(i+1+j) % len(rbyds)].block + for j in range(len(rbyds)-1)] return rbyd else: # block may encode a trunk @@ -472,7 +474,9 @@ class Rbyd: done = not tag_ or (rid_, tag_) < (rid, tag) - return done, rid_, tag_, w_, j, d, self.data[j+d:j+d+jump], path + return (done, rid_, tag_, w_, j, d, + self.data[j+d:j+d+jump], + path) def __bool__(self): return bool(self.trunk) @@ -564,8 +568,8 @@ class Rbyd: else: if 'h' not in alts[j_]: alts[j_]['h'] = max( - rec_height(alts[j_]['f']), - rec_height(alts[j_]['nf'])) + 1 + rec_height(alts[j_]['f']), + rec_height(alts[j_]['nf'])) + 1 return alts[j_]['h'] for j_ in alts.keys(): @@ -616,7 +620,7 @@ class Rbyd: w = 0 for i in it.count(): done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( - rid_, tag+0x1) + rid_, tag+0x1) if done or (i != 0 and rid__ != rid_): break @@ -659,7 +663,7 @@ class Rbyd: bid = -1 while True: done, bid, w, rbyd_, rid, tags, path = self.btree_lookup( - f, block_size, bid+1, depth=depth) + f, block_size, bid+1, depth=depth) if done: break @@ -674,7 +678,7 @@ class Rbyd: bid = -1 while True: done, bid, w, rbyd_, rid, tags, path = self.btree_lookup( - f, block_size, bid+1, depth=depth) + f, block_size, bid+1, depth=depth) if done: break @@ -707,8 +711,8 @@ class Rbyd: # connect our branch to the rbyd's root if leaf is not None: root = min(rtree, - key=lambda branch: branch.d, - default=None) + key=lambda branch: branch.d, + default=None) if root is not None: r_rid, r_tag = root.a @@ -734,9 +738,10 @@ class Rbyd: d_ += max(bdepths.get(d, 0), 1) leaf = (bid-(w-1), d, rid-(w-1), - next((tag for tag, _, _, _ in tags - if tag & 0xfff == TAG_BRANCH), - TAG_BRANCH)) + next( + (tag for tag, _, _, _ in tags + if tag & 0xfff == TAG_BRANCH), + TAG_BRANCH)) # remap branches to leaves if we aren't showing inner branches if not inner: @@ -793,7 +798,7 @@ class Rbyd: bid = -1 while True: done, bid, w, rbyd, rid, tags, path = self.btree_lookup( - f, block_size, bid+1, depth=depth) + f, block_size, bid+1, depth=depth) if done: break @@ -816,7 +821,7 @@ class Rbyd: continue b = (bid-(w-1), d, rid-(w-1), - (name if name else tags[0])[0]) + (name if name else tags[0])[0]) # remap branches to leaves if we aren't showing # inner branches @@ -826,8 +831,8 @@ class Rbyd: if not tags: continue branches[b] = ( - bid-(w-1), len(path)-1, rid-(w-1), - (name if name else tags[0])[0]) + bid-(w-1), len(path)-1, rid-(w-1), + (name if name else tags[0])[0]) b = branches[b] # found entry point? @@ -935,8 +940,8 @@ def main(disk, mroots=None, *, mbid = -1 while True: done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth) + f, block_size, mbid+1, + depth=args.get('depth', mdepth)-mdepth) if done: break @@ -947,10 +952,11 @@ def main(disk, mroots=None, *, mdir__ = None if (not args.get('depth') or mdepth+len(path) < args.get('depth')): - mdir__ = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) + mdir__ = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) if mdir__: # fetch the mdir @@ -981,8 +987,8 @@ def main(disk, mroots=None, *, # connect branch to our root if d > 0: root = min(rtree, - key=lambda branch: branch.d, - default=None) + key=lambda branch: branch.d, + default=None) if root: r_rid, r_tag = root.a @@ -1026,8 +1032,8 @@ def main(disk, mroots=None, *, # connect branch to our root root = min(rtree, - key=lambda branch: branch.d, - default=None) + key=lambda branch: branch.d, + default=None) if root: r_rid, r_tag = root.a @@ -1054,10 +1060,10 @@ def main(disk, mroots=None, *, # compute the mtree's rbyd-tree if there is one if mtree: tree_, tdepth = mtree.btree_tree( - f, block_size, - depth=args.get('depth', mdepth)-mdepth, - inner=args.get('inner'), - rbyd=args.get('rbyd')) + f, block_size, + depth=args.get('depth', mdepth)-mdepth, + inner=args.get('inner'), + rbyd=args.get('rbyd')) # connect a branch to the root of the tree root = min(tree_, key=lambda branch: branch.d, default=None) @@ -1086,8 +1092,8 @@ def main(disk, mroots=None, *, mbid = -1 while True: done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth) + f, block_size, mbid+1, + depth=args.get('depth', mdepth)-mdepth) if done: break @@ -1098,10 +1104,11 @@ def main(disk, mroots=None, *, mdir__ = None if (not args.get('depth') or mdepth+len(path) < args.get('depth')): - mdir__ = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) + mdir__ = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) if mdir__: # fetch the mdir @@ -1116,8 +1123,8 @@ def main(disk, mroots=None, *, mbid = -1 while True: done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth) + f, block_size, mbid+1, + depth=args.get('depth', mdepth)-mdepth) if done: break @@ -1128,10 +1135,11 @@ def main(disk, mroots=None, *, mdir__ = None if (not args.get('depth') or mdepth+len(path) < args.get('depth')): - mdir__ = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) + mdir__ = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) if mdir__: # fetch the mdir @@ -1143,19 +1151,19 @@ def main(disk, mroots=None, *, # connect the root to the mtree branch = max( - (branch for branch in tree - if branch.b[0] == mbid-(mw-1)), - key=lambda branch: branch.d, - default=None) - if branch: - root = min(rtree, + (branch for branch in tree + if branch.b[0] == mbid-(mw-1)), key=lambda branch: branch.d, default=None) + if branch: + root = min(rtree, + key=lambda branch: branch.d, + default=None) if root: r_rid, r_tag = root.a else: _, r_rid, r_tag, _, _, _, _, _ = ( - mdir_.lookup(-1, 0x1)) + mdir_.lookup(-1, 0x1)) tree.add(TBranch( a=branch.b, b=(mbid-(mw-1), len(path), 0, r_rid, r_tag), @@ -1183,7 +1191,7 @@ def main(disk, mroots=None, *, # keep track of the original bids, unfortunately because we # store the bids in the branches we overwrite these tree = {(branch.b[0] - branch.b[2], branch) - for branch in tree} + for branch in tree} for bd in reversed(range(b_depth-1)): # find leaf-roots at this level @@ -1274,9 +1282,9 @@ def main(disk, mroots=None, *, # compute the mtree's B-tree if there is one if mtree: tree_, tdepth = mtree.btree_btree( - f, block_size, - depth=args.get('depth', mdepth)-mdepth, - inner=args.get('inner')) + f, block_size, + depth=args.get('depth', mdepth)-mdepth, + inner=args.get('inner')) # connect a branch to the root of the tree root = min(tree_, key=lambda branch: branch.d, default=None) @@ -1306,8 +1314,8 @@ def main(disk, mroots=None, *, while True: done, mbid, mw, rbyd, rid, tags, path = ( mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth)) + f, block_size, mbid+1, + depth=args.get('depth', mdepth)-mdepth)) if done: break @@ -1318,10 +1326,11 @@ def main(disk, mroots=None, *, mdir__ = None if (not args.get('depth') or mdepth+len(path) < args.get('depth')): - mdir__ = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) + mdir__ = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) if mdir__: # fetch the mdir @@ -1332,7 +1341,7 @@ def main(disk, mroots=None, *, # find the first entry in the mdir, map branches # to this entry done, rid, tag, _, j, d, data, _ = ( - mdir_.lookup(-1, 0x1)) + mdir_.lookup(-1, 0x1)) tree_ = set() for branch in tree: @@ -1373,8 +1382,8 @@ def main(disk, mroots=None, *, for branch in tree): return '+-', branch.c, branch.c elif any(branch.d == d - and x > min(branch.a, branch.b) - and x < max(branch.a, branch.b) + and x > min(branch.a, branch.b) + and x < max(branch.a, branch.b) for branch in tree): return '|-', branch.c, branch.c elif branch.a < branch.b: @@ -1397,17 +1406,18 @@ def main(disk, mroots=None, *, was = None for d in range(t_depth): t, c, was = branchrepr( - (mbid-max(mw-1, 0), md, mrid-max(mw-1, 0), rid, tag), - d, was) + (mbid-max(mw-1, 0), md, + mrid-max(mw-1, 0), rid, tag), + d, was) trunk.append('%s%s%s%s' % ( - '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color and c == 'b' - else '', - t, - ('>' if was else ' ') if d == t_depth-1 else '', - '\x1b[m' if color and c else '')) + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == t_depth-1 else '', + '\x1b[m' if color and c else '')) return '%s ' % ''.join(trunk) @@ -1416,41 +1426,45 @@ def main(disk, mroots=None, *, for i, (rid, tag, w, j, d, data) in enumerate(mdir): # show human-readable tag representation print('%12s %s%s' % ( - '{%s}:' % ','.join('%04x' % block - for block in it.chain([mdir.block], - mdir.redund_blocks)) - if i == 0 else '', - treerepr(mbid-max(mw-1, 0), 0, md, 0, rid, tag) - if args.get('tree') - or args.get('rbyd') - or args.get('btree') else '', - '%*s %-*s%s' % ( - 2*w_width+1, '%d.%d-%d' % ( - mbid//mleaf_weight, rid-(w-1), rid) - if w > 1 else '%d.%d' % (mbid//mleaf_weight, rid) - if w > 0 or i == 0 else '', - 21+w_width, tagrepr(tag, w, len(data), j), - ' %s' % next(xxd(data, 8), '') - if not args.get('raw') - and not args.get('no_truncate') - else ''))) + '{%s}:' % ','.join('%04x' % block + for block in it.chain( + [mdir.block], + mdir.redund_blocks)) + if i == 0 else '', + treerepr(mbid-max(mw-1, 0), 0, md, 0, rid, tag) + if args.get('tree') + or args.get('rbyd') + or args.get('btree') + else '', + '%*s %-*s%s' % ( + 2*w_width+1, '%d.%d-%d' % ( + mbid//mleaf_weight, rid-(w-1), rid) + if w > 1 + else '%d.%d' % (mbid//mleaf_weight, rid) + if w > 0 or i == 0 + else '', + 21+w_width, tagrepr(tag, w, len(data), j), + ' %s' % next(xxd(data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else ''))) # show on-disk encoding of tags if args.get('raw'): for o, line in enumerate(xxd(mdir.data[j:j+d])): print('%11s: %*s%*s %s' % ( - '%04x' % (j + o*16), - t_width, '', - 2*w_width+1, '', - line)) + '%04x' % (j + o*16), + t_width, '', + 2*w_width+1, '', + line)) if args.get('raw') or args.get('no_truncate'): if not tag & TAG_ALT: for o, line in enumerate(xxd(data)): print('%11s: %*s%*s %s' % ( - '%04x' % (j+d + o*16), - t_width, '', - 2*w_width+1, '', - line)) + '%04x' % (j+d + o*16), + t_width, '', + 2*w_width+1, '', + line)) # prbyd here means the last rendered rbyd, we update # in dbg_branch to always print interleaved addresses @@ -1461,59 +1475,61 @@ def main(disk, mroots=None, *, # show human-readable representation for i, (tag, j, d, data) in enumerate(tags): print('%12s %s%*s %-*s %s' % ( - '%04x.%04x:' % (rbyd.block, rbyd.trunk) - if prbyd is None or rbyd != prbyd - else '', - treerepr(bid, w, bd, rid, 0, tag) - if args.get('tree') - or args.get('rbyd') - or args.get('btree') else '', - 2*w_width+1, '' if i != 0 - else '%d-%d' % ( + '%04x.%04x:' % (rbyd.block, rbyd.trunk) + if prbyd is None or rbyd != prbyd + else '', + treerepr(bid, w, bd, rid, 0, tag) + if args.get('tree') + or args.get('rbyd') + or args.get('btree') + else '', + 2*w_width+1, '' if i != 0 + else '%d-%d' % ( (bid-(w-1))//mleaf_weight, bid//mleaf_weight) if (w//mleaf_weight) > 1 - else bid//mleaf_weight if w > 0 - else '', - 21+w_width, tagrepr( - tag, w if i == 0 else 0, len(data), None), - next(xxd(data, 8), '') - if not args.get('raw') and not args.get('no_truncate') - else '')) + else bid//mleaf_weight if w > 0 + else '', + 21+w_width, tagrepr( + tag, w if i == 0 else 0, len(data), None), + next(xxd(data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else '')) prbyd = rbyd # show on-disk encoding of tags/data if args.get('raw'): for o, line in enumerate(xxd(rbyd.data[j:j+d])): print('%11s: %*s%*s %s' % ( - '%04x' % (j + o*16), - t_width, '', - 2*w_width+1, '', - line)) + '%04x' % (j + o*16), + t_width, '', + 2*w_width+1, '', + line)) if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data)): print('%11s: %*s%*s %s' % ( - '%04x' % (j+d + o*16), - t_width, '', - 2*w_width+1, '', - line)) + '%04x' % (j+d + o*16), + t_width, '', + 2*w_width+1, '', + line)) #### actual debugging begins here # print some information about the mtree print('mtree %s w%d.%d, rev %08x, cksum %08x' % ( - mroot.addr(), - bweight//mleaf_weight, 1*mleaf_weight, - mroot.rev, - mroot.cksum)) + mroot.addr(), + bweight//mleaf_weight, 1*mleaf_weight, + mroot.rev, + mroot.cksum)) # dynamically size the id field w_width = max( - mt.ceil(mt.log10(max(1, bweight//mleaf_weight)+1)), - mt.ceil(mt.log10(max(1, rweight)+1)), - # in case of -1.-1 - 2) + mt.ceil(mt.log10(max(1, bweight//mleaf_weight)+1)), + mt.ceil(mt.log10(max(1, rweight)+1)), + # in case of -1.-1 + 2) # show each mroot prbyd = None @@ -1525,12 +1541,13 @@ def main(disk, mroots=None, *, # corrupted? if not mroot: print('{%s}: %s%s%s' % ( - ','.join('%04x' % block - for block in it.chain([mroot.block], - mroot.redund_blocks)), - '\x1b[31m' if color else '', - '(corrupted mroot %s)' % mroot.addr(), - '\x1b[m' if color else '')) + ','.join('%04x' % block + for block in it.chain( + [mroot.block], + mroot.redund_blocks)), + '\x1b[31m' if color else '', + '(corrupted mroot %s)' % mroot.addr(), + '\x1b[m' if color else '')) corrupted = True break else: @@ -1560,12 +1577,13 @@ def main(disk, mroots=None, *, # corrupted? if not mdir: print('{%s}: %s%s%s' % ( - ','.join('%04x' % block - for block in it.chain([mdir.block], - mdir.redund_blocks)), - '\x1b[31m' if color else '', - '(corrupted mdir %s)' % mdir.addr(), - '\x1b[m' if color else '')) + ','.join('%04x' % block + for block in it.chain( + [mdir.block], + mdir.redund_blocks)), + '\x1b[31m' if color else '', + '(corrupted mdir %s)' % mdir.addr(), + '\x1b[m' if color else '')) corrupted = True else: # show the mdir @@ -1582,8 +1600,8 @@ def main(disk, mroots=None, *, mbid = -1 while True: done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth) + f, block_size, mbid+1, + depth=args.get('depth', mdepth)-mdepth) if done: break @@ -1607,11 +1625,11 @@ def main(disk, mroots=None, *, # corrupted? try to keep printing the tree if not rbyd: print('%11s: %*s%s%s%s' % ( - '%04x.%04x' % (rbyd.block, rbyd.trunk), - t_width, '', - '\x1b[31m' if color else '', - '(corrupted rbyd %s)' % rbyd.addr(), - '\x1b[m' if color else '')) + '%04x.%04x' % (rbyd.block, rbyd.trunk), + t_width, '', + '\x1b[31m' if color else '', + '(corrupted rbyd %s)' % rbyd.addr(), + '\x1b[m' if color else '')) prbyd = rbyd corrupted = True continue @@ -1630,17 +1648,18 @@ def main(disk, mroots=None, *, if name is not None: tags = [name] + [(tag, j, d, data) - for tag, j, d, data in tags - if tag & 0x7f00 != TAG_NAME] + for tag, j, d, data in tags + if tag & 0x7f00 != TAG_NAME] # found an mdir in the tags? mdir__ = None if (not args.get('depth') or mdepth+len(path) < args.get('depth')): - mdir__ = next(((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) + mdir__ = next( + ((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) # show other btree entries in certain cases if args.get('inner') or not mdir__: @@ -1657,13 +1676,14 @@ def main(disk, mroots=None, *, # corrupted? if not mdir_: print('{%s}: %*s%s%s%s' % ( - ','.join('%04x' % block - for block in it.chain([mdir_.block], - mdir_.redund_blocks)), - t_width, '', - '\x1b[31m' if color else '', - '(corrupted mdir %s)' % mdir_.addr(), - '\x1b[m' if color else '')) + ','.join('%04x' % block + for block in it.chain( + [mdir_.block], + mdir_.redund_blocks)), + t_width, '', + '\x1b[31m' if color else '', + '(corrupted mdir %s)' % mdir_.addr(), + '\x1b[m' if color else '')) corrupted = True else: # show the mdir @@ -1680,63 +1700,63 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Debug littlefs's metadata tree.", - allow_abbrev=False) + description="Debug littlefs's metadata tree.", + allow_abbrev=False) parser.add_argument( - 'disk', - help="File containing the block device.") + 'disk', + help="File containing the block device.") parser.add_argument( - 'mroots', - nargs='*', - type=rbydaddr, - help="Block address of the mroots. Defaults to 0x{0,1}.") + 'mroots', + nargs='*', + type=rbydaddr, + help="Block address of the mroots. Defaults to 0x{0,1}.") parser.add_argument( - '-b', '--block-size', - type=bdgeom, - help="Block size/geometry in bytes.") + '-b', '--block-size', + type=bdgeom, + help="Block size/geometry in bytes.") parser.add_argument( - '--block-count', - type=lambda x: int(x, 0), - help="Block count in blocks.") + '--block-count', + type=lambda x: int(x, 0), + help="Block count in blocks.") parser.add_argument( - '--color', - choices=['never', 'always', 'auto'], - default='auto', - help="When to use terminal colors. Defaults to 'auto'.") + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( - '-r', '--raw', - action='store_true', - help="Show the raw data including tag encodings.") + '-r', '--raw', + action='store_true', + help="Show the raw data including tag encodings.") parser.add_argument( - '-T', '--no-truncate', - action='store_true', - help="Don't truncate, show the full contents.") + '-T', '--no-truncate', + action='store_true', + help="Don't truncate, show the full contents.") parser.add_argument( - '-t', '--tree', - action='store_true', - help="Show the underlying rbyd trees.") + '-t', '--tree', + action='store_true', + help="Show the underlying rbyd trees.") parser.add_argument( - '-B', '--btree', - action='store_true', - help="Show the underlying B-trees.") + '-B', '--btree', + action='store_true', + help="Show the underlying B-trees.") parser.add_argument( - '-R', '--rbyd', - action='store_true', - help="Show the full underlying rbyd trees.") + '-R', '--rbyd', + action='store_true', + help="Show the full underlying rbyd trees.") parser.add_argument( - '-i', '--inner', - action='store_true', - help="Show inner branches.") + '-i', '--inner', + action='store_true', + help="Show inner branches.") parser.add_argument( - '-z', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of tree to show.") + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of tree to show.") parser.add_argument( - '-e', '--error-on-corrupt', - action='store_true', - help="Error if the filesystem is corrupt.") + '-e', '--error-on-corrupt', + action='store_true', + help="Error if the filesystem is corrupt.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index feca2ae0..c4cf6b7f 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -7,6 +7,7 @@ import math as mt import os import struct + COLORS = [ '34', # blue '31', # red @@ -156,108 +157,108 @@ def fromtag(data): def xxd(data, width=16): for i in range(0, len(data), width): yield '%-*s %-*s' % ( - 3*width, - ' '.join('%02x' % b for b in data[i:i+width]), - width, - ''.join( - b if b >= ' ' and b <= '~' else '.' - for b in map(chr, data[i:i+width]))) + 3*width, + ' '.join('%02x' % b for b in data[i:i+width]), + width, + ''.join( + b if b >= ' ' and b <= '~' else '.' + for b in map(chr, data[i:i+width]))) def tagrepr(tag, w=None, size=None, off=None): if (tag & 0x6fff) == TAG_NULL: return '%snull%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - ' w%d' % w if w else '', - ' %d' % size if size else '') + 'shrub' if tag & TAG_SHRUB else '', + ' w%d' % w if w else '', + ' %d' % size if size else '') elif (tag & 0x6f00) == TAG_CONFIG: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'magic' if (tag & 0xfff) == TAG_MAGIC - else 'version' if (tag & 0xfff) == TAG_VERSION - else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT - else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT - else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT - else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY - else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT - else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT - else 'config 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'magic' if (tag & 0xfff) == TAG_MAGIC + else 'version' if (tag & 0xfff) == TAG_VERSION + else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT + else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT + else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT + else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY + else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT + else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT + else 'config 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_GDELTA: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA - else 'gdelta 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gdelta 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_NAME: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'name' if (tag & 0xfff) == TAG_NAME - else 'reg' if (tag & 0xfff) == TAG_REG - else 'dir' if (tag & 0xfff) == TAG_DIR - else 'orphan' if (tag & 0xfff) == TAG_ORPHAN - else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK - else 'name 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'name' if (tag & 0xfff) == TAG_NAME + else 'reg' if (tag & 0xfff) == TAG_REG + else 'dir' if (tag & 0xfff) == TAG_DIR + else 'orphan' if (tag & 0xfff) == TAG_ORPHAN + else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK + else 'name 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_STRUCT: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'data' if (tag & 0xfff) == TAG_DATA - else 'block' if (tag & 0xfff) == TAG_BLOCK - else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB - else 'btree' if (tag & 0xfff) == TAG_BTREE - else 'mroot' if (tag & 0xfff) == TAG_MROOT - else 'mdir' if (tag & 0xfff) == TAG_MDIR - else 'mtree' if (tag & 0xfff) == TAG_MTREE - else 'did' if (tag & 0xfff) == TAG_DID - else 'branch' if (tag & 0xfff) == TAG_BRANCH - else 'struct 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'data' if (tag & 0xfff) == TAG_DATA + else 'block' if (tag & 0xfff) == TAG_BLOCK + else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB + else 'btree' if (tag & 0xfff) == TAG_BTREE + else 'mroot' if (tag & 0xfff) == TAG_MROOT + else 'mdir' if (tag & 0xfff) == TAG_MDIR + else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'did' if (tag & 0xfff) == TAG_DID + else 'branch' if (tag & 0xfff) == TAG_BRANCH + else 'struct 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6e00) == TAG_ATTR: return '%s%sattr 0x%02x%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 's' if tag & 0x100 else 'u', - ((tag & 0x100) >> 1) ^ (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 's' if tag & 0x100 else 'u', + ((tag & 0x100) >> 1) ^ (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( - 'r' if tag & TAG_R else 'b', - 'a' if tag & 0x0fff == 0 and tag & TAG_GT - else 'n' if tag & 0x0fff == 0 - else 'gt' if tag & TAG_GT - else 'le', - ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', - ' w%d' % w if w is not None else '', - ' 0x%x' % (0xffffffff & (off-size)) - if size and off is not None - else ' -%d' % size if size - else '') + 'r' if tag & TAG_R else 'b', + 'a' if tag & 0x0fff == 0 and tag & TAG_GT + else 'n' if tag & 0x0fff == 0 + else 'gt' if tag & TAG_GT + else 'le', + ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', + ' w%d' % w if w is not None else '', + ' 0x%x' % (0xffffffff & (off-size)) + if size and off is not None + else ' -%d' % size if size + else '') elif (tag & 0x7f00) == TAG_CKSUM: return 'cksum%s%s%s%s%s' % ( - 'q' if not tag & 0xfc and tag & TAG_Q else '', - 'p' if not tag & 0xfc and tag & TAG_P else '', - ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'q' if not tag & 0xfc and tag & TAG_Q else '', + 'p' if not tag & 0xfc and tag & TAG_P else '', + ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_NOTE: return 'note%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_ECKSUM: return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( - tag, - ' w%d' % w if w is not None else '', - ' %d' % size if size is not None else '') + tag, + ' w%d' % w if w is not None else '', + ' %d' % size if size is not None else '') def dbg_log(data, block_size, rev, eoff, weight, *, @@ -291,9 +292,9 @@ def dbg_log(data, block_size, rev, eoff, weight, *, x = 0 while any( max(a, b) >= min(a_, b_) - and max(a_, b_) >= min(a, b) - and x == x_ - for a_, b_, x_, _ in jumps[:j]): + and max(a_, b_) >= min(a, b) + and x == x_ + for a_, b_, x_, _ in jumps[:j]): x += 1 jumps[j] = a, b, x, c @@ -303,9 +304,9 @@ def dbg_log(data, block_size, rev, eoff, weight, *, for a, b, x, c in jumps: c_start = ( '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color - else '') + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color + else '') c_stop = '\x1b[m' if color else '' if j == a: @@ -321,7 +322,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, chars[2*x+1] = '%s|%s' % (c_start, c_stop) return ''.join(chars.get(x, ' ') - for x in range(max(chars.keys(), default=0)+1)) + for x in range(max(chars.keys(), default=0)+1)) # preprocess lifetimes lifetime_width = 0 @@ -333,7 +334,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, self.tags = set() self.color = COLORS[self.__class__.color_i] self.__class__.color_i = ( - self.__class__.color_i + 1) % len(COLORS) + self.__class__.color_i + 1) % len(COLORS) def add(self, j): self.tags.add(j) @@ -357,8 +358,8 @@ def dbg_log(data, block_size, rev, eoff, weight, *, def checkpoint(j, weights, lifetimes, grows, shrinks, tags): checkpoint_js.append(j) checkpoints.append(( - weights.copy(), lifetimes.copy(), - grows, shrinks, tags)) + weights.copy(), lifetimes.copy(), + grows, shrinks, tags)) lower_, upper_ = 0, 0 weight_ = 0 @@ -397,7 +398,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, if rid_ > 0: weights[i:i+1] = [rid_, delta, weights[i]-rid_] lifetimes[i:i+1] = [ - lifetimes[i], Lifetime(j), lifetimes[i]] + lifetimes[i], Lifetime(j), lifetimes[i]] else: weights[i:i] = [delta] lifetimes[i:i] = [Lifetime(j)] @@ -438,9 +439,9 @@ def dbg_log(data, block_size, rev, eoff, weight, *, checkpoint(j, weights, lifetimes, set(), set(), {i}) lifetime_width = 2*max(( - sum(1 for lifetime in lifetimes if lifetime) - for _, lifetimes, _, _, _ in checkpoints), - default=0) + sum(1 for lifetime in lifetimes if lifetime) + for _, lifetimes, _, _, _ in checkpoints), + default=0) def lifetimerepr(j): x = bisect.bisect(checkpoint_js, j)-1 @@ -476,12 +477,12 @@ def dbg_log(data, block_size, rev, eoff, weight, *, colors.append(lifetime.color) return '%s%*s' % ( - ''.join('%s%s%s' % ( - '\x1b[%sm' % c if color else '', - r, - '\x1b[m' if color else '') - for r, c in zip(reprs, colors)), - lifetime_width - sum(len(r) for r in reprs), '') + ''.join('%s%s%s' % ( + '\x1b[%sm' % c if color else '', + r, + '\x1b[m' if color else '') + for r, c in zip(reprs, colors)), + lifetime_width - sum(len(r) for r in reprs), '') # dynamically size the id field @@ -518,10 +519,10 @@ def dbg_log(data, block_size, rev, eoff, weight, *, # print revision count if args.get('raw'): print('%8s: %*s%*s %s' % ( - '%04x' % 0, - lifetime_width, '', - 2*w_width+1, '', - next(xxd(data[0:4])))) + '%04x' % 0, + lifetime_width, '', + 2*w_width+1, '', + next(xxd(data[0:4])))) # print tags cksum = crc32c(data[0:4]) @@ -582,44 +583,49 @@ def dbg_log(data, block_size, rev, eoff, weight, *, # show human-readable tag representation print('%s%08x:%s %*s%s%*s %-*s%s%s%s' % ( - '\x1b[90m' if color and j >= eoff else '', - j, - '\x1b[m' if color and j >= eoff else '', - lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '', - '\x1b[90m' if color and j >= eoff else '', - 2*w_width+1, '' if (tag & 0xe000) != 0x0000 - else '%d-%d' % (rid-(w-1), rid) if w > 1 - else rid, - 56+w_width, '%-*s %s' % ( - 21+w_width, tagrepr(tag, w, size, j), - next(xxd(data[j+d:j+d+min(size, 8)], 8), '') - if not args.get('raw') and not args.get('no_truncate') - and not tag & TAG_ALT else ''), - ' (%s)' % ', '.join(notes) if notes else '', - '\x1b[m' if color and j >= eoff else '', - ' %s' % jumprepr(j) - if args.get('jumps') and not notes else '')) + '\x1b[90m' if color and j >= eoff else '', + j, + '\x1b[m' if color and j >= eoff else '', + lifetime_width, lifetimerepr(j) + if args.get('lifetimes') + else '', + '\x1b[90m' if color and j >= eoff else '', + 2*w_width+1, '' if (tag & 0xe000) != 0x0000 + else '%d-%d' % (rid-(w-1), rid) if w > 1 + else rid, + 56+w_width, '%-*s %s' % ( + 21+w_width, tagrepr(tag, w, size, j), + next(xxd(data[j+d:j+d+min(size, 8)], 8), '') + if not args.get('raw') + and not args.get('no_truncate') + and not tag & TAG_ALT + else ''), + ' (%s)' % ', '.join(notes) if notes else '', + '\x1b[m' if color and j >= eoff else '', + ' %s' % jumprepr(j) + if args.get('jumps') and not notes + else '')) # show on-disk encoding of tags if args.get('raw'): for o, line in enumerate(xxd(data[j:j+d])): print('%s%8s: %*s%*s %s%s' % ( - '\x1b[90m' if color and j >= eoff else '', - '%04x' % (j + o*16), - lifetime_width, '', - 2*w_width+1, '', - line, - '\x1b[m' if color and j >= eoff else '')) - if args.get('raw') or args.get('no_truncate'): - if not tag & TAG_ALT: - for o, line in enumerate(xxd(data[j+d:j+d+size])): - print('%s%8s: %*s%*s %s%s' % ( '\x1b[90m' if color and j >= eoff else '', - '%04x' % (j+d + o*16), + '%04x' % (j + o*16), lifetime_width, '', 2*w_width+1, '', line, '\x1b[m' if color and j >= eoff else '')) + if args.get('raw') or args.get('no_truncate'): + if not tag & TAG_ALT: + for o, line in enumerate(xxd(data[j+d:j+d+size])): + print('%s%8s: %*s%*s %s%s' % ( + '\x1b[90m' if color and j >= eoff else '', + '%04x' % (j+d + o*16), + lifetime_width, '', + 2*w_width+1, '', + line, + '\x1b[m' if color and j >= eoff else '')) def dbg_tree(data, block_size, rev, trunk, weight, *, @@ -757,8 +763,8 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, else: if 'h' not in alts[j_]: alts[j_]['h'] = max( - rec_height(alts[j_]['f']), - rec_height(alts[j_]['nf'])) + 1 + rec_height(alts[j_]['f']), + rec_height(alts[j_]['nf'])) + 1 return alts[j_]['h'] for j_ in alts.keys(): @@ -801,8 +807,8 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, for branch in tree): return '+-', branch.c, branch.c elif any(branch.d == d - and x > min(branch.a, branch.b) - and x < max(branch.a, branch.b) + and x > min(branch.a, branch.b) + and x < max(branch.a, branch.b) for branch in tree): return '|-', branch.c, branch.c elif branch.a < branch.b: @@ -827,13 +833,13 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, t, c, was = branchrepr((rid, tag), d, was) trunk.append('%s%s%s%s' % ( - '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color and c == 'b' - else '', - t, - ('>' if was else ' ') if d == t_depth-1 else '', - '\x1b[m' if color and c else '')) + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == t_depth-1 else '', + '\x1b[m' if color and c else '')) return '%s ' % ''.join(trunk) @@ -850,33 +856,36 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, # show human-readable tag representation print('%08x: %s%*s %-*s %s' % ( - j, - treerepr(rid, tag) - if args.get('tree') or args.get('rbyd') else '', - 2*w_width+1, '%d-%d' % (rid-(w-1), rid) - if w > 1 else rid - if w > 0 or i == 0 else '', - 21+w_width, tagrepr(tag, w, size, j), - next(xxd(data[j+d:j+d+min(size, 8)], 8), '') - if not args.get('raw') and not args.get('no_truncate') - and not tag & TAG_ALT else '')) + j, + treerepr(rid, tag) + if args.get('tree') or args.get('rbyd') + else '', + 2*w_width+1, '%d-%d' % (rid-(w-1), rid) if w > 1 + else rid if w > 0 or i == 0 + else '', + 21+w_width, tagrepr(tag, w, size, j), + next(xxd(data[j+d:j+d+min(size, 8)], 8), '') + if not args.get('raw') + and not args.get('no_truncate') + and not tag & TAG_ALT + else '')) # show on-disk encoding of tags if args.get('raw'): for o, line in enumerate(xxd(data[j:j+d])): print('%8s: %*s%*s %s' % ( - '%04x' % (j + o*16), - t_width, '', - 2*w_width+1, '', - line)) + '%04x' % (j + o*16), + t_width, '', + 2*w_width+1, '', + line)) if args.get('raw') or args.get('no_truncate'): if not tag & TAG_ALT: for o, line in enumerate(xxd(data[j+d:j+d+size])): print('%8s: %*s%*s %s' % ( - '%04x' % (j+d + o*16), - t_width, '', - 2*w_width+1, '', - line)) + '%04x' % (j+d + o*16), + t_width, '', + 2*w_width+1, '', + line)) def main(disk, blocks=None, *, @@ -912,12 +921,12 @@ def main(disk, blocks=None, *, # blocks may also encode trunks blocks, trunks = ( - [block[0] if isinstance(block, tuple) else block - for block in blocks], - [trunk if trunk is not None - else block[1] if isinstance(block, tuple) - else None - for block in blocks]) + [block[0] if isinstance(block, tuple) else block + for block in blocks], + [trunk if trunk is not None + else block[1] if isinstance(block, tuple) + else None + for block in blocks]) # read each block datas = [] @@ -1020,41 +1029,41 @@ def main(disk, blocks=None, *, # compare with sequence arithmetic if trunk_ and ( not trunks_[i] - or not ((rev - revs[i]) & 0x80000000) - or (rev == revs[i] and trunk_ > trunks_[i])): + or not ((rev - revs[i]) & 0x80000000) + or (rev == revs[i] and trunk_ > trunks_[i])): i = i_ # print contents of the winning metadata block block, data, rev, eoff, trunk_, weight, cksum = ( - blocks[i], - datas[i], - revs[i], - eoffs[i], - trunks_[i], - weights[i], - cksums[i]) + blocks[i], + datas[i], + revs[i], + eoffs[i], + trunks_[i], + weights[i], + cksums[i]) print('rbyd %s w%d, rev %08x, size %d, cksum %08x' % ( - '0x%x.%x' % (block, trunk_) - if len(blocks) == 1 - else '0x{%x,%s}.%x' % ( - block, - ','.join('%x' % blocks[(i+1+j) % len(blocks)] - for j in range(len(blocks)-1)), - trunk_), - weight, - rev, - eoff, - cksum)) + '0x%x.%x' % (block, trunk_) + if len(blocks) == 1 + else '0x{%x,%s}.%x' % ( + block, + ','.join('%x' % blocks[(i+1+j) % len(blocks)] + for j in range(len(blocks)-1)), + trunk_), + weight, + rev, + eoff, + cksum)) if args.get('log'): dbg_log(data, block_size, rev, eoff, weight, - color=color, - **args) + color=color, + **args) else: dbg_tree(data, block_size, rev, trunk_, weight, - color=color, - **args) + color=color, + **args) if args.get('error_on_corrupt') and eoff == 0: sys.exit(2) @@ -1064,69 +1073,69 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Debug rbyd metadata.", - allow_abbrev=False) + description="Debug rbyd metadata.", + allow_abbrev=False) parser.add_argument( - 'disk', - help="File containing the block device.") + 'disk', + help="File containing the block device.") parser.add_argument( - 'blocks', - nargs='*', - type=rbydaddr, - help="Block address of metadata blocks.") + 'blocks', + nargs='*', + type=rbydaddr, + help="Block address of metadata blocks.") parser.add_argument( - '-b', '--block-size', - type=bdgeom, - help="Block size/geometry in bytes.") + '-b', '--block-size', + type=bdgeom, + help="Block size/geometry in bytes.") parser.add_argument( - '--block-count', - type=lambda x: int(x, 0), - help="Block count in blocks.") + '--block-count', + type=lambda x: int(x, 0), + help="Block count in blocks.") parser.add_argument( - '--trunk', - type=lambda x: int(x, 0), - help="Use this offset as the trunk of the tree.") + '--trunk', + type=lambda x: int(x, 0), + help="Use this offset as the trunk of the tree.") parser.add_argument( - '--color', - choices=['never', 'always', 'auto'], - default='auto', - help="When to use terminal colors. Defaults to 'auto'.") + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( - '-a', '--all', - action='store_true', - help="Don't stop parsing on bad commits.") + '-a', '--all', + action='store_true', + help="Don't stop parsing on bad commits.") parser.add_argument( - '-l', '--log', - action='store_true', - help="Show the raw tags as they appear in the log.") + '-l', '--log', + action='store_true', + help="Show the raw tags as they appear in the log.") parser.add_argument( - '-r', '--raw', - action='store_true', - help="Show the raw data including tag encodings.") + '-r', '--raw', + action='store_true', + help="Show the raw data including tag encodings.") parser.add_argument( - '-T', '--no-truncate', - action='store_true', - help="Don't truncate, show the full contents.") + '-T', '--no-truncate', + action='store_true', + help="Don't truncate, show the full contents.") parser.add_argument( - '-t', '--tree', - action='store_true', - help="Show the rbyd tree.") + '-t', '--tree', + action='store_true', + help="Show the rbyd tree.") parser.add_argument( - '-R', '--rbyd', - action='store_true', - help="Show the full rbyd tree.") + '-R', '--rbyd', + action='store_true', + help="Show the full rbyd tree.") parser.add_argument( - '-j', '--jumps', - action='store_true', - help="Show alt pointer jumps in the margin.") + '-j', '--jumps', + action='store_true', + help="Show alt pointer jumps in the margin.") parser.add_argument( - '-g', '--lifetimes', - action='store_true', - help="Show inserts/deletes of ids in the margin.") + '-g', '--lifetimes', + action='store_true', + help="Show inserts/deletes of ids in the margin.") parser.add_argument( - '-e', '--error-on-corrupt', - action='store_true', - help="Error if no valid commit is found.") + '-e', '--error-on-corrupt', + action='store_true', + help="Error if no valid commit is found.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/dbgtag.py b/scripts/dbgtag.py index 73a9341d..0173a273 100755 --- a/scripts/dbgtag.py +++ b/scripts/dbgtag.py @@ -121,98 +121,98 @@ def fromleb128(data): def tagrepr(tag, w=None, size=None, off=None): if (tag & 0x6fff) == TAG_NULL: return '%snull%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - ' w%d' % w if w else '', - ' %d' % size if size else '') + 'shrub' if tag & TAG_SHRUB else '', + ' w%d' % w if w else '', + ' %d' % size if size else '') elif (tag & 0x6f00) == TAG_CONFIG: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'magic' if (tag & 0xfff) == TAG_MAGIC - else 'version' if (tag & 0xfff) == TAG_VERSION - else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT - else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT - else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT - else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY - else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT - else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT - else 'config 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'magic' if (tag & 0xfff) == TAG_MAGIC + else 'version' if (tag & 0xfff) == TAG_VERSION + else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT + else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT + else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT + else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY + else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT + else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT + else 'config 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_GDELTA: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA - else 'gdelta 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gdelta 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_NAME: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'name' if (tag & 0xfff) == TAG_NAME - else 'reg' if (tag & 0xfff) == TAG_REG - else 'dir' if (tag & 0xfff) == TAG_DIR - else 'orphan' if (tag & 0xfff) == TAG_ORPHAN - else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK - else 'name 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'name' if (tag & 0xfff) == TAG_NAME + else 'reg' if (tag & 0xfff) == TAG_REG + else 'dir' if (tag & 0xfff) == TAG_DIR + else 'orphan' if (tag & 0xfff) == TAG_ORPHAN + else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK + else 'name 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_STRUCT: return '%s%s%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 'data' if (tag & 0xfff) == TAG_DATA - else 'block' if (tag & 0xfff) == TAG_BLOCK - else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB - else 'btree' if (tag & 0xfff) == TAG_BTREE - else 'mroot' if (tag & 0xfff) == TAG_MROOT - else 'mdir' if (tag & 0xfff) == TAG_MDIR - else 'mtree' if (tag & 0xfff) == TAG_MTREE - else 'did' if (tag & 0xfff) == TAG_DID - else 'branch' if (tag & 0xfff) == TAG_BRANCH - else 'struct 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 'data' if (tag & 0xfff) == TAG_DATA + else 'block' if (tag & 0xfff) == TAG_BLOCK + else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB + else 'btree' if (tag & 0xfff) == TAG_BTREE + else 'mroot' if (tag & 0xfff) == TAG_MROOT + else 'mdir' if (tag & 0xfff) == TAG_MDIR + else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'did' if (tag & 0xfff) == TAG_DID + else 'branch' if (tag & 0xfff) == TAG_BRANCH + else 'struct 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x6e00) == TAG_ATTR: return '%s%sattr 0x%02x%s%s' % ( - 'shrub' if tag & TAG_SHRUB else '', - 's' if tag & 0x100 else 'u', - ((tag & 0x100) >> 1) ^ (tag & 0xff), - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'shrub' if tag & TAG_SHRUB else '', + 's' if tag & 0x100 else 'u', + ((tag & 0x100) >> 1) ^ (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( - 'r' if tag & TAG_R else 'b', - 'a' if tag & 0x0fff == 0 and tag & TAG_GT - else 'n' if tag & 0x0fff == 0 - else 'gt' if tag & TAG_GT - else 'le', - ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', - ' w%d' % w if w is not None else '', - ' 0x%x' % (0xffffffff & (off-size)) - if size and off is not None - else ' -%d' % size if size - else '') + 'r' if tag & TAG_R else 'b', + 'a' if tag & 0x0fff == 0 and tag & TAG_GT + else 'n' if tag & 0x0fff == 0 + else 'gt' if tag & TAG_GT + else 'le', + ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', + ' w%d' % w if w is not None else '', + ' 0x%x' % (0xffffffff & (off-size)) + if size and off is not None + else ' -%d' % size if size + else '') elif (tag & 0x7f00) == TAG_CKSUM: return 'cksum%s%s%s%s%s' % ( - 'q' if not tag & 0xfc and tag & TAG_Q else '', - 'p' if not tag & 0xfc and tag & TAG_P else '', - ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + 'q' if not tag & 0xfc and tag & TAG_Q else '', + 'p' if not tag & 0xfc and tag & TAG_P else '', + ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_NOTE: return 'note%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_ECKSUM: return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( - tag, - ' w%d' % w if w is not None else '', - ' %d' % size if size is not None else '') + tag, + ' w%d' % w if w is not None else '', + ' %d' % size if size is not None else '') def list_tags(): @@ -221,7 +221,7 @@ def list_tags(): import re tags = [] tag_pattern = re.compile( - '^(?PTAG_[^ ]*) *= *(?P[^ #]+) *#+ *(?P.*)$') + '^(?PTAG_[^ ]*) *= *(?P[^ #]+) *#+ *(?P.*)$') for line in inspect.getsourcelines( inspect.getmodule(inspect.currentframe()))[0]: m = tag_pattern.match(line) @@ -236,8 +236,8 @@ def list_tags(): # print for n, t, c in tags: print('%-*s %s' % ( - w[0], 'LFSR_'+n, - c)) + w[0], 'LFSR_'+n, + c)) def dbg_tag(data): if isinstance(data, int): @@ -305,12 +305,12 @@ def main(tags, *, # blocks may also encode offsets blocks, offs = ( - [block[0] if isinstance(block, tuple) else block - for block in blocks], - [off if off is not None - else block[1] if isinstance(block, tuple) - else None - for block in blocks]) + [block[0] if isinstance(block, tuple) else block + for block in blocks], + [off if off is not None + else block[1] if isinstance(block, tuple) + else None + for block in blocks]) # read each tag for block, off in zip(blocks, offs): @@ -319,40 +319,41 @@ def main(tags, *, data = f.read(2+5+5) dbg_tag(data) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Decode littlefs tags.", - allow_abbrev=False) + description="Decode littlefs tags.", + allow_abbrev=False) parser.add_argument( - 'tags', - nargs='*', - help="Tags to decode.") + 'tags', + nargs='*', + help="Tags to decode.") parser.add_argument( - '-l', '--list', - action='store_true', - help="List all known tags.") + '-l', '--list', + action='store_true', + help="List all known tags.") parser.add_argument( - '-x', '--hex', - action='store_true', - help="Interpret as a sequence of hex bytes.") + '-x', '--hex', + action='store_true', + help="Interpret as a sequence of hex bytes.") parser.add_argument( - '-s', '--string', - action='store_true', - help="Interpret as strings.") + '-s', '--string', + action='store_true', + help="Interpret as strings.") parser.add_argument( - '-b', '--block-size', - type=bdgeom, - help="Block size/geometry in bytes.") + '-b', '--block-size', + type=bdgeom, + help="Block size/geometry in bytes.") parser.add_argument( - '--block-count', - type=lambda x: int(x, 0), - help="Block count in blocks.") + '--block-count', + type=lambda x: int(x, 0), + help="Block count in blocks.") parser.add_argument( - '--off', - type=lambda x: int(x, 0), - help="Use this offset.") + '--off', + type=lambda x: int(x, 0), + help="Use this offset.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/parity.py b/scripts/parity.py index 0ae3932e..468c3dc0 100755 --- a/scripts/parity.py +++ b/scripts/parity.py @@ -29,17 +29,17 @@ def main(paths, **args): # interpret as sequence of hex bytes if args.get('hex'): print('%01x' % parity(ft.reduce( - op.xor, - bytes(int(path, 16) for path in paths), - 0))) + op.xor, + bytes(int(path, 16) for path in paths), + 0))) # interpret as strings elif args.get('string'): for path in paths: print('%01x' % parity(ft.reduce( - op.xor, - path.encode('utf8'), - 0))) + op.xor, + path.encode('utf8'), + 0))) # default to interpreting as paths else: @@ -63,24 +63,25 @@ def main(paths, **args): else: print('%01x' % parity(xor)) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Calculates parity.", - allow_abbrev=False) + description="Calculates parity.", + allow_abbrev=False) parser.add_argument( - 'paths', - nargs='*', - help="Paths to read. Reads stdin by default.") + 'paths', + nargs='*', + help="Paths to read. Reads stdin by default.") parser.add_argument( - '-x', '--hex', - action='store_true', - help="Interpret as a sequence of hex bytes.") + '-x', '--hex', + action='store_true', + help="Interpret as a sequence of hex bytes.") parser.add_argument( - '-s', '--string', - action='store_true', - help="Interpret as strings.") + '-s', '--string', + action='store_true', + help="Interpret as strings.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/perf.py b/scripts/perf.py index b277918d..69c1582f 100755 --- a/scripts/perf.py +++ b/scripts/perf.py @@ -27,8 +27,8 @@ import subprocess as sp import tempfile import zipfile -# TODO support non-zip perf results? +# TODO support non-zip perf results? PERF_PATH = ['perf'] PERF_EVENTS = 'cycles,branch-misses,branches,cache-misses,cache-references' @@ -121,28 +121,28 @@ class PerfResult(co.namedtuple('PerfResult', [ _fields = ['cycles', 'bmisses', 'branches', 'cmisses', 'caches'] _sort = ['cycles', 'bmisses', 'cmisses', 'branches', 'caches'] _types = { - 'cycles': RInt, - 'bmisses': RInt, 'branches': RInt, - 'cmisses': RInt, 'caches': RInt} + 'cycles': RInt, + 'bmisses': RInt, 'branches': RInt, + 'cmisses': RInt, 'caches': RInt} __slots__ = () def __new__(cls, file='', function='', line=0, cycles=0, bmisses=0, branches=0, cmisses=0, caches=0, children=[]): return super().__new__(cls, file, function, int(RInt(line)), - RInt(cycles), - RInt(bmisses), RInt(branches), - RInt(cmisses), RInt(caches), - children) + RInt(cycles), + RInt(bmisses), RInt(branches), + RInt(cmisses), RInt(caches), + children) def __add__(self, other): return PerfResult(self.file, self.function, self.line, - self.cycles + other.cycles, - self.bmisses + other.bmisses, - self.branches + other.branches, - self.cmisses + other.cmisses, - self.caches + other.caches, - self.children + other.children) + self.cycles + other.cycles, + self.bmisses + other.bmisses, + self.branches + other.branches, + self.cmisses + other.cmisses, + self.caches + other.caches, + self.children + other.children) def openio(path, mode='r', buffering=-1): @@ -168,17 +168,17 @@ def record(command, *, with tempfile.NamedTemporaryFile('rb') as f: # figure out our perf invocation perf = perf_path + list(filter(None, [ - 'record', - '-F%s' % perf_freq - if perf_freq is not None - and perf_period is None else None, - '-c%s' % perf_period - if perf_period is not None else None, - '-B', - '-g', - '--all-user', - '-e%s' % perf_events, - '-o%s' % f.name])) + 'record', + '-F%s' % perf_freq + if perf_freq is not None + and perf_period is None else None, + '-c%s' % perf_period + if perf_period is not None else None, + '-B', + '-g', + '--all-user', + '-e%s' % perf_events, + '-o%s' % f.name])) # run our command try: @@ -237,27 +237,27 @@ def collect_syms_and_lines(obj_path, *, objdump_path=None, **args): symbol_pattern = re.compile( - '^(?P[0-9a-fA-F]+)' - '\s+.*' - '\s+(?P[0-9a-fA-F]+)' - '\s+(?P[^\s]+)\s*$') - line_pattern = re.compile( - '^\s+(?:' - # matches dir/file table - '(?P[0-9]+)' - '(?:\s+(?P[0-9]+))?' + '^(?P[0-9a-fA-F]+)' '\s+.*' - '\s+(?P[^\s]+)' - # matches line opcodes - '|' '\[[^\]]*\]\s+' - '(?:' + '\s+(?P[0-9a-fA-F]+)' + '\s+(?P[^\s]+)\s*$') + line_pattern = re.compile( + '^\s+(?:' + # matches dir/file table + '(?P[0-9]+)' + '(?:\s+(?P[0-9]+))?' + '\s+.*' + '\s+(?P[^\s]+)' + # matches line opcodes + '|' '\[[^\]]*\]\s+' '(?:' '(?PSpecial)' '|' '(?PCopy)' '|' '(?PEnd of Sequence)' '|' 'File .*?to (?:entry )?(?P\d+)' '|' 'Line .*?to (?P[0-9]+)' '|' '(?:Address|PC) .*?to (?P[0x0-9a-fA-F]+)' - '|' '.' ')*' + '|' '.' + ')*' ')$', re.IGNORECASE) # figure out symbol addresses and file+line ranges @@ -267,11 +267,11 @@ def collect_syms_and_lines(obj_path, *, if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) for line in proc.stdout: m = symbol_pattern.match(line) if m: @@ -316,11 +316,11 @@ def collect_syms_and_lines(obj_path, *, if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=None if args.get('verbose') else sp.DEVNULL, - universal_newlines=True, - errors='replace', - close_fds=False) + stdout=sp.PIPE, + stderr=None if args.get('verbose') else sp.DEVNULL, + universal_newlines=True, + errors='replace', + close_fds=False) for line in proc.stdout: m = line_pattern.match(line) if m: @@ -332,8 +332,8 @@ def collect_syms_and_lines(obj_path, *, dir = int(m.group('dir')) if dir in dirs: files[int(m.group('no'))] = os.path.join( - dirs[dir], - m.group('path')) + dirs[dir], + m.group('path')) else: files[int(m.group('no'))] = m.group('path') else: @@ -391,26 +391,26 @@ def collect_decompressed(path, *, depth=1, **args): sample_pattern = re.compile( - '(?P\w+)' - '\s+(?P\w+)' - '\s+(?P