MagnitudeOS/littlefs
1
0
Fork 0
forked from Imagelibrary/littlefs
Code Pull Requests Activity

Tweaked scripts to share more code, added coverage calls/hits

Browse Source 
The main change is requiring field names for -b/-f/-s/-S, this
is a bit more powerful, and supports hidden extra fields, but
can require a bit more typing in some cases.
This commit is contained in:
Christopher Haster
2022-10-02 03:07:30 -05:00
parent 296c5afea7
commit ca66993812
8 changed files with 1972 additions and 1454 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Makefile
View File

@@ -149,23 +149,23 @@ bench-list: bench-runner
.PHONY: code .PHONY: code
code: $(OBJ) code: $(OBJ)
./scripts/code.py $^ -S $(CODEFLAGS) ./scripts/code.py $^ -Ssize $(CODEFLAGS)
.PHONY: data .PHONY: data
data: $(OBJ) data: $(OBJ)
./scripts/data.py $^ -S $(DATAFLAGS) ./scripts/data.py $^ -Ssize $(DATAFLAGS)
.PHONY: stack .PHONY: stack
stack: $(CI) stack: $(CI)
./scripts/stack.py $^ -S $(STACKFLAGS) ./scripts/stack.py $^ -Slimit -Sframe $(STACKFLAGS)
.PHONY: struct .PHONY: struct
struct: $(OBJ) struct: $(OBJ)
./scripts/struct_.py $^ -S $(STRUCTFLAGS) ./scripts/struct_.py $^ -Ssize $(STRUCTFLAGS)
.PHONY: coverage .PHONY: coverage
coverage: $(GCDA) coverage: $(GCDA)
./scripts/coverage.py $^ -s $(COVERAGEFLAGS) ./scripts/coverage.py $^ -slines -sbranches $(COVERAGEFLAGS)
.PHONY: summary sizes .PHONY: summary sizes
summary sizes: $(BUILDDIR)lfs.csv summary sizes: $(BUILDDIR)lfs.csv

415
scripts/code.py
View File

@@ -29,10 +29,10 @@ TYPE = 'tTrRdD'
# integer fields # integer fields
class IntField(co.namedtuple('IntField', 'x')): class Int(co.namedtuple('Int', 'x')):
__slots__ = () __slots__ = ()
def __new__(cls, x=0): def __new__(cls, x=0):
if isinstance(x, IntField): if isinstance(x, Int):
return x return x
if isinstance(x, str): if isinstance(x, str):
try: try:
@@ -98,35 +98,30 @@ class IntField(co.namedtuple('IntField', 'x')):
return (new-old) / old return (new-old) / old
def __add__(self, other): def __add__(self, other):
return IntField(self.x + other.x) return self.__class__(self.x + other.x)
def __sub__(self, other): def __sub__(self, other):
return IntField(self.x - other.x) return self.__class__(self.x - other.x)
def __mul__(self, other): def __mul__(self, other):
return IntField(self.x * other.x) return self.__class__(self.x * other.x)
def __lt__(self, other):
return self.x < other.x
def __gt__(self, other):
return self.__class__.__lt__(other, self)
def __le__(self, other):
return not self.__gt__(other)
def __ge__(self, other):
return not self.__lt__(other)
# code size results # code size results
class CodeResult(co.namedtuple('CodeResult', 'file,function,code_size')): class CodeResult(co.namedtuple('CodeResult', [
'file', 'function',
'size'])):
_by = ['file', 'function']
_fields = ['size']
_types = {'size': Int}
__slots__ = () __slots__ = ()
def __new__(cls, file, function, code_size): def __new__(cls, file='', function='', size=0):
return super().__new__(cls, file, function, IntField(code_size)) return super().__new__(cls, file, function,
Int(size))
def __add__(self, other): def __add__(self, other):
return CodeResult(self.file, self.function, return CodeResult(self.file, self.function,
self.code_size + other.code_size) self.size + other.size)
def openio(path, mode='r'): def openio(path, mode='r'):
@@ -188,9 +183,27 @@ def collect(paths, *,
return results return results
def fold(results, *, def fold(Result, results, *,
by=['file', 'function'], by=None,
defines=None,
**_): **_):
if by is None:
by = Result._by
for k in it.chain(by or [], (k for k, _ in defines or [])):
if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k)
sys.exit(-1)
# filter by matching defines
if defines is not None:
results_ = []
for r in results:
if all(getattr(r, k) in vs for k, vs in defines):
results_.append(r)
results = results_
# organize results into conflicts
folding = co.OrderedDict() folding = co.OrderedDict()
for r in results: for r in results:
name = tuple(getattr(r, k) for k in by) name = tuple(getattr(r, k) for k in by)
@@ -198,157 +211,220 @@ def fold(results, *,
folding[name] = [] folding[name] = []
folding[name].append(r) folding[name].append(r)
# merge conflicts
folded = [] folded = []
for rs in folding.values(): for name, rs in folding.items():
folded.append(sum(rs[1:], start=rs[0])) folded.append(sum(rs[1:], start=rs[0]))
return folded return folded
def table(Result, results, diff_results=None, *,
def table(results, diff_results=None, *, by=None,
by_file=False, fields=None,
size_sort=False, sort=None,
reverse_size_sort=False,
summary=False, summary=False,
all=False, all=False,
percent=False, percent=False,
**_): **_):
all_, all = all, __builtins__.all all_, all = all, __builtins__.all
# fold if by is None:
results = fold(results, by=['file' if by_file else 'function']) by = Result._by
if diff_results is not None: if fields is None:
diff_results = fold(diff_results, fields = Result._fields
by=['file' if by_file else 'function']) types = Result._types
# fold again
results = fold(Result, results, by=by)
if diff_results is not None:
diff_results = fold(Result, diff_results, by=by)
# organize by name
table = { table = {
r.file if by_file else r.function: r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
r.file if by_file else r.function: r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
# sort, note that python's sort is stable
names = list(table.keys() | diff_table.keys()) names = list(table.keys() | diff_table.keys())
# sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: -IntField.ratio( names.sort(key=lambda n: tuple(
table[n].code_size if n in table else None, types[k].ratio(
diff_table[n].code_size if n in diff_table else None)) getattr(table.get(n), k, None),
if size_sort: getattr(diff_table.get(n), k, None))
names.sort(key=lambda n: (table[n].code_size,) if n in table else (), for k in fields),
reverse=True) reverse=True)
elif reverse_size_sort: if sort:
names.sort(key=lambda n: (table[n].code_size,) if n in table else (), for k, reverse in reversed(sort):
reverse=False) names.sort(key=lambda n: (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else (),
reverse=reverse ^ (not k or k in Result._fields))
# print header
if not summary: # build up our lines
title = '%s%s' % ( lines = []
'file' if by_file else 'function',
# header
line = []
line.append('%s%s' % (
','.join(by),
' (%d added, %d removed)' % ( ' (%d added, %d removed)' % (
sum(1 for n in table if n not in diff_table), sum(1 for n in table if n not in diff_table),
sum(1 for n in diff_table if n not in table)) sum(1 for n in diff_table if n not in table))
if diff_results is not None and not percent else '') if diff_results is not None and not percent else '')
name_width = max(it.chain([23, len(title)], (len(n) for n in names))) if not summary else '')
else:
title = ''
name_width = 23
name_width = 4*((name_width+1+4-1)//4)-1
print('%-*s ' % (name_width, title), end='')
if diff_results is None: if diff_results is None:
print(' %s' % ('size'.rjust(len(IntField.none)))) for k in fields:
line.append(k)
elif percent: elif percent:
print(' %s' % ('size'.rjust(len(IntField.diff_none)))) for k in fields:
line.append(k)
else: else:
print(' %s %s %s' % ( for k in fields:
'old'.rjust(len(IntField.diff_none)), line.append('o'+k)
'new'.rjust(len(IntField.diff_none)), for k in fields:
'diff'.rjust(len(IntField.diff_none)))) line.append('n'+k)
for k in fields:
line.append('d'+k)
line.append('')
lines.append(line)
# print entries # entries
if not summary: if not summary:
for name in names: for name in names:
r = table.get(name) r = table.get(name)
if diff_results is not None: if diff_results is not None:
diff_r = diff_table.get(name) diff_r = diff_table.get(name)
ratio = IntField.ratio( ratios = [
r.code_size if r else None, types[k].ratio(
diff_r.code_size if diff_r else None) getattr(r, k, None),
if not ratio and not all_: getattr(diff_r, k, None))
for k in fields]
if not any(ratios) and not all_:
continue continue
print('%-*s ' % (name_width, name), end='') line = []
line.append(name)
if diff_results is None: if diff_results is None:
print(' %s' % ( for k in fields:
r.code_size.table() line.append(getattr(r, k).table()
if r else IntField.none)) if getattr(r, k, None) is not None
else types[k].none)
elif percent: elif percent:
print(' %s%s' % ( for k in fields:
r.code_size.diff_table() line.append(getattr(r, k).diff_table()
if r else IntField.diff_none, if getattr(r, k, None) is not None
' (%s)' % ( else types[k].diff_none)
'+∞%' if ratio == +m.inf
else '-∞%' if ratio == -m.inf
else '%+.1f%%' % (100*ratio))))
else: else:
print(' %s %s %s%s' % ( for k in fields:
diff_r.code_size.diff_table() line.append(getattr(diff_r, k).diff_table()
if diff_r else IntField.diff_none, if getattr(diff_r, k, None) is not None
r.code_size.diff_table() else types[k].diff_none)
if r else IntField.diff_none, for k in fields:
IntField.diff_diff( line.append(getattr(r, k).diff_table()
r.code_size if r else None, if getattr(r, k, None) is not None
diff_r.code_size if diff_r else None) else types[k].diff_none)
if r or diff_r else IntField.diff_none, for k in fields:
' (%s)' % ( line.append(types[k].diff_diff(
'+∞%' if ratio == +m.inf getattr(r, k, None),
else '-∞%' if ratio == -m.inf getattr(diff_r, k, None)))
else '%+.1f%%' % (100*ratio)) if diff_results is None:
if ratio else '')) line.append('')
elif percent:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios))
else:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t)
if any(ratios) else '')
lines.append(line)
# print total # total
total = fold(results, by=[]) r = next(iter(fold(Result, results, by=[])), None)
r = total[0] if total else None
if diff_results is not None: if diff_results is not None:
diff_total = fold(diff_results, by=[]) diff_r = next(iter(fold(Result, diff_results, by=[])), None)
diff_r = diff_total[0] if diff_total else None ratios = [
ratio = IntField.ratio( types[k].ratio(
r.code_size if r else None, getattr(r, k, None),
diff_r.code_size if diff_r else None) getattr(diff_r, k, None))
for k in fields]
print('%-*s ' % (name_width, 'TOTAL'), end='') line = []
line.append('TOTAL')
if diff_results is None: if diff_results is None:
print(' %s' % ( for k in fields:
r.code_size.table() line.append(getattr(r, k).table()
if r else IntField.none)) if getattr(r, k, None) is not None
else types[k].none)
elif percent: elif percent:
print(' %s%s' % ( for k in fields:
r.code_size.diff_table() line.append(getattr(r, k).diff_table()
if r else IntField.diff_none, if getattr(r, k, None) is not None
' (%s)' % ( else types[k].diff_none)
'+∞%' if ratio == +m.inf
else '-∞%' if ratio == -m.inf
else '%+.1f%%' % (100*ratio))))
else: else:
print(' %s %s %s%s' % ( for k in fields:
diff_r.code_size.diff_table() line.append(getattr(diff_r, k).diff_table()
if diff_r else IntField.diff_none, if getattr(diff_r, k, None) is not None
r.code_size.diff_table() else types[k].diff_none)
if r else IntField.diff_none, for k in fields:
IntField.diff_diff( line.append(getattr(r, k).diff_table()
r.code_size if r else None, if getattr(r, k, None) is not None
diff_r.code_size if diff_r else None) else types[k].diff_none)
if r or diff_r else IntField.diff_none, for k in fields:
' (%s)' % ( line.append(types[k].diff_diff(
'+∞%' if ratio == +m.inf getattr(r, k, None),
else '-∞%' if ratio == -m.inf getattr(diff_r, k, None)))
else '%+.1f%%' % (100*ratio)) if diff_results is None:
if ratio else '')) line.append('')
elif percent:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios))
else:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t)
if any(ratios) else '')
lines.append(line)
# find the best widths, note that column 0 contains the names and column -1
# the ratios, so those are handled a bit differently
widths = [
((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
for w, i in zip(
it.chain([23], it.repeat(7)),
range(len(lines[0])-1))]
# print our table
for line in lines:
print('%-*s %s%s' % (
widths[0], line[0],
' '.join('%*s' % (w, x)
for w, x in zip(widths[1:], line[1:-1])),
line[-1]))
def main(obj_paths, **args): def main(obj_paths, *,
by=None,
fields=None,
defines=None,
sort=None,
**args):
# find sizes # find sizes
if not args.get('use', None): if not args.get('use', None):
# find .o files # find .o files
@@ -361,7 +437,7 @@ def main(obj_paths, **args):
paths.append(path) paths.append(path)
if not paths: if not paths:
print('no .obj files found in %r?' % obj_paths) print("error: no .obj files found in %r?" % obj_paths)
sys.exit(-1) sys.exit(-1)
results = collect(paths, **args) results = collect(paths, **args)
@@ -371,25 +447,35 @@ def main(obj_paths, **args):
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
results.append(CodeResult(**{ results.append(CodeResult(
k: v for k, v in r.items() **{k: r[k] for k in CodeResult._by
if k in CodeResult._fields})) if k in r and r[k].strip()},
**{k: r['code_'+k] for k in CodeResult._fields
if 'code_'+k in r and r['code_'+k].strip()}))
except TypeError: except TypeError:
pass pass
# fold to remove duplicates # fold
results = fold(results) results = fold(CodeResult, results, by=by, defines=defines)
# sort because why not # sort, note that python's sort is stable
results.sort() results.sort()
if sort:
for k, reverse in reversed(sort):
results.sort(key=lambda r: (getattr(r, k),)
if getattr(r, k) is not None else (),
reverse=reverse ^ (not k or k in CodeResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, CodeResult._fields) writer = csv.DictWriter(f, CodeResult._by
+ ['code_'+k for k in CodeResult._fields])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow(r._asdict()) writer.writerow(
{k: getattr(r, k) for k in CodeResult._by}
| {'code_'+k: getattr(r, k) for k in CodeResult._fields})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -399,22 +485,26 @@ def main(obj_paths, **args):
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
diff_results.append(CodeResult(**{ diff_results.append(CodeResult(
k: v for k, v in r.items() **{k: r[k] for k in CodeResult._by
if k in CodeResult._fields})) if k in r and r[k].strip()},
**{k: r['code_'+k] for k in CodeResult._fields
if 'code_'+k in r and r['code_'+k].strip()}))
except TypeError: except TypeError:
pass pass
except FileNotFoundError: except FileNotFoundError:
pass pass
# fold to remove duplicates # fold
diff_results = fold(diff_results) diff_results = fold(CodeResult, diff_results, by=by, defines=defines)
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
table( table(CodeResult, results,
results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['function'],
fields=fields,
sort=sort,
**args) **args)
@@ -455,22 +545,39 @@ if __name__ == "__main__":
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by-file', '-b', '--by',
action='store_true', action='append',
help="Group by file. Note this does not include padding " choices=CodeResult._by,
"so sizes may differ from other tools.") help="Group by this field.")
parser.add_argument( parser.add_argument(
'-s', '--size-sort', '-f', '--field',
action='store_true', dest='fields',
help="Sort by size.") action='append',
choices=CodeResult._fields,
help="Show this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-size-sort', '-D', '--define',
action='store_true', dest='defines',
help="Sort by size, but backwards.") action='append',
type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
help="Only include results where this field is this value.")
class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option):
if namespace.sort is None:
namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument(
'-s', '--sort',
action=AppendSort,
help="Sort by this fields.")
parser.add_argument(
'-S', '--reverse-sort',
action=AppendSort,
help="Sort by this fields, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total size.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'-A', '--everything', '-A', '--everything',
action='store_true', action='store_true',

612
scripts/coverage.py
View File

@@ -30,10 +30,10 @@ GCOV_TOOL = ['gcov']
# integer fields # integer fields
class IntField(co.namedtuple('IntField', 'x')): class Int(co.namedtuple('Int', 'x')):
__slots__ = () __slots__ = ()
def __new__(cls, x=0): def __new__(cls, x=0):
if isinstance(x, IntField): if isinstance(x, Int):
return x return x
if isinstance(x, str): if isinstance(x, str):
try: try:
@@ -99,37 +99,25 @@ class IntField(co.namedtuple('IntField', 'x')):
return (new-old) / old return (new-old) / old
def __add__(self, other): def __add__(self, other):
return IntField(self.x + other.x) return self.__class__(self.x + other.x)
def __sub__(self, other): def __sub__(self, other):
return IntField(self.x - other.x) return self.__class__(self.x - other.x)
def __mul__(self, other): def __mul__(self, other):
return IntField(self.x * other.x) return self.__class__(self.x * other.x)
def __lt__(self, other):
return self.x < other.x
def __gt__(self, other):
return self.__class__.__lt__(other, self)
def __le__(self, other):
return not self.__gt__(other)
def __ge__(self, other):
return not self.__lt__(other)
# fractional fields, a/b # fractional fields, a/b
class FracField(co.namedtuple('FracField', 'a,b')): class Frac(co.namedtuple('Frac', 'a,b')):
__slots__ = () __slots__ = ()
def __new__(cls, a=0, b=None): def __new__(cls, a=0, b=None):
if isinstance(a, FracField) and b is None: if isinstance(a, Frac) and b is None:
return a return a
if isinstance(a, str) and b is None: if isinstance(a, str) and b is None:
a, b = a.split('/', 1) a, b = a.split('/', 1)
if b is None: if b is None:
b = a b = a
return super().__new__(cls, IntField(a), IntField(b)) return super().__new__(cls, Int(a), Int(b))
def __str__(self): def __str__(self):
return '%s/%s' % (self.a, self.b) return '%s/%s' % (self.a, self.b)
@@ -139,10 +127,7 @@ class FracField(co.namedtuple('FracField', 'a,b')):
none = '%11s %7s' % ('-', '-') none = '%11s %7s' % ('-', '-')
def table(self): def table(self):
if not self.b.x: t = self.a.x/self.b.x if self.b.x else 1.0
return self.none
t = self.a.x/self.b.x
return '%11s %7s' % ( return '%11s %7s' % (
self, self,
'∞%' if t == +m.inf '∞%' if t == +m.inf
@@ -151,38 +136,35 @@ class FracField(co.namedtuple('FracField', 'a,b')):
diff_none = '%11s' % '-' diff_none = '%11s' % '-'
def diff_table(self): def diff_table(self):
if not self.b.x:
return self.diff_none
return '%11s' % (self,) return '%11s' % (self,)
def diff_diff(self, other): def diff_diff(self, other):
new_a, new_b = self if self else (IntField(0), IntField(0)) new_a, new_b = self if self else (Int(0), Int(0))
old_a, old_b = other if other else (IntField(0), IntField(0)) old_a, old_b = other if other else (Int(0), Int(0))
return '%11s' % ('%s/%s' % ( return '%11s' % ('%s/%s' % (
new_a.diff_diff(old_a).strip(), new_a.diff_diff(old_a).strip(),
new_b.diff_diff(old_b).strip())) new_b.diff_diff(old_b).strip()))
def ratio(self, other): def ratio(self, other):
new_a, new_b = self if self else (IntField(0), IntField(0)) new_a, new_b = self if self else (Int(0), Int(0))
old_a, old_b = other if other else (IntField(0), IntField(0)) old_a, old_b = other if other else (Int(0), Int(0))
new = new_a.x/new_b.x if new_b.x else 1.0 new = new_a.x/new_b.x if new_b.x else 1.0
old = old_a.x/old_b.x if old_b.x else 1.0 old = old_a.x/old_b.x if old_b.x else 1.0
return new - old return new - old
def __add__(self, other): def __add__(self, other):
return FracField(self.a + other.a, self.b + other.b) return self.__class__(self.a + other.a, self.b + other.b)
def __sub__(self, other): def __sub__(self, other):
return FracField(self.a - other.a, self.b - other.b) return self.__class__(self.a - other.a, self.b - other.b)
def __mul__(self, other): def __mul__(self, other):
return FracField(self.a * other.a, self.b + other.b) return self.__class__(self.a * other.a, self.b + other.b)
def __lt__(self, other): def __lt__(self, other):
self_r = self.a.x/self.b.x if self.b.x else -m.inf self_t = self.a.x/self.b.x if self.b.x else 1.0
other_r = other.a.x/other.b.x if other.b.x else -m.inf other_t = other.a.x/other.b.x if other.b.x else 1.0
return self_r < other_r return (self_t, self.a.x) < (other_t, other.a.x)
def __gt__(self, other): def __gt__(self, other):
return self.__class__.__lt__(other, self) return self.__class__.__lt__(other, self)
@@ -194,22 +176,28 @@ class FracField(co.namedtuple('FracField', 'a,b')):
return not self.__lt__(other) return not self.__lt__(other)
# coverage results # coverage results
class CoverageResult(co.namedtuple('CoverageResult', class CoverageResult(co.namedtuple('CoverageResult', [
'file,function,line,' 'file', 'function', 'line',
'coverage_hits,coverage_lines,coverage_branches')): 'calls', 'hits', 'funcs', 'lines', 'branches'])):
_by = ['file', 'function', 'line']
_fields = ['calls', 'hits', 'funcs', 'lines', 'branches']
_types = {
'calls': Int, 'hits': Int,
'funcs': Frac, 'lines': Frac, 'branches': Frac}
__slots__ = () __slots__ = ()
def __new__(cls, file, function, line, def __new__(cls, file='', function='', line=0,
coverage_hits, coverage_lines, coverage_branches): calls=0, hits=0, funcs=0, lines=0, branches=0):
return super().__new__(cls, file, function, int(IntField(line)), return super().__new__(cls, file, function, int(Int(line)),
IntField(coverage_hits), Int(calls), Int(hits), Frac(funcs), Frac(lines), Frac(branches))
FracField(coverage_lines),
FracField(coverage_branches))
def __add__(self, other): def __add__(self, other):
return CoverageResult(self.file, self.function, self.line, return CoverageResult(self.file, self.function, self.line,
max(self.coverage_hits, other.coverage_hits), max(self.calls, other.calls),
self.coverage_lines + other.coverage_lines, max(self.hits, other.hits),
self.coverage_branches + other.coverage_branches) self.funcs + other.funcs,
self.lines + other.lines,
self.branches + other.branches)
def openio(path, mode='r'): def openio(path, mode='r'):
@@ -257,20 +245,37 @@ def collect(paths, *,
if file['file'] != src_path: if file['file'] != src_path:
continue continue
for line in file['lines']: for func in file['functions']:
func = line.get('function_name', '(inlined)') func_name = func.get('name', '(inlined)')
# discard internal function (this includes injected test cases) # discard internal function (this includes injected test cases)
if not everything: if not everything:
if func.startswith('__'): if func_name.startswith('__'):
continue continue
# go ahead and add functions, later folding will merge this if
# there are other hits on this line
results.append(CoverageResult( results.append(CoverageResult(
src_path, func, line['line_number'], src_path, func_name, func['start_line'],
line['count'], func['execution_count'], 0,
FracField( Frac(1 if func['execution_count'] > 0 else 0, 1),
1 if line['count'] > 0 else 0, 0,
1), 0))
FracField(
for line in file['lines']:
func_name = line.get('function_name', '(inlined)')
# discard internal function (this includes injected test cases)
if not everything:
if func_name.startswith('__'):
continue
# go ahead and add lines, later folding will merge this if
# there are other hits on this line
results.append(CoverageResult(
src_path, func_name, line['line_number'],
0, line['count'],
0,
Frac(1 if line['count'] > 0 else 0, 1),
Frac(
sum(1 if branch['count'] > 0 else 0 sum(1 if branch['count'] > 0 else 0
for branch in line['branches']), for branch in line['branches']),
len(line['branches'])))) len(line['branches']))))
@@ -278,9 +283,27 @@ def collect(paths, *,
return results return results
def fold(results, *, def fold(Result, results, *,
by=['file', 'function', 'line'], by=None,
defines=None,
**_): **_):
if by is None:
by = Result._by
for k in it.chain(by or [], (k for k, _ in defines or [])):
if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k)
sys.exit(-1)
# filter by matching defines
if defines is not None:
results_ = []
for r in results:
if all(getattr(r, k) in vs for k, vs in defines):
results_.append(r)
results = results_
# organize results into conflicts
folding = co.OrderedDict() folding = co.OrderedDict()
for r in results: for r in results:
name = tuple(getattr(r, k) for k in by) name = tuple(getattr(r, k) for k in by)
@@ -288,231 +311,224 @@ def fold(results, *,
folding[name] = [] folding[name] = []
folding[name].append(r) folding[name].append(r)
# merge conflicts
folded = [] folded = []
for rs in folding.values(): for name, rs in folding.items():
folded.append(sum(rs[1:], start=rs[0])) folded.append(sum(rs[1:], start=rs[0]))
return folded return folded
def table(Result, results, diff_results=None, *,
def table(results, diff_results=None, *, by=None,
by_file=False, fields=None,
by_line=False, sort=None,
line_sort=False,
reverse_line_sort=False,
branch_sort=False,
reverse_branch_sort=False,
summary=False, summary=False,
all=False, all=False,
percent=False, percent=False,
**_): **_):
all_, all = all, __builtins__.all all_, all = all, __builtins__.all
# fold if by is None:
results = fold(results, by = Result._by
by=['file', 'line'] if by_line if fields is None:
else ['file'] if by_file fields = Result._fields
else ['function']) types = Result._types
if diff_results is not None:
diff_results = fold(diff_results,
by=['file', 'line'] if by_line
else ['file'] if by_file
else ['function'])
# fold again
results = fold(Result, results, by=by)
if diff_results is not None:
diff_results = fold(Result, diff_results, by=by)
# organize by name
table = { table = {
'%s:%s' % (r.file, r.line) if by_line ','.join(str(getattr(r, k) or '') for k in by): r
else r.file if by_file
else r.function: r
for r in results} for r in results}
diff_table = { diff_table = {
'%s:%s' % (r.file, r.line) if by_line ','.join(str(getattr(r, k) or '') for k in by): r
else r.file if by_file
else r.function: r
for r in diff_results or []} for r in diff_results or []}
# sort, note that python's sort is stable
names = list(table.keys() | diff_table.keys()) names = list(table.keys() | diff_table.keys())
# sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: ( names.sort(key=lambda n: tuple(
-FracField.ratio( types[k].ratio(
table[n].coverage_lines if n in table else None, getattr(table.get(n), k, None),
diff_table[n].coverage_lines if n in diff_table else None), getattr(diff_table.get(n), k, None))
-FracField.ratio( for k in fields),
table[n].coverage_branches if n in table else None,
diff_table[n].coverage_branches if n in diff_table else None)))
if line_sort:
names.sort(key=lambda n: (table[n].coverage_lines,)
if n in table else (),
reverse=True) reverse=True)
elif reverse_line_sort: if sort:
names.sort(key=lambda n: (table[n].coverage_lines,) for k, reverse in reversed(sort):
if n in table else (), names.sort(key=lambda n: (getattr(table[n], k),)
reverse=False) if getattr(table.get(n), k, None) is not None else (),
elif branch_sort: reverse=reverse ^ (not k or k in Result._fields))
names.sort(key=lambda n: (table[n].coverage_branches,)
if n in table else (),
reverse=True)
elif reverse_branch_sort:
names.sort(key=lambda n: (table[n].coverage_branches,)
if n in table else (),
reverse=False)
# print header
if not summary: # build up our lines
title = '%s%s' % ( lines = []
'line' if by_line else 'file' if by_file else 'function',
# header
line = []
line.append('%s%s' % (
','.join(by),
' (%d added, %d removed)' % ( ' (%d added, %d removed)' % (
sum(1 for n in table if n not in diff_table), sum(1 for n in table if n not in diff_table),
sum(1 for n in diff_table if n not in table)) sum(1 for n in diff_table if n not in table))
if diff_results is not None and not percent else '') if diff_results is not None and not percent else '')
name_width = max(it.chain([23, len(title)], (len(n) for n in names))) if not summary else '')
else:
title = ''
name_width = 23
name_width = 4*((name_width+1+4-1)//4)-1
print('%-*s ' % (name_width, title), end='')
if diff_results is None: if diff_results is None:
print(' %s %s' % ( for k in fields:
'hits/line'.rjust(len(FracField.none)), line.append(k)
'hits/branch'.rjust(len(FracField.none))))
elif percent: elif percent:
print(' %s %s' % ( for k in fields:
'hits/line'.rjust(len(FracField.diff_none)), line.append(k)
'hits/branch'.rjust(len(FracField.diff_none))))
else: else:
print(' %s %s %s %s %s %s' % ( for k in fields:
'oh/line'.rjust(len(FracField.diff_none)), line.append('o'+k)
'oh/branch'.rjust(len(FracField.diff_none)), for k in fields:
'nh/line'.rjust(len(FracField.diff_none)), line.append('n'+k)
'nh/branch'.rjust(len(FracField.diff_none)), for k in fields:
'dh/line'.rjust(len(FracField.diff_none)), line.append('d'+k)
'dh/branch'.rjust(len(FracField.diff_none)))) line.append('')
lines.append(line)
# print entries # entries
if not summary: if not summary:
for name in names: for name in names:
r = table.get(name) r = table.get(name)
if diff_results is not None: if diff_results is not None:
diff_r = diff_table.get(name) diff_r = diff_table.get(name)
line_ratio = FracField.ratio( ratios = [
r.coverage_lines if r else None, types[k].ratio(
diff_r.coverage_lines if diff_r else None) getattr(r, k, None),
branch_ratio = FracField.ratio( getattr(diff_r, k, None))
r.coverage_branches if r else None, for k in fields]
diff_r.coverage_branches if diff_r else None) if not any(ratios) and not all_:
if not line_ratio and not branch_ratio and not all_:
continue continue
print('%-*s ' % (name_width, name), end='') line = []
line.append(name)
if diff_results is None: if diff_results is None:
print(' %s %s' % ( for k in fields:
r.coverage_lines.table() line.append(getattr(r, k).table()
if r else FracField.none, if getattr(r, k, None) is not None
r.coverage_branches.table() else types[k].none)
if r else FracField.none))
elif percent: elif percent:
print(' %s %s%s' % ( for k in fields:
r.coverage_lines.diff_table() line.append(getattr(r, k).diff_table()
if r else FracField.diff_none, if getattr(r, k, None) is not None
r.coverage_branches.diff_table() else types[k].diff_none)
if r else FracField.diff_none,
' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in [line_ratio, branch_ratio])))
else: else:
print(' %s %s %s %s %s %s%s' % ( for k in fields:
diff_r.coverage_lines.diff_table() line.append(getattr(diff_r, k).diff_table()
if diff_r else FracField.diff_none, if getattr(diff_r, k, None) is not None
diff_r.coverage_branches.diff_table() else types[k].diff_none)
if diff_r else FracField.diff_none, for k in fields:
r.coverage_lines.diff_table() line.append(getattr(r, k).diff_table()
if r else FracField.diff_none, if getattr(r, k, None) is not None
r.coverage_branches.diff_table() else types[k].diff_none)
if r else FracField.diff_none, for k in fields:
FracField.diff_diff( line.append(types[k].diff_diff(
r.coverage_lines if r else None, getattr(r, k, None),
diff_r.coverage_lines if diff_r else None) getattr(diff_r, k, None)))
if r or diff_r else FracField.diff_none, if diff_results is None:
FracField.diff_diff( line.append('')
r.coverage_branches if r else None, elif percent:
diff_r.coverage_branches if diff_r else None) line.append(' (%s)' % ', '.join(
if r or diff_r else FracField.diff_none,
' (%s)' % ', '.join(
'+∞%' if t == +m.inf '+∞%' if t == +m.inf
else '-∞%' if t == -m.inf else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t) else '%+.1f%%' % (100*t)
for t in [line_ratio, branch_ratio] for t in ratios))
else:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t) if t)
if line_ratio or branch_ratio else '')) if any(ratios) else '')
lines.append(line)
# print total # total
total = fold(results, by=[]) r = next(iter(fold(Result, results, by=[])), None)
r = total[0] if total else None
if diff_results is not None: if diff_results is not None:
diff_total = fold(diff_results, by=[]) diff_r = next(iter(fold(Result, diff_results, by=[])), None)
diff_r = diff_total[0] if diff_total else None ratios = [
line_ratio = FracField.ratio( types[k].ratio(
r.coverage_lines if r else None, getattr(r, k, None),
diff_r.coverage_lines if diff_r else None) getattr(diff_r, k, None))
branch_ratio = FracField.ratio( for k in fields]
r.coverage_branches if r else None,
diff_r.coverage_branches if diff_r else None)
print('%-*s ' % (name_width, 'TOTAL'), end='') line = []
line.append('TOTAL')
if diff_results is None: if diff_results is None:
print(' %s %s' % ( for k in fields:
r.coverage_lines.table() line.append(getattr(r, k).table()
if r else FracField.none, if getattr(r, k, None) is not None
r.coverage_branches.table() else types[k].none)
if r else FracField.none))
elif percent: elif percent:
print(' %s %s%s' % ( for k in fields:
r.coverage_lines.diff_table() line.append(getattr(r, k).diff_table()
if r else FracField.diff_none, if getattr(r, k, None) is not None
r.coverage_branches.diff_table() else types[k].diff_none)
if r else FracField.diff_none,
' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in [line_ratio, branch_ratio])))
else: else:
print(' %s %s %s %s %s %s%s' % ( for k in fields:
diff_r.coverage_lines.diff_table() line.append(getattr(diff_r, k).diff_table()
if diff_r else FracField.diff_none, if getattr(diff_r, k, None) is not None
diff_r.coverage_branches.diff_table() else types[k].diff_none)
if diff_r else FracField.diff_none, for k in fields:
r.coverage_lines.diff_table() line.append(getattr(r, k).diff_table()
if r else FracField.diff_none, if getattr(r, k, None) is not None
r.coverage_branches.diff_table() else types[k].diff_none)
if r else FracField.diff_none, for k in fields:
FracField.diff_diff( line.append(types[k].diff_diff(
r.coverage_lines if r else None, getattr(r, k, None),
diff_r.coverage_lines if diff_r else None) getattr(diff_r, k, None)))
if r or diff_r else FracField.diff_none, if diff_results is None:
FracField.diff_diff( line.append('')
r.coverage_branches if r else None, elif percent:
diff_r.coverage_branches if diff_r else None) line.append(' (%s)' % ', '.join(
if r or diff_r else FracField.diff_none,
' (%s)' % ', '.join(
'+∞%' if t == +m.inf '+∞%' if t == +m.inf
else '-∞%' if t == -m.inf else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t) else '%+.1f%%' % (100*t)
for t in [line_ratio, branch_ratio] for t in ratios))
else:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t) if t)
if line_ratio or branch_ratio else '')) if any(ratios) else '')
lines.append(line)
# find the best widths, note that column 0 contains the names and column -1
# the ratios, so those are handled a bit differently
widths = [
((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
for w, i in zip(
it.chain([23], it.repeat(7)),
range(len(lines[0])-1))]
# print our table
for line in lines:
print('%-*s %s%s' % (
widths[0], line[0],
' '.join('%*s' % (w, x)
for w, x in zip(widths[1:], line[1:-1])),
line[-1]))
def annotate(paths, results, *, def annotate(Result, results, paths, *,
annotate=False, annotate=False,
lines=False, lines=False,
branches=False, branches=False,
build_dir=None, build_dir=None,
**args): **args):
# if neither branches/lines specified, color both
if annotate and not lines and not branches:
lines, branches = True, True
for path in paths: for path in paths:
# map to source file # map to source file
src_path = re.sub('\.t\.a\.gcda$', '.c', path) src_path = re.sub('\.t\.a\.gcda$', '.c', path)
@@ -521,7 +537,7 @@ def annotate(paths, results, *,
src_path) src_path)
# flatten to line info # flatten to line info
results = fold(results, by=['file', 'line']) results = fold(Result, results, by=['file', 'line'])
table = {r.line: r for r in results if r.file == src_path} table = {r.line: r for r in results if r.file == src_path}
# calculate spans to show # calculate spans to show
@@ -529,10 +545,8 @@ def annotate(paths, results, *,
spans = [] spans = []
last = None last = None
for line, r in sorted(table.items()): for line, r in sorted(table.items()):
if ((lines and int(r.coverage_hits) == 0) if ((lines and int(r.hits) == 0)
or (branches or (branches and r.branches.a < r.branches.b)):
and r.coverage_branches.a
< r.coverage_branches.b)):
if last is not None and line - last.stop <= args['context']: if last is not None and line - last.stop <= args['context']:
last = range( last = range(
last.start, last.start,
@@ -568,24 +582,29 @@ def annotate(paths, results, *,
if i+1 in table: if i+1 in table:
r = table[i+1] r = table[i+1]
line = '%-*s // %s hits, %s branches' % ( line = '%-*s // %s hits%s' % (
args['width'], args['width'],
line, line,
r.coverage_hits, r.hits,
r.coverage_branches) ', %s branches' % (r.branches,)
if int(r.branches.b) else '')
if args['color']: if args['color']:
if lines and int(r.coverage_hits) == 0: if lines and int(r.hits) == 0:
line = '\x1b[1;31m%s\x1b[m' % line line = '\x1b[1;31m%s\x1b[m' % line
elif (branches elif branches and r.branches.a < r.branches.b:
and r.coverage_branches.a
< r.coverage_branches.b):
line = '\x1b[35m%s\x1b[m' % line line = '\x1b[35m%s\x1b[m' % line
print(line) print(line)
def main(gcda_paths, **args): def main(gcda_paths, *,
by=None,
fields=None,
defines=None,
sort=None,
hits=False,
**args):
# figure out what color should be # figure out what color should be
if args.get('color') == 'auto': if args.get('color') == 'auto':
args['color'] = sys.stdout.isatty() args['color'] = sys.stdout.isatty()
@@ -606,7 +625,7 @@ def main(gcda_paths, **args):
paths.append(path) paths.append(path)
if not paths: if not paths:
print('no .gcda files found in %r?' % gcda_paths) print("error: no .gcda files found in %r?" % gcda_paths)
sys.exit(-1) sys.exit(-1)
results = collect(paths, **args) results = collect(paths, **args)
@@ -616,25 +635,38 @@ def main(gcda_paths, **args):
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
results.append(CoverageResult(**{ results.append(CoverageResult(
k: v for k, v in r.items() **{k: r[k] for k in CoverageResult._by
if k in CoverageResult._fields})) if k in r and r[k].strip()},
**{k: r['coverage_'+k]
for k in CoverageResult._fields
if 'coverage_'+k in r
and r['coverage_'+k].strip()}))
except TypeError: except TypeError:
pass pass
# fold to remove duplicates # fold
results = fold(results) results = fold(CoverageResult, results, by=by, defines=defines)
# sort because why not # sort, note that python's sort is stable
results.sort() results.sort()
if sort:
for k, reverse in reversed(sort):
results.sort(key=lambda r: (getattr(r, k),)
if getattr(r, k) is not None else (),
reverse=reverse ^ (not k or k in CoverageResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, CoverageResult._fields) writer = csv.DictWriter(f, CoverageResult._by
+ ['coverage_'+k for k in CoverageResult._fields])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow(r._asdict()) writer.writerow(
{k: getattr(r, k) for k in CoverageResult._by}
| {'coverage_'+k: getattr(r, k)
for k in CoverageResult._fields})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -644,31 +676,39 @@ def main(gcda_paths, **args):
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
diff_results.append(CoverageResult(**{ diff_results.append(CoverageResult(
k: v for k, v in r.items() **{k: r[k] for k in CoverageResult._by
if k in CoverageResult._fields})) if k in r and r[k].strip()},
**{k: r['coverage_'+k]
for k in CoverageResult._fields
if 'coverage_'+k in r
and r['coverage_'+k].strip()}))
except TypeError: except TypeError:
pass pass
except FileNotFoundError: except FileNotFoundError:
pass pass
# fold to remove duplicates # fold
diff_results = fold(diff_results) diff_results = fold(CoverageResult, diff_results,
by=by, defines=defines)
# print table
if not args.get('quiet'): if not args.get('quiet'):
if (args.get('annotate') if (args.get('annotate')
or args.get('lines') or args.get('lines')
or args.get('branches')): or args.get('branches')):
# annotate sources # annotate sources
annotate( annotate(CoverageResult, results, paths,
paths,
results,
**args) **args)
else: else:
# print table # print table
table( table(CoverageResult, results,
results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['function'],
fields=fields if fields is not None
else ['lines', 'branches'] if not hits
else ['calls', 'hits'],
sort=sort,
**args) **args)
# catch lack of coverage # catch lack of coverage
@@ -717,33 +757,47 @@ if __name__ == "__main__":
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by-file', '-b', '--by',
action='store_true', action='append',
help="Group by file.") choices=CoverageResult._by,
help="Group by this field.")
parser.add_argument( parser.add_argument(
'--by-line', '-f', '--field',
action='store_true', dest='fields',
help="Group by line.") action='append',
choices=CoverageResult._fields,
help="Show this field.")
parser.add_argument( parser.add_argument(
'-s', '--line-sort', '-D', '--define',
action='store_true', dest='defines',
help="Sort by line coverage.") action='append',
type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
help="Only include results where this field is this value.")
class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option):
if namespace.sort is None:
namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument( parser.add_argument(
'-S', '--reverse-line-sort', '-s', '--sort',
action='store_true', action=AppendSort,
help="Sort by line coverage, but backwards.") help="Sort by this field.")
parser.add_argument( parser.add_argument(
'--branch-sort', '-S', '--reverse-sort',
action='store_true', action=AppendSort,
help="Sort by branch coverage.") help="Sort by this field, but backwards.")
parser.add_argument(
'--reverse-branch-sort',
action='store_true',
help="Sort by branch coverage, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total size.") help="Only show the total.")
parser.add_argument(
'-A', '--everything',
action='store_true',
help="Include builtin and libc specific symbols.")
parser.add_argument(
'-H', '--hits',
action='store_true',
help="Show total hits instead of coverage.")
parser.add_argument( parser.add_argument(
'-l', '--annotate', '-l', '--annotate',
action='store_true', action='store_true',
@@ -779,10 +833,6 @@ if __name__ == "__main__":
'-E', '--error-on-branches', '-E', '--error-on-branches',
action='store_true', action='store_true',
help="Error if any branches are not covered.") help="Error if any branches are not covered.")
parser.add_argument(
'-A', '--everything',
action='store_true',
help="Include builtin and libc specific symbols.")
parser.add_argument( parser.add_argument(
'--gcov-tool', '--gcov-tool',
default=GCOV_TOOL, default=GCOV_TOOL,

418
scripts/data.py
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# #
# Script to find data size at the function level. Basically just a bit wrapper # Script to find data size at the function level. Basically just a big wrapper
# around nm with some extra conveniences for comparing builds. Heavily inspired # around nm with some extra conveniences for comparing builds. Heavily inspired
# by Linux's Bloat-O-Meter. # by Linux's Bloat-O-Meter.
# #
@@ -28,12 +28,11 @@ NM_TOOL = ['nm']
TYPE = 'dDbB' TYPE = 'dDbB'
# integer fields # integer fields
class IntField(co.namedtuple('IntField', 'x')): class Int(co.namedtuple('Int', 'x')):
__slots__ = () __slots__ = ()
def __new__(cls, x=0): def __new__(cls, x=0):
if isinstance(x, IntField): if isinstance(x, Int):
return x return x
if isinstance(x, str): if isinstance(x, str):
try: try:
@@ -99,35 +98,30 @@ class IntField(co.namedtuple('IntField', 'x')):
return (new-old) / old return (new-old) / old
def __add__(self, other): def __add__(self, other):
return IntField(self.x + other.x) return self.__class__(self.x + other.x)
def __sub__(self, other): def __sub__(self, other):
return IntField(self.x - other.x) return self.__class__(self.x - other.x)
def __mul__(self, other): def __mul__(self, other):
return IntField(self.x * other.x) return self.__class__(self.x * other.x)
def __lt__(self, other):
return self.x < other.x
def __gt__(self, other):
return self.__class__.__lt__(other, self)
def __le__(self, other):
return not self.__gt__(other)
def __ge__(self, other):
return not self.__lt__(other)
# data size results # data size results
class DataResult(co.namedtuple('DataResult', 'file,function,data_size')): class DataResult(co.namedtuple('DataResult', [
'file', 'function',
'size'])):
_by = ['file', 'function']
_fields = ['size']
_types = {'size': Int}
__slots__ = () __slots__ = ()
def __new__(cls, file, function, data_size): def __new__(cls, file='', function='', size=0):
return super().__new__(cls, file, function, IntField(data_size)) return super().__new__(cls, file, function,
Int(size))
def __add__(self, other): def __add__(self, other):
return DataResult(self.file, self.function, return DataResult(self.file, self.function,
self.data_size + other.data_size) self.size + other.size)
def openio(path, mode='r'): def openio(path, mode='r'):
@@ -189,9 +183,27 @@ def collect(paths, *,
return results return results
def fold(results, *, def fold(Result, results, *,
by=['file', 'function'], by=None,
defines=None,
**_): **_):
if by is None:
by = Result._by
for k in it.chain(by or [], (k for k, _ in defines or [])):
if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k)
sys.exit(-1)
# filter by matching defines
if defines is not None:
results_ = []
for r in results:
if all(getattr(r, k) in vs for k, vs in defines):
results_.append(r)
results = results_
# organize results into conflicts
folding = co.OrderedDict() folding = co.OrderedDict()
for r in results: for r in results:
name = tuple(getattr(r, k) for k in by) name = tuple(getattr(r, k) for k in by)
@@ -199,157 +211,220 @@ def fold(results, *,
folding[name] = [] folding[name] = []
folding[name].append(r) folding[name].append(r)
# merge conflicts
folded = [] folded = []
for rs in folding.values(): for name, rs in folding.items():
folded.append(sum(rs[1:], start=rs[0])) folded.append(sum(rs[1:], start=rs[0]))
return folded return folded
def table(Result, results, diff_results=None, *,
def table(results, diff_results=None, *, by=None,
by_file=False, fields=None,
size_sort=False, sort=None,
reverse_size_sort=False,
summary=False, summary=False,
all=False, all=False,
percent=False, percent=False,
**_): **_):
all_, all = all, __builtins__.all all_, all = all, __builtins__.all
# fold if by is None:
results = fold(results, by=['file' if by_file else 'function']) by = Result._by
if diff_results is not None: if fields is None:
diff_results = fold(diff_results, fields = Result._fields
by=['file' if by_file else 'function']) types = Result._types
# fold again
results = fold(Result, results, by=by)
if diff_results is not None:
diff_results = fold(Result, diff_results, by=by)
# organize by name
table = { table = {
r.file if by_file else r.function: r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
r.file if by_file else r.function: r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
# sort, note that python's sort is stable
names = list(table.keys() | diff_table.keys()) names = list(table.keys() | diff_table.keys())
# sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: -IntField.ratio( names.sort(key=lambda n: tuple(
table[n].data_size if n in table else None, types[k].ratio(
diff_table[n].data_size if n in diff_table else None)) getattr(table.get(n), k, None),
if size_sort: getattr(diff_table.get(n), k, None))
names.sort(key=lambda n: (table[n].data_size,) if n in table else (), for k in fields),
reverse=True) reverse=True)
elif reverse_size_sort: if sort:
names.sort(key=lambda n: (table[n].data_size,) if n in table else (), for k, reverse in reversed(sort):
reverse=False) names.sort(key=lambda n: (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else (),
reverse=reverse ^ (not k or k in Result._fields))
# print header
if not summary: # build up our lines
title = '%s%s' % ( lines = []
'file' if by_file else 'function',
# header
line = []
line.append('%s%s' % (
','.join(by),
' (%d added, %d removed)' % ( ' (%d added, %d removed)' % (
sum(1 for n in table if n not in diff_table), sum(1 for n in table if n not in diff_table),
sum(1 for n in diff_table if n not in table)) sum(1 for n in diff_table if n not in table))
if diff_results is not None and not percent else '') if diff_results is not None and not percent else '')
name_width = max(it.chain([23, len(title)], (len(n) for n in names))) if not summary else '')
else:
title = ''
name_width = 23
name_width = 4*((name_width+1+4-1)//4)-1
print('%-*s ' % (name_width, title), end='')
if diff_results is None: if diff_results is None:
print(' %s' % ('size'.rjust(len(IntField.none)))) for k in fields:
line.append(k)
elif percent: elif percent:
print(' %s' % ('size'.rjust(len(IntField.diff_none)))) for k in fields:
line.append(k)
else: else:
print(' %s %s %s' % ( for k in fields:
'old'.rjust(len(IntField.diff_none)), line.append('o'+k)
'new'.rjust(len(IntField.diff_none)), for k in fields:
'diff'.rjust(len(IntField.diff_none)))) line.append('n'+k)
for k in fields:
line.append('d'+k)
line.append('')
lines.append(line)
# print entries # entries
if not summary: if not summary:
for name in names: for name in names:
r = table.get(name) r = table.get(name)
if diff_results is not None: if diff_results is not None:
diff_r = diff_table.get(name) diff_r = diff_table.get(name)
ratio = IntField.ratio( ratios = [
r.data_size if r else None, types[k].ratio(
diff_r.data_size if diff_r else None) getattr(r, k, None),
if not ratio and not all_: getattr(diff_r, k, None))
for k in fields]
if not any(ratios) and not all_:
continue continue
print('%-*s ' % (name_width, name), end='') line = []
line.append(name)
if diff_results is None: if diff_results is None:
print(' %s' % ( for k in fields:
r.data_size.table() line.append(getattr(r, k).table()
if r else IntField.none)) if getattr(r, k, None) is not None
else types[k].none)
elif percent: elif percent:
print(' %s%s' % ( for k in fields:
r.data_size.diff_table() line.append(getattr(r, k).diff_table()
if r else IntField.diff_none, if getattr(r, k, None) is not None
' (%s)' % ( else types[k].diff_none)
'+∞%' if ratio == +m.inf
else '-∞%' if ratio == -m.inf
else '%+.1f%%' % (100*ratio))))
else: else:
print(' %s %s %s%s' % ( for k in fields:
diff_r.data_size.diff_table() line.append(getattr(diff_r, k).diff_table()
if diff_r else IntField.diff_none, if getattr(diff_r, k, None) is not None
r.data_size.diff_table() else types[k].diff_none)
if r else IntField.diff_none, for k in fields:
IntField.diff_diff( line.append(getattr(r, k).diff_table()
r.data_size if r else None, if getattr(r, k, None) is not None
diff_r.data_size if diff_r else None) else types[k].diff_none)
if r or diff_r else IntField.diff_none, for k in fields:
' (%s)' % ( line.append(types[k].diff_diff(
'+∞%' if ratio == +m.inf getattr(r, k, None),
else '-∞%' if ratio == -m.inf getattr(diff_r, k, None)))
else '%+.1f%%' % (100*ratio)) if diff_results is None:
if ratio else '')) line.append('')
elif percent:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios))
else:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t)
if any(ratios) else '')
lines.append(line)
# print total # total
total = fold(results, by=[]) r = next(iter(fold(Result, results, by=[])), None)
r = total[0] if total else None
if diff_results is not None: if diff_results is not None:
diff_total = fold(diff_results, by=[]) diff_r = next(iter(fold(Result, diff_results, by=[])), None)
diff_r = diff_total[0] if diff_total else None ratios = [
ratio = IntField.ratio( types[k].ratio(
r.data_size if r else None, getattr(r, k, None),
diff_r.data_size if diff_r else None) getattr(diff_r, k, None))
for k in fields]
print('%-*s ' % (name_width, 'TOTAL'), end='') line = []
line.append('TOTAL')
if diff_results is None: if diff_results is None:
print(' %s' % ( for k in fields:
r.data_size.table() line.append(getattr(r, k).table()
if r else IntField.none)) if getattr(r, k, None) is not None
else types[k].none)
elif percent: elif percent:
print(' %s%s' % ( for k in fields:
r.data_size.diff_table() line.append(getattr(r, k).diff_table()
if r else IntField.diff_none, if getattr(r, k, None) is not None
' (%s)' % ( else types[k].diff_none)
'+∞%' if ratio == +m.inf
else '-∞%' if ratio == -m.inf
else '%+.1f%%' % (100*ratio))))
else: else:
print(' %s %s %s%s' % ( for k in fields:
diff_r.data_size.diff_table() line.append(getattr(diff_r, k).diff_table()
if diff_r else IntField.diff_none, if getattr(diff_r, k, None) is not None
r.data_size.diff_table() else types[k].diff_none)
if r else IntField.diff_none, for k in fields:
IntField.diff_diff( line.append(getattr(r, k).diff_table()
r.data_size if r else None, if getattr(r, k, None) is not None
diff_r.data_size if diff_r else None) else types[k].diff_none)
if r or diff_r else IntField.diff_none, for k in fields:
' (%s)' % ( line.append(types[k].diff_diff(
'+∞%' if ratio == +m.inf getattr(r, k, None),
else '-∞%' if ratio == -m.inf getattr(diff_r, k, None)))
else '%+.1f%%' % (100*ratio)) if diff_results is None:
if ratio else '')) line.append('')
elif percent:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios))
else:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t)
if any(ratios) else '')
lines.append(line)
# find the best widths, note that column 0 contains the names and column -1
# the ratios, so those are handled a bit differently
widths = [
((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
for w, i in zip(
it.chain([23], it.repeat(7)),
range(len(lines[0])-1))]
# print our table
for line in lines:
print('%-*s %s%s' % (
widths[0], line[0],
' '.join('%*s' % (w, x)
for w, x in zip(widths[1:], line[1:-1])),
line[-1]))
def main(obj_paths, **args): def main(obj_paths, *,
by=None,
fields=None,
defines=None,
sort=None,
**args):
# find sizes # find sizes
if not args.get('use', None): if not args.get('use', None):
# find .o files # find .o files
@@ -362,7 +437,7 @@ def main(obj_paths, **args):
paths.append(path) paths.append(path)
if not paths: if not paths:
print('no .obj files found in %r?' % obj_paths) print("error: no .obj files found in %r?" % obj_paths)
sys.exit(-1) sys.exit(-1)
results = collect(paths, **args) results = collect(paths, **args)
@@ -372,25 +447,35 @@ def main(obj_paths, **args):
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
results.append(DataResult(**{ results.append(DataResult(
k: v for k, v in r.items() **{k: r[k] for k in DataResult._by
if k in DataResult._fields})) if k in r and r[k].strip()},
**{k: r['data_'+k] for k in DataResult._fields
if 'data_'+k in r and r['data_'+k].strip()}))
except TypeError: except TypeError:
pass pass
# fold to remove duplicates # fold
results = fold(results) results = fold(DataResult, results, by=by, defines=defines)
# sort because why not # sort, note that python's sort is stable
results.sort() results.sort()
if sort:
for k, reverse in reversed(sort):
results.sort(key=lambda r: (getattr(r, k),)
if getattr(r, k) is not None else (),
reverse=reverse ^ (not k or k in DataResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, DataResult._fields) writer = csv.DictWriter(f, DataResult._by
+ ['data_'+k for k in DataResult._fields])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow(r._asdict()) writer.writerow(
{k: getattr(r, k) for k in DataResult._by}
| {'data_'+k: getattr(r, k) for k in DataResult._fields})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -400,22 +485,26 @@ def main(obj_paths, **args):
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
diff_results.append(DataResult(**{ diff_results.append(DataResult(
k: v for k, v in r.items() **{k: r[k] for k in DataResult._by
if k in DataResult._fields})) if k in r and r[k].strip()},
**{k: r['data_'+k] for k in DataResult._fields
if 'data_'+k in r and r['data_'+k].strip()}))
except TypeError: except TypeError:
pass pass
except FileNotFoundError: except FileNotFoundError:
pass pass
# fold to remove duplicates # fold
diff_results = fold(diff_results) diff_results = fold(DataResult, diff_results, by=by, defines=defines)
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
table( table(DataResult, results,
results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['function'],
fields=fields,
sort=sort,
**args) **args)
@@ -456,22 +545,39 @@ if __name__ == "__main__":
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by-file', '-b', '--by',
action='store_true', action='append',
help="Group by file. Note this does not include padding " choices=DataResult._by,
"so sizes may differ from other tools.") help="Group by this field.")
parser.add_argument( parser.add_argument(
'-s', '--size-sort', '-f', '--field',
action='store_true', dest='fields',
help="Sort by size.") action='append',
choices=DataResult._fields,
help="Show this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-size-sort', '-D', '--define',
action='store_true', dest='defines',
help="Sort by size, but backwards.") action='append',
type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
help="Only include results where this field is this value.")
class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option):
if namespace.sort is None:
namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument(
'-s', '--sort',
action=AppendSort,
help="Sort by this fields.")
parser.add_argument(
'-S', '--reverse-sort',
action=AppendSort,
help="Sort by this fields, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total size.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'-A', '--everything', '-A', '--everything',
action='store_true', action='store_true',

36
scripts/plot.py
View File

@@ -438,9 +438,7 @@ def datasets(results, by=None, x=None, y=None, define=[]):
y = co.OrderedDict() y = co.OrderedDict()
for r in results: for r in results:
for k, v in r.items(): for k, v in r.items():
if by is not None and k in by: if (by is None or k not in by) and v.strip():
continue
if y.get(k, True):
try: try:
dat(v) dat(v)
y[k] = True y[k] = True
@@ -462,7 +460,7 @@ def datasets(results, by=None, x=None, y=None, define=[]):
for y_ in y: for y_ in y:
# hide x/y if there is only one field # hide x/y if there is only one field
k_x = x_ if len(x or []) > 1 else '' k_x = x_ if len(x or []) > 1 else ''
k_y = y_ if len(y or []) > 1 else '' k_y = y_ if len(y or []) > 1 or (not ks_ and not k_x) else ''
datasets[ks_ + (k_x, k_y)] = dataset( datasets[ks_ + (k_x, k_y)] = dataset(
results, results,
@@ -509,15 +507,15 @@ def main(csv_paths, *,
ylim = (0, ylim[0]) ylim = (0, ylim[0])
# separate out renames # separate out renames
renames = [k.split('=', 1) renames = list(it.chain.from_iterable(
for k in it.chain(by or [], x or [], y or []) ((k, v) for v in vs)
if '=' in k] for k, vs in it.chain(by or [], x or [], y or [])))
if by is not None: if by is not None:
by = [k.split('=', 1)[0] for k in by] by = [k for k, _ in by]
if x is not None: if x is not None:
x = [k.split('=', 1)[0] for k in x] x = [k for k, _ in x]
if y is not None: if y is not None:
y = [k.split('=', 1)[0] for k in y] y = [k for k, _ in y]
def draw(f): def draw(f):
def writeln(s=''): def writeln(s=''):
@@ -739,23 +737,31 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
help="Fields to render as separate plots. All other fields will be " type=lambda x: (
"summed as needed. Can rename fields with new_name=old_name.") lambda k,v=None: (k, v.split(',') if v is not None else ())
)(*x.split('=', 1)),
help="Group by this field. Can rename fields with new_name=old_name.")
parser.add_argument( parser.add_argument(
'-x', '-x',
action='append', action='append',
help="Fields to use for the x-axis. Can rename fields with " type=lambda x: (
lambda k,v=None: (k, v.split(',') if v is not None else ())
)(*x.split('=', 1)),
help="Field to use for the x-axis. Can rename fields with "
"new_name=old_name.") "new_name=old_name.")
parser.add_argument( parser.add_argument(
'-y', '-y',
action='append', action='append',
help="Fields to use for the y-axis. Can rename fields with " type=lambda x: (
lambda k,v=None: (k, v.split(',') if v is not None else ())
)(*x.split('=', 1)),
help="Field to use for the y-axis. Can rename fields with "
"new_name=old_name.") "new_name=old_name.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
action='append', action='append',
help="Only include rows where this field is this value. May include " help="Only include results where this field is this value. May include "
"comma-separated options.") "comma-separated options.")
parser.add_argument( parser.add_argument(
'--color', '--color',

642
scripts/stack.py
View File

@@ -23,10 +23,10 @@ CI_PATHS = ['*.ci']
# integer fields # integer fields
class IntField(co.namedtuple('IntField', 'x')): class Int(co.namedtuple('Int', 'x')):
__slots__ = () __slots__ = ()
def __new__(cls, x=0): def __new__(cls, x=0):
if isinstance(x, IntField): if isinstance(x, Int):
return x return x
if isinstance(x, str): if isinstance(x, str):
try: try:
@@ -92,38 +92,33 @@ class IntField(co.namedtuple('IntField', 'x')):
return (new-old) / old return (new-old) / old
def __add__(self, other): def __add__(self, other):
return IntField(self.x + other.x) return self.__class__(self.x + other.x)
def __sub__(self, other): def __sub__(self, other):
return IntField(self.x - other.x) return self.__class__(self.x - other.x)
def __mul__(self, other): def __mul__(self, other):
return IntField(self.x * other.x) return self.__class__(self.x * other.x)
def __lt__(self, other):
return self.x < other.x
def __gt__(self, other):
return self.__class__.__lt__(other, self)
def __le__(self, other):
return not self.__gt__(other)
def __ge__(self, other):
return not self.__lt__(other)
# size results # size results
class StackResult(co.namedtuple('StackResult', class StackResult(co.namedtuple('StackResult', [
'file,function,stack_frame,stack_limit')): 'file', 'function', 'frame', 'limit', 'calls'])):
_by = ['file', 'function']
_fields = ['frame', 'limit']
_types = {'frame': Int, 'limit': Int}
__slots__ = () __slots__ = ()
def __new__(cls, file, function, stack_frame, stack_limit): def __new__(cls, file='', function='',
frame=0, limit=0, calls=set()):
return super().__new__(cls, file, function, return super().__new__(cls, file, function,
IntField(stack_frame), IntField(stack_limit)) Int(frame), Int(limit),
calls)
def __add__(self, other): def __add__(self, other):
return StackResult(self.file, self.function, return StackResult(self.file, self.function,
self.stack_frame + other.stack_frame, self.frame + other.frame,
max(self.stack_limit, other.stack_limit)) max(self.limit, other.limit),
self.calls | other.calls)
def openio(path, mode='r'): def openio(path, mode='r'):
@@ -135,7 +130,6 @@ def openio(path, mode='r'):
else: else:
return open(path, mode) return open(path, mode)
def collect(paths, *, def collect(paths, *,
everything=False, everything=False,
**args): **args):
@@ -147,10 +141,10 @@ def collect(paths, *,
node = [] node = []
while True: while True:
rest = rest.lstrip() rest = rest.lstrip()
m = k_pattern.match(rest) m_ = k_pattern.match(rest)
if not m: if not m_:
return (node, rest) return (node, rest)
k, rest = m.group(1), rest[m.end(0):] k, rest = m_.group(1), rest[m_.end(0):]
rest = rest.lstrip() rest = rest.lstrip()
if rest.startswith('{'): if rest.startswith('{'):
@@ -159,9 +153,9 @@ def collect(paths, *,
rest = rest[1:] rest = rest[1:]
node.append((k, v)) node.append((k, v))
else: else:
m = v_pattern.match(rest) m_ = v_pattern.match(rest)
assert m, "unexpected %r" % rest[0:1] assert m_, "unexpected %r" % rest[0:1]
v, rest = m.group(1) or m.group(2), rest[m.end(0):] v, rest = m_.group(1) or m_.group(2), rest[m_.end(0):]
node.append((k, v)) node.append((k, v))
node, rest = parse_vcg(rest) node, rest = parse_vcg(rest)
@@ -181,13 +175,13 @@ def collect(paths, *,
for k, info in graph: for k, info in graph:
if k == 'node': if k == 'node':
info = dict(info) info = dict(info)
m = f_pattern.match(info['label']) m_ = f_pattern.match(info['label'])
if m: if m_:
function, file, size, type = m.groups() function, file, size, type = m_.groups()
if (not args.get('quiet') if (not args.get('quiet')
and 'static' not in type and 'static' not in type
and 'bounded' not in type): and 'bounded' not in type):
print('warning: found non-static stack for %s (%s)' print("warning: found non-static stack for %s (%s)"
% (function, type, size)) % (function, type, size))
_, _, _, targets = callgraph[info['title']] _, _, _, targets = callgraph[info['title']]
callgraph[info['title']] = ( callgraph[info['title']] = (
@@ -217,7 +211,7 @@ def collect(paths, *,
for target in targets: for target in targets:
if target in seen: if target in seen:
# found a cycle # found a cycle
return float('inf') return m.inf
limit_ = find_limit(target, seen | {target}) limit_ = find_limit(target, seen | {target})
limit = max(limit, limit_) limit = max(limit, limit_)
@@ -233,19 +227,35 @@ def collect(paths, *,
# build results # build results
results = [] results = []
calls = {}
for source, (s_file, s_function, frame, targets) in callgraph.items(): for source, (s_file, s_function, frame, targets) in callgraph.items():
limit = find_limit(source) limit = find_limit(source)
cs = find_calls(targets) calls = find_calls(targets)
results.append(StackResult(s_file, s_function, frame, limit)) results.append(StackResult(s_file, s_function, frame, limit, calls))
calls[(s_file, s_function)] = cs
return results, calls return results
def fold(results, *, def fold(Result, results, *,
by=['file', 'function'], by=None,
defines=None,
**_): **_):
if by is None:
by = Result._by
for k in it.chain(by or [], (k for k, _ in defines or [])):
if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k)
sys.exit(-1)
# filter by matching defines
if defines is not None:
results_ = []
for r in results:
if all(getattr(r, k) in vs for k, vs in defines):
results_.append(r)
results = results_
# organize results into conflicts
folding = co.OrderedDict() folding = co.OrderedDict()
for r in results: for r in results:
name = tuple(getattr(r, k) for k in by) name = tuple(getattr(r, k) for k in by)
@@ -253,36 +263,17 @@ def fold(results, *,
folding[name] = [] folding[name] = []
folding[name].append(r) folding[name].append(r)
# merge conflicts
folded = [] folded = []
for rs in folding.values(): for name, rs in folding.items():
folded.append(sum(rs[1:], start=rs[0])) folded.append(sum(rs[1:], start=rs[0]))
return folded return folded
def fold_calls(calls, *, def table(Result, results, diff_results=None, *,
by=['file', 'function'], by=None,
**_): fields=None,
def by_(name): sort=None,
file, function = name
return (((file,) if 'file' in by else ())
+ ((function,) if 'function' in by else ()))
folded = {}
for name, cs in calls.items():
name = by_(name)
if name not in folded:
folded[name] = set()
folded[name] |= {by_(c) for c in cs}
return folded
def table(results, calls, diff_results=None, *,
by_file=False,
limit_sort=False,
reverse_limit_sort=False,
frame_sort=False,
reverse_frame_sort=False,
summary=False, summary=False,
all=False, all=False,
percent=False, percent=False,
@@ -291,209 +282,268 @@ def table(results, calls, diff_results=None, *,
**_): **_):
all_, all = all, __builtins__.all all_, all = all, __builtins__.all
# tree doesn't really make sense with depth=0, assume depth=inf if by is None:
if depth is None: by = Result._by
depth = float('inf') if tree else 0 if fields is None:
fields = Result._fields
types = Result._types
# fold # fold again
results = fold(results, by=['file' if by_file else 'function']) results = fold(Result, results, by=by)
calls = fold_calls(calls, by=['file' if by_file else 'function'])
if diff_results is not None: if diff_results is not None:
diff_results = fold(diff_results, diff_results = fold(Result, diff_results, by=by)
by=['file' if by_file else 'function'])
# organize by name
table = { table = {
r.file if by_file else r.function: r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
r.file if by_file else r.function: r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
# sort, note that python's sort is stable
names = list(table.keys() | diff_table.keys()) names = list(table.keys() | diff_table.keys())
# sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: -IntField.ratio( names.sort(key=lambda n: tuple(
table[n].stack_frame if n in table else None, types[k].ratio(
diff_table[n].stack_frame if n in diff_table else None)) getattr(table.get(n), k, None),
if limit_sort: getattr(diff_table.get(n), k, None))
names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (), for k in fields),
reverse=True) reverse=True)
elif reverse_limit_sort: if sort:
names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (), for k, reverse in reversed(sort):
reverse=False) names.sort(key=lambda n: (getattr(table[n], k),)
elif frame_sort: if getattr(table.get(n), k, None) is not None else (),
names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (), reverse=reverse ^ (not k or k in Result._fields))
reverse=True)
elif reverse_frame_sort:
names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (),
reverse=False)
# print header
if not summary: # build up our lines
title = '%s%s' % ( lines = []
'file' if by_file else 'function',
# header
line = []
line.append('%s%s' % (
','.join(by),
' (%d added, %d removed)' % ( ' (%d added, %d removed)' % (
sum(1 for n in table if n not in diff_table), sum(1 for n in table if n not in diff_table),
sum(1 for n in diff_table if n not in table)) sum(1 for n in diff_table if n not in table))
if diff_results is not None and not percent else '') if diff_results is not None and not percent else '')
name_width = max(it.chain([23, len(title)], (len(n) for n in names))) if not summary else '')
else:
title = ''
name_width = 23
name_width = 4*((name_width+1+4-1)//4)-1
# adjust the name width based on the expected call depth, note that we
# can't always find the depth due to recursion
if not m.isinf(depth):
name_width += 4*depth
if not tree:
print('%-*s ' % (name_width, title), end='')
if diff_results is None: if diff_results is None:
print(' %s %s' % ( for k in fields:
'frame'.rjust(len(IntField.none)), line.append(k)
'limit'.rjust(len(IntField.none))))
elif percent: elif percent:
print(' %s %s' % ( for k in fields:
'frame'.rjust(len(IntField.diff_none)), line.append(k)
'limit'.rjust(len(IntField.diff_none))))
else: else:
print(' %s %s %s %s %s %s' % ( for k in fields:
'oframe'.rjust(len(IntField.diff_none)), line.append('o'+k)
'olimit'.rjust(len(IntField.diff_none)), for k in fields:
'nframe'.rjust(len(IntField.diff_none)), line.append('n'+k)
'nlimit'.rjust(len(IntField.diff_none)), for k in fields:
'dframe'.rjust(len(IntField.diff_none)), line.append('d'+k)
'dlimit'.rjust(len(IntField.diff_none)))) line.append('')
lines.append(line)
# print entries # entries
if not summary: if not summary:
# print the tree recursively for name in names:
def table_calls(names_, depth,
prefixes=('', '', '', '')):
for i, name in enumerate(names_):
r = table.get(name) r = table.get(name)
if diff_results is not None: if diff_results is not None:
diff_r = diff_table.get(name) diff_r = diff_table.get(name)
ratio = IntField.ratio( ratios = [
r.stack_limit if r else None, types[k].ratio(
diff_r.stack_limit if diff_r else None) getattr(r, k, None),
if not ratio and not all_: getattr(diff_r, k, None))
for k in fields]
if not any(ratios) and not all_:
continue continue
is_last = (i == len(names_)-1) line = []
print('%-*s ' % (name_width, prefixes[0+is_last]+name), end='') line.append(name)
if tree: if diff_results is None:
print() for k in fields:
elif diff_results is None: line.append(getattr(r, k).table()
print(' %s %s' % ( if getattr(r, k, None) is not None
r.stack_frame.table() else types[k].none)
if r else IntField.none,
r.stack_limit.table()
if r else IntField.none))
elif percent: elif percent:
print(' %s %s%s' % ( for k in fields:
r.stack_frame.diff_table() line.append(getattr(r, k).diff_table()
if r else IntField.diff_none, if getattr(r, k, None) is not None
r.stack_limit.diff_table() else types[k].diff_none)
if r else IntField.diff_none,
' (%s)' % (
'+∞%' if ratio == +m.inf
else '-∞%' if ratio == -m.inf
else '%+.1f%%' % (100*ratio))))
else: else:
print(' %s %s %s %s %s %s%s' % ( for k in fields:
diff_r.stack_frame.diff_table() line.append(getattr(diff_r, k).diff_table()
if diff_r else IntField.diff_none, if getattr(diff_r, k, None) is not None
diff_r.stack_limit.diff_table() else types[k].diff_none)
if diff_r else IntField.diff_none, for k in fields:
r.stack_frame.diff_table() line.append(getattr(r, k).diff_table()
if r else IntField.diff_none, if getattr(r, k, None) is not None
r.stack_limit.diff_table() else types[k].diff_none)
if r else IntField.diff_none, for k in fields:
IntField.diff_diff( line.append(types[k].diff_diff(
r.stack_frame if r else None, getattr(r, k, None),
diff_r.stack_frame if diff_r else None) getattr(diff_r, k, None)))
if r or diff_r else IntField.diff_none, if diff_results is None:
IntField.diff_diff( line.append('')
r.stack_limit if r else None, elif percent:
diff_r.stack_limit if diff_r else None) line.append(' (%s)' % ', '.join(
if r or diff_r else IntField.diff_none, '+∞%' if t == +m.inf
' (%s)' % ( else '-∞%' if t == -m.inf
'+∞%' if ratio == +m.inf else '%+.1f%%' % (100*t)
else '-∞%' if ratio == -m.inf for t in ratios))
else '%+.1f%%' % (100*ratio)) else:
if ratio else '')) line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t)
if any(ratios) else '')
lines.append(line)
# total
r = next(iter(fold(Result, results, by=[])), None)
if diff_results is not None:
diff_r = next(iter(fold(Result, diff_results, by=[])), None)
ratios = [
types[k].ratio(
getattr(r, k, None),
getattr(diff_r, k, None))
for k in fields]
line = []
line.append('TOTAL')
if diff_results is None:
for k in fields:
line.append(getattr(r, k).table()
if getattr(r, k, None) is not None
else types[k].none)
elif percent:
for k in fields:
line.append(getattr(r, k).diff_table()
if getattr(r, k, None) is not None
else types[k].diff_none)
else:
for k in fields:
line.append(getattr(diff_r, k).diff_table()
if getattr(diff_r, k, None) is not None
else types[k].diff_none)
for k in fields:
line.append(getattr(r, k).diff_table()
if getattr(r, k, None) is not None
else types[k].diff_none)
for k in fields:
line.append(types[k].diff_diff(
getattr(r, k, None),
getattr(diff_r, k, None)))
if diff_results is None:
line.append('')
elif percent:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios))
else:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t)
if any(ratios) else '')
lines.append(line)
# find the best widths, note that column 0 contains the names and column -1
# the ratios, so those are handled a bit differently
widths = [
((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
for w, i in zip(
it.chain([23], it.repeat(7)),
range(len(lines[0])-1))]
# adjust the name width based on the expected call depth, though
# note this doesn't really work with unbounded recursion
if not summary:
# it doesn't really make sense to not have a depth with tree,
# so assume depth=inf if tree by default
if depth is None:
depth = m.inf if tree else 0
elif depth == 0:
depth = m.inf
if not m.isinf(depth):
widths[0] += 4*depth
# print our table with optional call info
#
# note we try to adjust our name width based on expected call depth, but
# this doesn't work if there's unbounded recursion
if not tree:
print('%-*s %s%s' % (
widths[0], lines[0][0],
' '.join('%*s' % (w, x)
for w, x, in zip(widths[1:], lines[0][1:-1])),
lines[0][-1]))
# print the tree recursively
if not summary:
line_table = {n: l for n, l in zip(names, lines[1:-1])}
def recurse(names_, depth_, prefixes=('', '', '', '')):
for i, name in enumerate(names_):
if name not in line_table:
continue
line = line_table[name]
is_last = (i == len(names_)-1)
print('%s%-*s ' % (
prefixes[0+is_last],
widths[0] - (
len(prefixes[0+is_last])
if not m.isinf(depth) else 0),
line[0]),
end='')
if not tree:
print(' %s%s' % (
' '.join('%*s' % (w, x)
for w, x, in zip(widths[1:], line[1:-1])),
line[-1]),
end='')
print()
# recurse? # recurse?
if depth > 0: if name in table and depth_ > 0:
cs = calls.get((name,), set()) calls = {
table_calls( ','.join(str(getattr(Result(*c), k) or '') for k in by)
[n for n in names if (n,) in cs], for c in table[name].calls}
depth-1,
( prefixes[2+is_last] + "|-> ", recurse(
# note we're maintaining sort order
[n for n in names if n in calls],
depth_-1,
(prefixes[2+is_last] + "|-> ",
prefixes[2+is_last] + "'-> ", prefixes[2+is_last] + "'-> ",
prefixes[2+is_last] + "| ", prefixes[2+is_last] + "| ",
prefixes[2+is_last] + " ")) prefixes[2+is_last] + " "))
recurse(names, depth)
table_calls(names, depth)
# print total
if not tree: if not tree:
total = fold(results, by=[]) print('%-*s %s%s' % (
r = total[0] if total else None widths[0], lines[-1][0],
if diff_results is not None: ' '.join('%*s' % (w, x)
diff_total = fold(diff_results, by=[]) for w, x, in zip(widths[1:], lines[-1][1:-1])),
diff_r = diff_total[0] if diff_total else None lines[-1][-1]))
ratio = IntField.ratio(
r.stack_limit if r else None,
diff_r.stack_limit if diff_r else None)
print('%-*s ' % (name_width, 'TOTAL'), end='')
if diff_results is None:
print(' %s %s' % (
r.stack_frame.table()
if r else IntField.none,
r.stack_limit.table()
if r else IntField.none))
elif percent:
print(' %s %s%s' % (
r.stack_frame.diff_table()
if r else IntField.diff_none,
r.stack_limit.diff_table()
if r else IntField.diff_none,
' (%s)' % (
'+∞%' if ratio == +m.inf
else '-∞%' if ratio == -m.inf
else '%+.1f%%' % (100*ratio))))
else:
print(' %s %s %s %s %s %s%s' % (
diff_r.stack_frame.diff_table()
if diff_r else IntField.diff_none,
diff_r.stack_limit.diff_table()
if diff_r else IntField.diff_none,
r.stack_frame.diff_table()
if r else IntField.diff_none,
r.stack_limit.diff_table()
if r else IntField.diff_none,
IntField.diff_diff(
r.stack_frame if r else None,
diff_r.stack_frame if diff_r else None)
if r or diff_r else IntField.diff_none,
IntField.diff_diff(
r.stack_limit if r else None,
diff_r.stack_limit if diff_r else None)
if r or diff_r else IntField.diff_none,
' (%s)' % (
'+∞%' if ratio == +m.inf
else '-∞%' if ratio == -m.inf
else '%+.1f%%' % (100*ratio))
if ratio else ''))
def main(ci_paths, **args): def main(ci_paths,
by=None,
fields=None,
defines=None,
sort=None,
**args):
# find sizes # find sizes
if not args.get('use', None): if not args.get('use', None):
# find .ci files # find .ci files
@@ -506,37 +556,45 @@ def main(ci_paths, **args):
paths.append(path) paths.append(path)
if not paths: if not paths:
print('no .ci files found in %r?' % ci_paths) print("error: no .ci files found in %r?" % ci_paths)
sys.exit(-1) sys.exit(-1)
results, calls = collect(paths, **args) results = collect(paths, **args)
else: else:
results = [] results = []
with openio(args['use']) as f: with openio(args['use']) as f:
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
results.append(StackResult(**{ results.append(StackResult(
k: v for k, v in r.items() **{k: r[k] for k in StackResult._by
if k in StackResult._fields})) if k in r and r[k].strip()},
**{k: r['stack_'+k] for k in StackResult._fields
if 'stack_'+k in r and r['stack_'+k].strip()}))
except TypeError: except TypeError:
pass pass
calls = {} # fold
results = fold(StackResult, results, by=by, defines=defines)
# fold to remove duplicates # sort, note that python's sort is stable
results = fold(results)
# sort because why not
results.sort() results.sort()
if sort:
for k, reverse in reversed(sort):
results.sort(key=lambda r: (getattr(r, k),)
if getattr(r, k) is not None else (),
reverse=reverse ^ (not k or k in StackResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, StackResult._fields) writer = csv.DictWriter(f, StackResult._by
+ ['stack_'+k for k in StackResult._fields])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow(r._asdict()) writer.writerow(
{k: getattr(r, k) for k in StackResult._by}
| {'stack_'+k: getattr(r, k) for k in StackResult._fields})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -546,28 +604,31 @@ def main(ci_paths, **args):
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
diff_results.append(StackResult(**{ diff_results.append(StackResult(
k: v for k, v in r.items() **{k: r[k] for k in StackResult._by
if k in StackResult._fields})) if k in r and r[k].strip()},
**{k: r['stack_'+k] for k in StackResult._fields
if 'stack_'+k in r and r['stack_'+k].strip()}))
except TypeError: except TypeError:
pass raise
except FileNotFoundError: except FileNotFoundError:
pass pass
# fold to remove duplicates # fold
diff_results = fold(diff_results) diff_results = fold(StackResult, diff_results, by=by, defines=defines)
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
table( table(StackResult, results,
results,
calls,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['function'],
fields=fields,
sort=sort,
**args) **args)
# error on recursion # error on recursion
if args.get('error_on_recursion') and any( if args.get('error_on_recursion') and any(
m.isinf(float(r.stack_limit)) for r in results): m.isinf(float(r.limit)) for r in results):
sys.exit(2) sys.exit(2)
@@ -608,47 +669,58 @@ if __name__ == "__main__":
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-t', '--tree', '-b', '--by',
action='store_true', action='append',
help="Only show the function call tree.") choices=StackResult._by,
help="Group by this field.")
parser.add_argument( parser.add_argument(
'-b', '--by-file', '-f', '--field',
action='store_true', dest='fields',
help="Group by file.") action='append',
choices=StackResult._fields,
help="Show this field.")
parser.add_argument( parser.add_argument(
'-s', '--limit-sort', '-D', '--define',
action='store_true', dest='defines',
help="Sort by stack limit.") action='append',
type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
help="Only include results where this field is this value.")
class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option):
if namespace.sort is None:
namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument( parser.add_argument(
'-S', '--reverse-limit-sort', '-s', '--sort',
action='store_true', action=AppendSort,
help="Sort by stack limit, but backwards.") help="Sort by this fields.")
parser.add_argument( parser.add_argument(
'--frame-sort', '-S', '--reverse-sort',
action='store_true', action=AppendSort,
help="Sort by stack frame.") help="Sort by this fields, but backwards.")
parser.add_argument(
'--reverse-frame-sort',
action='store_true',
help="Sort by stack frame, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total size.") help="Only show the total.")
parser.add_argument(
'-L', '--depth',
nargs='?',
type=lambda x: int(x, 0),
const=float('inf'),
help="Depth of function calls to show.")
parser.add_argument(
'-e', '--error-on-recursion',
action='store_true',
help="Error if any functions are recursive.")
parser.add_argument( parser.add_argument(
'-A', '--everything', '-A', '--everything',
action='store_true', action='store_true',
help="Include builtin and libc specific symbols.") help="Include builtin and libc specific symbols.")
parser.add_argument(
'--tree',
action='store_true',
help="Only show the function call tree.")
parser.add_argument(
'-L', '--depth',
nargs='?',
type=lambda x: int(x, 0),
const=0,
help="Depth of function calls to show. 0 show all calls but may not "
"terminate!")
parser.add_argument(
'-e', '--error-on-recursion',
action='store_true',
help="Error if any functions are recursive.")
parser.add_argument( parser.add_argument(
'--build-dir', '--build-dir',
help="Specify the relative build directory. Used to map object files " help="Specify the relative build directory. Used to map object files "

419
scripts/struct_.py
View File

@@ -24,11 +24,12 @@ OBJ_PATHS = ['*.o']
OBJDUMP_TOOL = ['objdump'] OBJDUMP_TOOL = ['objdump']
# integer fields # integer fields
class IntField(co.namedtuple('IntField', 'x')): class Int(co.namedtuple('Int', 'x')):
__slots__ = () __slots__ = ()
def __new__(cls, x=0): def __new__(cls, x=0):
if isinstance(x, IntField): if isinstance(x, Int):
return x return x
if isinstance(x, str): if isinstance(x, str):
try: try:
@@ -94,35 +95,28 @@ class IntField(co.namedtuple('IntField', 'x')):
return (new-old) / old return (new-old) / old
def __add__(self, other): def __add__(self, other):
return IntField(self.x + other.x) return self.__class__(self.x + other.x)
def __sub__(self, other): def __sub__(self, other):
return IntField(self.x - other.x) return self.__class__(self.x - other.x)
def __mul__(self, other): def __mul__(self, other):
return IntField(self.x * other.x) return self.__class__(self.x * other.x)
def __lt__(self, other):
return self.x < other.x
def __gt__(self, other):
return self.__class__.__lt__(other, self)
def __le__(self, other):
return not self.__gt__(other)
def __ge__(self, other):
return not self.__lt__(other)
# struct size results # struct size results
class StructResult(co.namedtuple('StructResult', 'file,struct,struct_size')): class StructResult(co.namedtuple('StructResult', ['file', 'struct', 'size'])):
_by = ['file', 'struct']
_fields = ['size']
_types = {'size': Int}
__slots__ = () __slots__ = ()
def __new__(cls, file, struct, struct_size): def __new__(cls, file='', struct='', size=0):
return super().__new__(cls, file, struct, IntField(struct_size)) return super().__new__(cls, file, struct,
Int(size))
def __add__(self, other): def __add__(self, other):
return StructResult(self.file, self.struct, return StructResult(self.file, self.struct,
self.struct_size + other.struct_size) self.size + other.size)
def openio(path, mode='r'): def openio(path, mode='r'):
@@ -231,9 +225,27 @@ def collect(paths, *,
return results return results
def fold(results, *, def fold(Result, results, *,
by=['file', 'struct'], by=None,
defines=None,
**_): **_):
if by is None:
by = Result._by
for k in it.chain(by or [], (k for k, _ in defines or [])):
if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k)
sys.exit(-1)
# filter by matching defines
if defines is not None:
results_ = []
for r in results:
if all(getattr(r, k) in vs for k, vs in defines):
results_.append(r)
results = results_
# organize results into conflicts
folding = co.OrderedDict() folding = co.OrderedDict()
for r in results: for r in results:
name = tuple(getattr(r, k) for k in by) name = tuple(getattr(r, k) for k in by)
@@ -241,157 +253,220 @@ def fold(results, *,
folding[name] = [] folding[name] = []
folding[name].append(r) folding[name].append(r)
# merge conflicts
folded = [] folded = []
for rs in folding.values(): for name, rs in folding.items():
folded.append(sum(rs[1:], start=rs[0])) folded.append(sum(rs[1:], start=rs[0]))
return folded return folded
def table(Result, results, diff_results=None, *,
def table(results, diff_results=None, *, by=None,
by_file=False, fields=None,
size_sort=False, sort=None,
reverse_size_sort=False,
summary=False, summary=False,
all=False, all=False,
percent=False, percent=False,
**_): **_):
all_, all = all, __builtins__.all all_, all = all, __builtins__.all
# fold if by is None:
results = fold(results, by=['file' if by_file else 'struct']) by = Result._by
if diff_results is not None: if fields is None:
diff_results = fold(diff_results, fields = Result._fields
by=['file' if by_file else 'struct']) types = Result._types
# fold again
results = fold(Result, results, by=by)
if diff_results is not None:
diff_results = fold(Result, diff_results, by=by)
# organize by name
table = { table = {
r.file if by_file else r.struct: r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
r.file if by_file else r.struct: r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
# sort, note that python's sort is stable
names = list(table.keys() | diff_table.keys()) names = list(table.keys() | diff_table.keys())
# sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: -IntField.ratio( names.sort(key=lambda n: tuple(
table[n].struct_size if n in table else None, types[k].ratio(
diff_table[n].struct_size if n in diff_table else None)) getattr(table.get(n), k, None),
if size_sort: getattr(diff_table.get(n), k, None))
names.sort(key=lambda n: (table[n].struct_size,) if n in table else (), for k in fields),
reverse=True) reverse=True)
elif reverse_size_sort: if sort:
names.sort(key=lambda n: (table[n].struct_size,) if n in table else (), for k, reverse in reversed(sort):
reverse=False) names.sort(key=lambda n: (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else (),
reverse=reverse ^ (not k or k in Result._fields))
# print header
if not summary: # build up our lines
title = '%s%s' % ( lines = []
'file' if by_file else 'struct',
# header
line = []
line.append('%s%s' % (
','.join(by),
' (%d added, %d removed)' % ( ' (%d added, %d removed)' % (
sum(1 for n in table if n not in diff_table), sum(1 for n in table if n not in diff_table),
sum(1 for n in diff_table if n not in table)) sum(1 for n in diff_table if n not in table))
if diff_results is not None and not percent else '') if diff_results is not None and not percent else '')
name_width = max(it.chain([23, len(title)], (len(n) for n in names))) if not summary else '')
else:
title = ''
name_width = 23
name_width = 4*((name_width+1+4-1)//4)-1
print('%-*s ' % (name_width, title), end='')
if diff_results is None: if diff_results is None:
print(' %s' % ('size'.rjust(len(IntField.none)))) for k in fields:
line.append(k)
elif percent: elif percent:
print(' %s' % ('size'.rjust(len(IntField.diff_none)))) for k in fields:
line.append(k)
else: else:
print(' %s %s %s' % ( for k in fields:
'old'.rjust(len(IntField.diff_none)), line.append('o'+k)
'new'.rjust(len(IntField.diff_none)), for k in fields:
'diff'.rjust(len(IntField.diff_none)))) line.append('n'+k)
for k in fields:
line.append('d'+k)
line.append('')
lines.append(line)
# print entries # entries
if not summary: if not summary:
for name in names: for name in names:
r = table.get(name) r = table.get(name)
if diff_results is not None: if diff_results is not None:
diff_r = diff_table.get(name) diff_r = diff_table.get(name)
ratio = IntField.ratio( ratios = [
r.struct_size if r else None, types[k].ratio(
diff_r.struct_size if diff_r else None) getattr(r, k, None),
if not ratio and not all_: getattr(diff_r, k, None))
for k in fields]
if not any(ratios) and not all_:
continue continue
print('%-*s ' % (name_width, name), end='') line = []
line.append(name)
if diff_results is None: if diff_results is None:
print(' %s' % ( for k in fields:
r.struct_size.table() line.append(getattr(r, k).table()
if r else IntField.none)) if getattr(r, k, None) is not None
else types[k].none)
elif percent: elif percent:
print(' %s%s' % ( for k in fields:
r.struct_size.diff_table() line.append(getattr(r, k).diff_table()
if r else IntField.diff_none, if getattr(r, k, None) is not None
' (%s)' % ( else types[k].diff_none)
'+∞%' if ratio == +m.inf
else '-∞%' if ratio == -m.inf
else '%+.1f%%' % (100*ratio))))
else: else:
print(' %s %s %s%s' % ( for k in fields:
diff_r.struct_size.diff_table() line.append(getattr(diff_r, k).diff_table()
if diff_r else IntField.diff_none, if getattr(diff_r, k, None) is not None
r.struct_size.diff_table() else types[k].diff_none)
if r else IntField.diff_none, for k in fields:
IntField.diff_diff( line.append(getattr(r, k).diff_table()
r.struct_size if r else None, if getattr(r, k, None) is not None
diff_r.struct_size if diff_r else None) else types[k].diff_none)
if r or diff_r else IntField.diff_none, for k in fields:
' (%s)' % ( line.append(types[k].diff_diff(
'+∞%' if ratio == +m.inf getattr(r, k, None),
else '-∞%' if ratio == -m.inf getattr(diff_r, k, None)))
else '%+.1f%%' % (100*ratio)) if diff_results is None:
if ratio else '')) line.append('')
elif percent:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios))
else:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t)
if any(ratios) else '')
lines.append(line)
# print total # total
total = fold(results, by=[]) r = next(iter(fold(Result, results, by=[])), None)
r = total[0] if total else None
if diff_results is not None: if diff_results is not None:
diff_total = fold(diff_results, by=[]) diff_r = next(iter(fold(Result, diff_results, by=[])), None)
diff_r = diff_total[0] if diff_total else None ratios = [
ratio = IntField.ratio( types[k].ratio(
r.struct_size if r else None, getattr(r, k, None),
diff_r.struct_size if diff_r else None) getattr(diff_r, k, None))
for k in fields]
print('%-*s ' % (name_width, 'TOTAL'), end='') line = []
line.append('TOTAL')
if diff_results is None: if diff_results is None:
print(' %s' % ( for k in fields:
r.struct_size.table() line.append(getattr(r, k).table()
if r else IntField.none)) if getattr(r, k, None) is not None
else types[k].none)
elif percent: elif percent:
print(' %s%s' % ( for k in fields:
r.struct_size.diff_table() line.append(getattr(r, k).diff_table()
if r else IntField.diff_none, if getattr(r, k, None) is not None
' (%s)' % ( else types[k].diff_none)
'+∞%' if ratio == +m.inf
else '-∞%' if ratio == -m.inf
else '%+.1f%%' % (100*ratio))))
else: else:
print(' %s %s %s%s' % ( for k in fields:
diff_r.struct_size.diff_table() line.append(getattr(diff_r, k).diff_table()
if diff_r else IntField.diff_none, if getattr(diff_r, k, None) is not None
r.struct_size.diff_table() else types[k].diff_none)
if r else IntField.diff_none, for k in fields:
IntField.diff_diff( line.append(getattr(r, k).diff_table()
r.struct_size if r else None, if getattr(r, k, None) is not None
diff_r.struct_size if diff_r else None) else types[k].diff_none)
if r or diff_r else IntField.diff_none, for k in fields:
' (%s)' % ( line.append(types[k].diff_diff(
'+∞%' if ratio == +m.inf getattr(r, k, None),
else '-∞%' if ratio == -m.inf getattr(diff_r, k, None)))
else '%+.1f%%' % (100*ratio)) if diff_results is None:
if ratio else '')) line.append('')
elif percent:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios))
else:
line.append(' (%s)' % ', '.join(
'+∞%' if t == +m.inf
else '-∞%' if t == -m.inf
else '%+.1f%%' % (100*t)
for t in ratios
if t)
if any(ratios) else '')
lines.append(line)
# find the best widths, note that column 0 contains the names and column -1
# the ratios, so those are handled a bit differently
widths = [
((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
for w, i in zip(
it.chain([23], it.repeat(7)),
range(len(lines[0])-1))]
# print our table
for line in lines:
print('%-*s %s%s' % (
widths[0], line[0],
' '.join('%*s' % (w, x)
for w, x in zip(widths[1:], line[1:-1])),
line[-1]))
def main(obj_paths, **args): def main(obj_paths, *,
by=None,
fields=None,
defines=None,
sort=None,
**args):
# find sizes # find sizes
if not args.get('use', None): if not args.get('use', None):
# find .o files # find .o files
@@ -404,7 +479,7 @@ def main(obj_paths, **args):
paths.append(path) paths.append(path)
if not paths: if not paths:
print('no .obj files found in %r?' % obj_paths) print("error: no .obj files found in %r?" % obj_paths)
sys.exit(-1) sys.exit(-1)
results = collect(paths, **args) results = collect(paths, **args)
@@ -414,25 +489,38 @@ def main(obj_paths, **args):
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
results.append(StructResult(**{ results.append(StructResult(
k: v for k, v in r.items() **{k: r[k] for k in StructResult._by
if k in StructResult._fields})) if k in r and r[k].strip()},
**{k: r['struct_'+k]
for k in StructResult._fields
if 'struct_'+k in r
and r['struct_'+k].strip()}))
except TypeError: except TypeError:
pass pass
# fold to remove duplicates # fold
results = fold(results) results = fold(StructResult, results, by=by, defines=defines)
# sort because why not # sort, note that python's sort is stable
results.sort() results.sort()
if sort:
for k, reverse in reversed(sort):
results.sort(key=lambda r: (getattr(r, k),)
if getattr(r, k) is not None else (),
reverse=reverse ^ (not k or k in StructResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, StructResult._fields) writer = csv.DictWriter(f, StructResult._by
+ ['struct_'+k for k in StructResult._fields])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow(r._asdict()) writer.writerow(
{k: getattr(r, k) for k in StructResult._by}
| {'struct_'+k: getattr(r, k)
for k in StructResult._fields})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -442,22 +530,28 @@ def main(obj_paths, **args):
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
try: try:
diff_results.append(StructResult(**{ diff_results.append(StructResult(
k: v for k, v in r.items() **{k: r[k] for k in StructResult._by
if k in StructResult._fields})) if k in r and r[k].strip()},
**{k: r['struct_'+k]
for k in StructResult._fields
if 'struct_'+k in r
and r['struct_'+k].strip()}))
except TypeError: except TypeError:
pass pass
except FileNotFoundError: except FileNotFoundError:
pass pass
# fold to remove duplicates # fold
diff_results = fold(diff_results) diff_results = fold(StructResult, diff_results, by=by, defines=defines)
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
table( table(StructResult, results,
results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['struct'],
fields=fields,
sort=sort,
**args) **args)
@@ -498,22 +592,39 @@ if __name__ == "__main__":
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by-file', '-b', '--by',
action='store_true', action='append',
help="Group by file. Note this does not include padding " choices=StructResult._by,
"so sizes may differ from other tools.") help="Group by this field.")
parser.add_argument( parser.add_argument(
'-s', '--size-sort', '-f', '--field',
action='store_true', dest='fields',
help="Sort by size.") action='append',
choices=StructResult._fields,
help="Show this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-size-sort', '-D', '--define',
action='store_true', dest='defines',
help="Sort by size, but backwards.") action='append',
type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
help="Only include results where this field is this value.")
class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option):
if namespace.sort is None:
namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument(
'-s', '--sort',
action=AppendSort,
help="Sort by this field.")
parser.add_argument(
'-S', '--reverse-sort',
action=AppendSort,
help="Sort by this field, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total size.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'-A', '--everything', '-A', '--everything',
action='store_true', action='store_true',

776
scripts/summary.py
View File

File diff suppressed because it is too large Load Diff