Tweaked scripts to share more code, added coverage calls/hits

The main change is requiring field names for -b/-f/-s/-S, this
is a bit more powerful, and supports hidden extra fields, but
can require a bit more typing in some cases.

Makefile

@@ -149,23 +149,23 @@ bench-list: bench-runner
 
 .PHONY: code
 code: $(OBJ)
-	./scripts/code.py $^ -S $(CODEFLAGS)
+	./scripts/code.py $^ -Ssize $(CODEFLAGS)
 
 .PHONY: data
 data: $(OBJ)
-	./scripts/data.py $^ -S $(DATAFLAGS)
+	./scripts/data.py $^ -Ssize $(DATAFLAGS)
 
 .PHONY: stack
 stack: $(CI)
-	./scripts/stack.py $^ -S $(STACKFLAGS)
+	./scripts/stack.py $^ -Slimit -Sframe $(STACKFLAGS)
 
 .PHONY: struct
 struct: $(OBJ)
-	./scripts/struct_.py $^ -S $(STRUCTFLAGS)
+	./scripts/struct_.py $^ -Ssize $(STRUCTFLAGS)
 
 .PHONY: coverage
 coverage: $(GCDA)
-	./scripts/coverage.py $^ -s $(COVERAGEFLAGS)
+	./scripts/coverage.py $^ -slines -sbranches $(COVERAGEFLAGS)
 
 .PHONY: summary sizes
 summary sizes: $(BUILDDIR)lfs.csv

scripts/code.py

@@ -29,10 +29,10 @@ TYPE = 'tTrRdD'
 
 
 # integer fields
-class IntField(co.namedtuple('IntField', 'x')):
+class Int(co.namedtuple('Int', 'x')):
     __slots__ = ()
     def __new__(cls, x=0):
-        if isinstance(x, IntField):
+        if isinstance(x, Int):
             return x
         if isinstance(x, str):
             try:
@@ -98,35 +98,30 @@ class IntField(co.namedtuple('IntField', 'x')):
             return (new-old) / old
 
     def __add__(self, other):
-        return IntField(self.x + other.x)
+        return self.__class__(self.x + other.x)
 
     def __sub__(self, other):
-        return IntField(self.x - other.x)
+        return self.__class__(self.x - other.x)
 
     def __mul__(self, other):
-        return IntField(self.x * other.x)
-
-    def __lt__(self, other):
-        return self.x < other.x
-
-    def __gt__(self, other):
-        return self.__class__.__lt__(other, self)
-
-    def __le__(self, other):
-        return not self.__gt__(other)
-
-    def __ge__(self, other):
-        return not self.__lt__(other)
+        return self.__class__(self.x * other.x)
 
 # code size results
-class CodeResult(co.namedtuple('CodeResult', 'file,function,code_size')):
+class CodeResult(co.namedtuple('CodeResult', [
+        'file', 'function',
+        'size'])):
+    _by = ['file', 'function']
+    _fields = ['size']
+    _types = {'size': Int}
+
     __slots__ = ()
-    def __new__(cls, file, function, code_size):
-        return super().__new__(cls, file, function, IntField(code_size))
+    def __new__(cls, file='', function='', size=0):
+        return super().__new__(cls, file, function,
+            Int(size))
 
     def __add__(self, other):
         return CodeResult(self.file, self.function,
-            self.code_size + other.code_size)
+            self.size + other.size)
 
 
 def openio(path, mode='r'):
@@ -188,9 +183,27 @@ def collect(paths, *,
     return results
 
 
-def fold(results, *,
-        by=['file', 'function'],
+def fold(Result, results, *,
+        by=None,
+        defines=None,
         **_):
+    if by is None:
+        by = Result._by
+
+    for k in it.chain(by or [], (k for k, _ in defines or [])):
+        if k not in Result._by and k not in Result._fields:
+            print("error: could not find field %r?" % k)
+            sys.exit(-1)
+
+    # filter by matching defines
+    if defines is not None:
+        results_ = []
+        for r in results:
+            if all(getattr(r, k) in vs for k, vs in defines):
+                results_.append(r)
+        results = results_
+
+    # organize results into conflicts
     folding = co.OrderedDict()
     for r in results:
         name = tuple(getattr(r, k) for k in by)
@@ -198,157 +211,220 @@ def fold(results, *,
             folding[name] = []
         folding[name].append(r)
 
+    # merge conflicts
     folded = []
-    for rs in folding.values():
+    for name, rs in folding.items():
         folded.append(sum(rs[1:], start=rs[0]))
 
     return folded
 
-
-def table(results, diff_results=None, *,
-        by_file=False,
-        size_sort=False,
-        reverse_size_sort=False,
+def table(Result, results, diff_results=None, *,
+        by=None,
+        fields=None,
+        sort=None,
         summary=False,
         all=False,
         percent=False,
         **_):
     all_, all = all, __builtins__.all
 
-    # fold
-    results = fold(results, by=['file' if by_file else 'function'])
-    if diff_results is not None:
-        diff_results = fold(diff_results,
-            by=['file' if by_file else 'function'])
+    if by is None:
+        by = Result._by
+    if fields is None:
+        fields = Result._fields
+    types = Result._types
 
+    # fold again
+    results = fold(Result, results, by=by)
+    if diff_results is not None:
+        diff_results = fold(Result, diff_results, by=by)
+
+    # organize by name
     table = {
-        r.file if by_file else r.function: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in results}
     diff_table = {
-        r.file if by_file else r.function: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in diff_results or []}
-
-    # sort, note that python's sort is stable
     names = list(table.keys() | diff_table.keys())
+
+    # sort again, now with diff info, note that python's sort is stable
     names.sort()
     if diff_results is not None:
-        names.sort(key=lambda n: -IntField.ratio(
-            table[n].code_size if n in table else None,
-            diff_table[n].code_size if n in diff_table else None))
-    if size_sort:
-        names.sort(key=lambda n: (table[n].code_size,) if n in table else (),
+        names.sort(key=lambda n: tuple(
+            types[k].ratio(
+                getattr(table.get(n), k, None),
+                getattr(diff_table.get(n), k, None))
+            for k in fields),
             reverse=True)
-    elif reverse_size_sort:
-        names.sort(key=lambda n: (table[n].code_size,) if n in table else (),
-            reverse=False)
+    if sort:
+        for k, reverse in reversed(sort):
+            names.sort(key=lambda n: (getattr(table[n], k),)
+                if getattr(table.get(n), k, None) is not None else (),
+                reverse=reverse ^ (not k or k in Result._fields))
 
-    # print header
-    if not summary:
-        title = '%s%s' % (
-            'file' if by_file else 'function',
+
+    # build up our lines
+    lines = []
+
+    # header
+    line = []
+    line.append('%s%s' % (
+        ','.join(by),
         ' (%d added, %d removed)' % (
             sum(1 for n in table if n not in diff_table),
             sum(1 for n in diff_table if n not in table))
             if diff_results is not None and not percent else '')
-        name_width = max(it.chain([23, len(title)], (len(n) for n in names)))
-    else:
-        title = ''
-        name_width = 23
-    name_width = 4*((name_width+1+4-1)//4)-1
-
-    print('%-*s ' % (name_width, title), end='')
+        if not summary else '')
     if diff_results is None:
-        print(' %s' % ('size'.rjust(len(IntField.none))))
+        for k in fields:
+            line.append(k)
     elif percent:
-        print(' %s' % ('size'.rjust(len(IntField.diff_none))))
+        for k in fields:
+            line.append(k)
     else:
-        print(' %s %s %s' % (
-            'old'.rjust(len(IntField.diff_none)),
-            'new'.rjust(len(IntField.diff_none)),
-            'diff'.rjust(len(IntField.diff_none))))
+        for k in fields:
+            line.append('o'+k)
+        for k in fields:
+            line.append('n'+k)
+        for k in fields:
+            line.append('d'+k)
+    line.append('')
+    lines.append(line)
 
-    # print entries
+    # entries
     if not summary:
         for name in names:
             r = table.get(name)
             if diff_results is not None:
                 diff_r = diff_table.get(name)
-                ratio = IntField.ratio(
-                    r.code_size if r else None,
-                    diff_r.code_size if diff_r else None)
-                if not ratio and not all_:
+                ratios = [
+                    types[k].ratio(
+                        getattr(r, k, None),
+                        getattr(diff_r, k, None))
+                    for k in fields]
+                if not any(ratios) and not all_:
                     continue
 
-            print('%-*s ' % (name_width, name), end='')
+            line = []
+            line.append(name)
             if diff_results is None:
-                print(' %s' % (
-                    r.code_size.table()
-                        if r else IntField.none))
+                for k in fields:
+                    line.append(getattr(r, k).table()
+                        if getattr(r, k, None) is not None
+                        else types[k].none)
             elif percent:
-                print(' %s%s' % (
-                    r.code_size.diff_table()
-                        if r else IntField.diff_none,
-                    ' (%s)' % (
-                        '+∞%' if ratio == +m.inf
-                        else '-∞%' if ratio == -m.inf
-                        else '%+.1f%%' % (100*ratio))))
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
             else:
-                print(' %s %s %s%s' % (
-                    diff_r.code_size.diff_table()
-                        if diff_r else IntField.diff_none,
-                    r.code_size.diff_table()
-                        if r else IntField.diff_none,
-                    IntField.diff_diff(
-                        r.code_size if r else None,
-                        diff_r.code_size if diff_r else None)
-                        if r or diff_r else IntField.diff_none,
-                    ' (%s)' % (
-                        '+∞%' if ratio == +m.inf
-                        else '-∞%' if ratio == -m.inf
-                        else '%+.1f%%' % (100*ratio))
-                        if ratio else ''))
+                for k in fields:
+                    line.append(getattr(diff_r, k).diff_table()
+                        if getattr(diff_r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(types[k].diff_diff(
+                            getattr(r, k, None),
+                            getattr(diff_r, k, None)))
+            if diff_results is None:
+                line.append('')
+            elif percent:
+                line.append(' (%s)' % ', '.join(
+                    '+∞%' if t == +m.inf
+                    else '-∞%' if t == -m.inf
+                    else '%+.1f%%' % (100*t)
+                    for t in ratios))
+            else:
+                line.append(' (%s)' % ', '.join(
+                        '+∞%' if t == +m.inf
+                        else '-∞%' if t == -m.inf
+                        else '%+.1f%%' % (100*t)
+                        for t in ratios
+                        if t)
+                    if any(ratios) else '')
+            lines.append(line)
 
-    # print total
-    total = fold(results, by=[])
-    r = total[0] if total else None
+    # total
+    r = next(iter(fold(Result, results, by=[])), None)
     if diff_results is not None:
-        diff_total = fold(diff_results, by=[])
-        diff_r = diff_total[0] if diff_total else None
-        ratio = IntField.ratio(
-            r.code_size if r else None,
-            diff_r.code_size if diff_r else None)
+        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
+        ratios = [
+            types[k].ratio(
+                getattr(r, k, None),
+                getattr(diff_r, k, None))
+            for k in fields]
 
-    print('%-*s ' % (name_width, 'TOTAL'), end='')
+    line = []
+    line.append('TOTAL')
     if diff_results is None:
-        print(' %s' % (
-            r.code_size.table()
-                if r else IntField.none))
+        for k in fields:
+            line.append(getattr(r, k).table()
+                if getattr(r, k, None) is not None
+                else types[k].none)
     elif percent:
-        print(' %s%s' % (
-            r.code_size.diff_table()
-                if r else IntField.diff_none,
-            ' (%s)' % (
-                '+∞%' if ratio == +m.inf
-                else '-∞%' if ratio == -m.inf
-                else '%+.1f%%' % (100*ratio))))
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
     else:
-        print(' %s %s %s%s' % (
-            diff_r.code_size.diff_table()
-                if diff_r else IntField.diff_none,
-            r.code_size.diff_table()
-                if r else IntField.diff_none,
-            IntField.diff_diff(
-                r.code_size if r else None,
-                diff_r.code_size if diff_r else None)
-                if r or diff_r else IntField.diff_none,
-            ' (%s)' % (
-                '+∞%' if ratio == +m.inf
-                else '-∞%' if ratio == -m.inf
-                else '%+.1f%%' % (100*ratio))
-                if ratio else ''))
+        for k in fields:
+            line.append(getattr(diff_r, k).diff_table()
+                if getattr(diff_r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(types[k].diff_diff(
+                    getattr(r, k, None),
+                    getattr(diff_r, k, None)))
+    if diff_results is None:
+        line.append('')
+    elif percent:
+        line.append(' (%s)' % ', '.join(
+            '+∞%' if t == +m.inf
+            else '-∞%' if t == -m.inf
+            else '%+.1f%%' % (100*t)
+            for t in ratios))
+    else:
+        line.append(' (%s)' % ', '.join(
+                '+∞%' if t == +m.inf
+                else '-∞%' if t == -m.inf
+                else '%+.1f%%' % (100*t)
+                for t in ratios
+                if t)
+            if any(ratios) else '')
+    lines.append(line)
+
+    # find the best widths, note that column 0 contains the names and column -1
+    # the ratios, so those are handled a bit differently
+    widths = [
+        ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
+        for w, i in zip(
+            it.chain([23], it.repeat(7)),
+            range(len(lines[0])-1))]
+
+    # print our table
+    for line in lines:
+        print('%-*s  %s%s' % (
+            widths[0], line[0],
+            ' '.join('%*s' % (w, x)
+                for w, x in zip(widths[1:], line[1:-1])),
+            line[-1]))
 
 
-def main(obj_paths, **args):
+def main(obj_paths, *,
+        by=None,
+        fields=None,
+        defines=None,
+        sort=None,
+        **args):
     # find sizes
     if not args.get('use', None):
         # find .o files
@@ -361,7 +437,7 @@ def main(obj_paths, **args):
                 paths.append(path)
 
         if not paths:
-            print('no .obj files found in %r?' % obj_paths)
+            print("error: no .obj files found in %r?" % obj_paths)
             sys.exit(-1)
 
         results = collect(paths, **args)
@@ -371,25 +447,35 @@ def main(obj_paths, **args):
             reader = csv.DictReader(f, restval='')
             for r in reader:
                 try:
-                    results.append(CodeResult(**{
-                        k: v for k, v in r.items()
-                        if k in CodeResult._fields}))
+                    results.append(CodeResult(
+                        **{k: r[k] for k in CodeResult._by
+                            if k in r and r[k].strip()},
+                        **{k: r['code_'+k] for k in CodeResult._fields
+                            if 'code_'+k in r and r['code_'+k].strip()}))
                 except TypeError:
                     pass
 
-    # fold to remove duplicates
-    results = fold(results)
+    # fold
+    results = fold(CodeResult, results, by=by, defines=defines)
 
-    # sort because why not
+    # sort, note that python's sort is stable
     results.sort()
+    if sort:
+        for k, reverse in reversed(sort):
+            results.sort(key=lambda r: (getattr(r, k),)
+                if getattr(r, k) is not None else (),
+                reverse=reverse ^ (not k or k in CodeResult._fields))
 
     # write results to CSV
     if args.get('output'):
         with openio(args['output'], 'w') as f:
-            writer = csv.DictWriter(f, CodeResult._fields)
+            writer = csv.DictWriter(f, CodeResult._by
+                + ['code_'+k for k in CodeResult._fields])
             writer.writeheader()
             for r in results:
-                writer.writerow(r._asdict())
+                writer.writerow(
+                    {k: getattr(r, k) for k in CodeResult._by}
+                    | {'code_'+k: getattr(r, k) for k in CodeResult._fields})
 
     # find previous results?
     if args.get('diff'):
@@ -399,22 +485,26 @@ def main(obj_paths, **args):
                 reader = csv.DictReader(f, restval='')
                 for r in reader:
                     try:
-                        diff_results.append(CodeResult(**{
-                            k: v for k, v in r.items()
-                            if k in CodeResult._fields}))
+                        diff_results.append(CodeResult(
+                            **{k: r[k] for k in CodeResult._by
+                                if k in r and r[k].strip()},
+                            **{k: r['code_'+k] for k in CodeResult._fields
+                                if 'code_'+k in r and r['code_'+k].strip()}))
                     except TypeError:
                         pass
         except FileNotFoundError:
             pass
 
-        # fold to remove duplicates
-        diff_results = fold(diff_results)
+        # fold
+        diff_results = fold(CodeResult, diff_results, by=by, defines=defines)
 
     # print table
     if not args.get('quiet'):
-        table(
-            results,
+        table(CodeResult, results,
             diff_results if args.get('diff') else None,
+            by=by if by is not None else ['function'],
+            fields=fields,
+            sort=sort,
             **args)
 
 
@@ -455,22 +545,39 @@ if __name__ == "__main__":
         action='store_true',
         help="Only show percentage change, not a full diff.")
     parser.add_argument(
-        '-b', '--by-file',
-        action='store_true',
-        help="Group by file. Note this does not include padding "
-            "so sizes may differ from other tools.")
+        '-b', '--by',
+        action='append',
+        choices=CodeResult._by,
+        help="Group by this field.")
     parser.add_argument(
-        '-s', '--size-sort',
-        action='store_true',
-        help="Sort by size.")
+        '-f', '--field',
+        dest='fields',
+        action='append',
+        choices=CodeResult._fields,
+        help="Show this field.")
     parser.add_argument(
-        '-S', '--reverse-size-sort',
-        action='store_true',
-        help="Sort by size, but backwards.")
+        '-D', '--define',
+        dest='defines',
+        action='append',
+        type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
+        help="Only include results where this field is this value.")
+    class AppendSort(argparse.Action):
+        def __call__(self, parser, namespace, value, option):
+            if namespace.sort is None:
+                namespace.sort = []
+            namespace.sort.append((value, True if option == '-S' else False))
+    parser.add_argument(
+        '-s', '--sort',
+        action=AppendSort,
+        help="Sort by this fields.")
+    parser.add_argument(
+        '-S', '--reverse-sort',
+        action=AppendSort,
+        help="Sort by this fields, but backwards.")
     parser.add_argument(
         '-Y', '--summary',
         action='store_true',
-        help="Only show the total size.")
+        help="Only show the total.")
     parser.add_argument(
         '-A', '--everything',
         action='store_true',

scripts/coverage.py

@@ -30,10 +30,10 @@ GCOV_TOOL = ['gcov']
 
 
 # integer fields
-class IntField(co.namedtuple('IntField', 'x')):
+class Int(co.namedtuple('Int', 'x')):
     __slots__ = ()
     def __new__(cls, x=0):
-        if isinstance(x, IntField):
+        if isinstance(x, Int):
             return x
         if isinstance(x, str):
             try:
@@ -99,37 +99,25 @@ class IntField(co.namedtuple('IntField', 'x')):
             return (new-old) / old
 
     def __add__(self, other):
-        return IntField(self.x + other.x)
+        return self.__class__(self.x + other.x)
 
     def __sub__(self, other):
-        return IntField(self.x - other.x)
+        return self.__class__(self.x - other.x)
 
     def __mul__(self, other):
-        return IntField(self.x * other.x)
-
-    def __lt__(self, other):
-        return self.x < other.x
-
-    def __gt__(self, other):
-        return self.__class__.__lt__(other, self)
-
-    def __le__(self, other):
-        return not self.__gt__(other)
-
-    def __ge__(self, other):
-        return not self.__lt__(other)
+        return self.__class__(self.x * other.x)
 
 # fractional fields, a/b
-class FracField(co.namedtuple('FracField', 'a,b')):
+class Frac(co.namedtuple('Frac', 'a,b')):
     __slots__ = ()
     def __new__(cls, a=0, b=None):
-        if isinstance(a, FracField) and b is None:
+        if isinstance(a, Frac) and b is None:
             return a
         if isinstance(a, str) and b is None:
             a, b = a.split('/', 1)
         if b is None:
             b = a
-        return super().__new__(cls, IntField(a), IntField(b))
+        return super().__new__(cls, Int(a), Int(b))
 
     def __str__(self):
         return '%s/%s' % (self.a, self.b)
@@ -139,10 +127,7 @@ class FracField(co.namedtuple('FracField', 'a,b')):
 
     none = '%11s %7s' % ('-', '-')
     def table(self):
-        if not self.b.x:
-            return self.none
-
-        t = self.a.x/self.b.x
+        t = self.a.x/self.b.x if self.b.x else 1.0
         return '%11s %7s' % (
             self,
             '∞%' if t == +m.inf
@@ -151,38 +136,35 @@ class FracField(co.namedtuple('FracField', 'a,b')):
 
     diff_none = '%11s' % '-'
     def diff_table(self):
-        if not self.b.x:
-            return self.diff_none
-
         return '%11s' % (self,)
 
     def diff_diff(self, other):
-        new_a, new_b = self if self else (IntField(0), IntField(0))
-        old_a, old_b = other if other else (IntField(0), IntField(0))
+        new_a, new_b = self if self else (Int(0), Int(0))
+        old_a, old_b = other if other else (Int(0), Int(0))
         return '%11s' % ('%s/%s' % (
             new_a.diff_diff(old_a).strip(),
             new_b.diff_diff(old_b).strip()))
 
     def ratio(self, other):
-        new_a, new_b = self if self else (IntField(0), IntField(0))
-        old_a, old_b = other if other else (IntField(0), IntField(0))
+        new_a, new_b = self if self else (Int(0), Int(0))
+        old_a, old_b = other if other else (Int(0), Int(0))
         new = new_a.x/new_b.x if new_b.x else 1.0
         old = old_a.x/old_b.x if old_b.x else 1.0
         return new - old
 
     def __add__(self, other):
-        return FracField(self.a + other.a, self.b + other.b)
+        return self.__class__(self.a + other.a, self.b + other.b)
 
     def __sub__(self, other):
-        return FracField(self.a - other.a, self.b - other.b)
+        return self.__class__(self.a - other.a, self.b - other.b)
 
     def __mul__(self, other):
-        return FracField(self.a * other.a, self.b + other.b)
+        return self.__class__(self.a * other.a, self.b + other.b)
 
     def __lt__(self, other):
-        self_r = self.a.x/self.b.x if self.b.x else -m.inf
-        other_r = other.a.x/other.b.x if other.b.x else -m.inf
-        return self_r < other_r
+        self_t = self.a.x/self.b.x if self.b.x else 1.0
+        other_t = other.a.x/other.b.x if other.b.x else 1.0
+        return (self_t, self.a.x) < (other_t, other.a.x)
 
     def __gt__(self, other):
         return self.__class__.__lt__(other, self)
@@ -194,22 +176,28 @@ class FracField(co.namedtuple('FracField', 'a,b')):
         return not self.__lt__(other)
 
 # coverage results
-class CoverageResult(co.namedtuple('CoverageResult',
-        'file,function,line,'
-        'coverage_hits,coverage_lines,coverage_branches')):
+class CoverageResult(co.namedtuple('CoverageResult', [
+        'file', 'function', 'line',
+        'calls', 'hits', 'funcs', 'lines', 'branches'])):
+    _by = ['file', 'function', 'line']
+    _fields = ['calls', 'hits', 'funcs', 'lines', 'branches']
+    _types = {
+        'calls': Int, 'hits': Int,
+        'funcs': Frac, 'lines': Frac, 'branches': Frac}
+
     __slots__ = ()
-    def __new__(cls, file, function, line,
-            coverage_hits, coverage_lines, coverage_branches):
-        return super().__new__(cls, file, function, int(IntField(line)),
-            IntField(coverage_hits),
-            FracField(coverage_lines),
-            FracField(coverage_branches))
+    def __new__(cls, file='', function='', line=0,
+            calls=0, hits=0, funcs=0, lines=0, branches=0):
+        return super().__new__(cls, file, function, int(Int(line)),
+            Int(calls), Int(hits), Frac(funcs), Frac(lines), Frac(branches))
 
     def __add__(self, other):
         return CoverageResult(self.file, self.function, self.line,
-            max(self.coverage_hits, other.coverage_hits),
-            self.coverage_lines + other.coverage_lines,
-            self.coverage_branches + other.coverage_branches)
+            max(self.calls, other.calls),
+            max(self.hits, other.hits),
+            self.funcs + other.funcs,
+            self.lines + other.lines,
+            self.branches + other.branches)
 
 
 def openio(path, mode='r'):
@@ -257,20 +245,37 @@ def collect(paths, *,
             if file['file'] != src_path:
                 continue
 
-            for line in file['lines']:
-                func = line.get('function_name', '(inlined)')
+            for func in file['functions']:
+                func_name = func.get('name', '(inlined)')
                 # discard internal function (this includes injected test cases)
                 if not everything:
-                    if func.startswith('__'):
+                    if func_name.startswith('__'):
                         continue
 
+                # go ahead and add functions, later folding will merge this if
+                # there are other hits on this line
                 results.append(CoverageResult(
-                    src_path, func, line['line_number'],
-                    line['count'],
-                    FracField(
-                        1 if line['count'] > 0 else 0,
-                        1),
-                    FracField(
+                    src_path, func_name, func['start_line'],
+                    func['execution_count'], 0,
+                    Frac(1 if func['execution_count'] > 0 else 0, 1),
+                    0,
+                    0))
+
+            for line in file['lines']:
+                func_name = line.get('function_name', '(inlined)')
+                # discard internal function (this includes injected test cases)
+                if not everything:
+                    if func_name.startswith('__'):
+                        continue
+
+                # go ahead and add lines, later folding will merge this if
+                # there are other hits on this line
+                results.append(CoverageResult(
+                    src_path, func_name, line['line_number'],
+                    0, line['count'],
+                    0,
+                    Frac(1 if line['count'] > 0 else 0, 1),
+                    Frac(
                         sum(1 if branch['count'] > 0 else 0
                             for branch in line['branches']),
                         len(line['branches']))))
@@ -278,9 +283,27 @@ def collect(paths, *,
     return results
 
 
-def fold(results, *,
-        by=['file', 'function', 'line'],
+def fold(Result, results, *,
+        by=None,
+        defines=None,
         **_):
+    if by is None:
+        by = Result._by
+
+    for k in it.chain(by or [], (k for k, _ in defines or [])):
+        if k not in Result._by and k not in Result._fields:
+            print("error: could not find field %r?" % k)
+            sys.exit(-1)
+
+    # filter by matching defines
+    if defines is not None:
+        results_ = []
+        for r in results:
+            if all(getattr(r, k) in vs for k, vs in defines):
+                results_.append(r)
+        results = results_
+
+    # organize results into conflicts
     folding = co.OrderedDict()
     for r in results:
         name = tuple(getattr(r, k) for k in by)
@@ -288,231 +311,224 @@ def fold(results, *,
             folding[name] = []
         folding[name].append(r)
 
+    # merge conflicts
     folded = []
-    for rs in folding.values():
+    for name, rs in folding.items():
         folded.append(sum(rs[1:], start=rs[0]))
 
     return folded
 
-
-def table(results, diff_results=None, *,
-        by_file=False,
-        by_line=False,
-        line_sort=False,
-        reverse_line_sort=False,
-        branch_sort=False,
-        reverse_branch_sort=False,
+def table(Result, results, diff_results=None, *,
+        by=None,
+        fields=None,
+        sort=None,
         summary=False,
         all=False,
         percent=False,
         **_):
     all_, all = all, __builtins__.all
 
-    # fold
-    results = fold(results,
-        by=['file', 'line'] if by_line
-            else ['file'] if by_file
-            else ['function'])
-    if diff_results is not None:
-        diff_results = fold(diff_results,
-            by=['file', 'line'] if by_line
-                else ['file'] if by_file
-                else ['function'])
+    if by is None:
+        by = Result._by
+    if fields is None:
+        fields = Result._fields
+    types = Result._types
 
+    # fold again
+    results = fold(Result, results, by=by)
+    if diff_results is not None:
+        diff_results = fold(Result, diff_results, by=by)
+
+    # organize by name
     table = {
-        '%s:%s' % (r.file, r.line) if by_line
-            else r.file if by_file
-            else r.function: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in results}
     diff_table = {
-        '%s:%s' % (r.file, r.line) if by_line
-            else r.file if by_file
-            else r.function: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in diff_results or []}
-
-    # sort, note that python's sort is stable
     names = list(table.keys() | diff_table.keys())
+
+    # sort again, now with diff info, note that python's sort is stable
     names.sort()
     if diff_results is not None:
-        names.sort(key=lambda n: (
-            -FracField.ratio(
-                table[n].coverage_lines if n in table else None,
-                diff_table[n].coverage_lines if n in diff_table else None),
-            -FracField.ratio(
-                table[n].coverage_branches if n in table else None,
-                diff_table[n].coverage_branches if n in diff_table else None)))
-    if line_sort:
-        names.sort(key=lambda n: (table[n].coverage_lines,)
-            if n in table else (),
+        names.sort(key=lambda n: tuple(
+            types[k].ratio(
+                getattr(table.get(n), k, None),
+                getattr(diff_table.get(n), k, None))
+            for k in fields),
             reverse=True)
-    elif reverse_line_sort:
-        names.sort(key=lambda n: (table[n].coverage_lines,)
-            if n in table else (),
-            reverse=False)
-    elif branch_sort:
-        names.sort(key=lambda n: (table[n].coverage_branches,)
-            if n in table else (),
-            reverse=True)
-    elif reverse_branch_sort:
-        names.sort(key=lambda n: (table[n].coverage_branches,)
-            if n in table else (),
-            reverse=False)
+    if sort:
+        for k, reverse in reversed(sort):
+            names.sort(key=lambda n: (getattr(table[n], k),)
+                if getattr(table.get(n), k, None) is not None else (),
+                reverse=reverse ^ (not k or k in Result._fields))
 
-    # print header
-    if not summary:
-        title = '%s%s' % (
-            'line' if by_line else 'file' if by_file else 'function',
+
+    # build up our lines
+    lines = []
+
+    # header
+    line = []
+    line.append('%s%s' % (
+        ','.join(by),
         ' (%d added, %d removed)' % (
             sum(1 for n in table if n not in diff_table),
             sum(1 for n in diff_table if n not in table))
             if diff_results is not None and not percent else '')
-        name_width = max(it.chain([23, len(title)], (len(n) for n in names)))
-    else:
-        title = ''
-        name_width = 23
-    name_width = 4*((name_width+1+4-1)//4)-1
-
-    print('%-*s ' % (name_width, title), end='')
+        if not summary else '')
     if diff_results is None:
-        print(' %s %s' % (
-            'hits/line'.rjust(len(FracField.none)),
-            'hits/branch'.rjust(len(FracField.none))))
+        for k in fields:
+            line.append(k)
     elif percent:
-        print(' %s %s' % (
-            'hits/line'.rjust(len(FracField.diff_none)),
-            'hits/branch'.rjust(len(FracField.diff_none))))
+        for k in fields:
+            line.append(k)
     else:
-        print(' %s %s %s %s %s %s' % (
-            'oh/line'.rjust(len(FracField.diff_none)),
-            'oh/branch'.rjust(len(FracField.diff_none)),
-            'nh/line'.rjust(len(FracField.diff_none)),
-            'nh/branch'.rjust(len(FracField.diff_none)),
-            'dh/line'.rjust(len(FracField.diff_none)),
-            'dh/branch'.rjust(len(FracField.diff_none))))
+        for k in fields:
+            line.append('o'+k)
+        for k in fields:
+            line.append('n'+k)
+        for k in fields:
+            line.append('d'+k)
+    line.append('')
+    lines.append(line)
 
-    # print entries
+    # entries
     if not summary:
         for name in names:
             r = table.get(name)
             if diff_results is not None:
                 diff_r = diff_table.get(name)
-                line_ratio = FracField.ratio(
-                    r.coverage_lines if r else None,
-                    diff_r.coverage_lines if diff_r else None)
-                branch_ratio = FracField.ratio(
-                    r.coverage_branches if r else None,
-                    diff_r.coverage_branches if diff_r else None)
-                if not line_ratio and not branch_ratio and not all_:
+                ratios = [
+                    types[k].ratio(
+                        getattr(r, k, None),
+                        getattr(diff_r, k, None))
+                    for k in fields]
+                if not any(ratios) and not all_:
                     continue
 
-            print('%-*s ' % (name_width, name), end='')
+            line = []
+            line.append(name)
             if diff_results is None:
-                print(' %s %s' % (
-                    r.coverage_lines.table()
-                        if r else FracField.none,
-                    r.coverage_branches.table()
-                        if r else FracField.none))
+                for k in fields:
+                    line.append(getattr(r, k).table()
+                        if getattr(r, k, None) is not None
+                        else types[k].none)
             elif percent:
-                print(' %s %s%s' % (
-                    r.coverage_lines.diff_table()
-                        if r else FracField.diff_none,
-                    r.coverage_branches.diff_table()
-                        if r else FracField.diff_none,
-                    ' (%s)' % ', '.join(
-                            '+∞%' if t == +m.inf
-                            else '-∞%' if t == -m.inf
-                            else '%+.1f%%' % (100*t)
-                            for t in [line_ratio, branch_ratio])))
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
             else:
-                print(' %s %s %s %s %s %s%s' % (
-                    diff_r.coverage_lines.diff_table()
-                        if diff_r else FracField.diff_none,
-                    diff_r.coverage_branches.diff_table()
-                        if diff_r else FracField.diff_none,
-                    r.coverage_lines.diff_table()
-                        if r else FracField.diff_none,
-                    r.coverage_branches.diff_table()
-                        if r else FracField.diff_none,
-                    FracField.diff_diff(
-                        r.coverage_lines if r else None,
-                        diff_r.coverage_lines if diff_r else None)
-                        if r or diff_r else FracField.diff_none,
-                    FracField.diff_diff(
-                        r.coverage_branches if r else None,
-                        diff_r.coverage_branches if diff_r else None)
-                        if r or diff_r else FracField.diff_none,
-                    ' (%s)' % ', '.join(
+                for k in fields:
+                    line.append(getattr(diff_r, k).diff_table()
+                        if getattr(diff_r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(types[k].diff_diff(
+                            getattr(r, k, None),
+                            getattr(diff_r, k, None)))
+            if diff_results is None:
+                line.append('')
+            elif percent:
+                line.append(' (%s)' % ', '.join(
                     '+∞%' if t == +m.inf
                     else '-∞%' if t == -m.inf
                     else '%+.1f%%' % (100*t)
-                            for t in [line_ratio, branch_ratio]
+                    for t in ratios))
+            else:
+                line.append(' (%s)' % ', '.join(
+                        '+∞%' if t == +m.inf
+                        else '-∞%' if t == -m.inf
+                        else '%+.1f%%' % (100*t)
+                        for t in ratios
                         if t)
-                        if line_ratio or branch_ratio else ''))
+                    if any(ratios) else '')
+            lines.append(line)
 
-    # print total
-    total = fold(results, by=[])
-    r = total[0] if total else None
+    # total
+    r = next(iter(fold(Result, results, by=[])), None)
     if diff_results is not None:
-        diff_total = fold(diff_results, by=[])
-        diff_r = diff_total[0] if diff_total else None
-        line_ratio = FracField.ratio(
-            r.coverage_lines if r else None,
-            diff_r.coverage_lines if diff_r else None)
-        branch_ratio = FracField.ratio(
-            r.coverage_branches if r else None,
-            diff_r.coverage_branches if diff_r else None)
+        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
+        ratios = [
+            types[k].ratio(
+                getattr(r, k, None),
+                getattr(diff_r, k, None))
+            for k in fields]
 
-    print('%-*s ' % (name_width, 'TOTAL'), end='')
+    line = []
+    line.append('TOTAL')
     if diff_results is None:
-        print(' %s %s' % (
-            r.coverage_lines.table()
-                if r else FracField.none,
-            r.coverage_branches.table()
-                if r else FracField.none))
+        for k in fields:
+            line.append(getattr(r, k).table()
+                if getattr(r, k, None) is not None
+                else types[k].none)
     elif percent:
-        print(' %s %s%s' % (
-            r.coverage_lines.diff_table()
-                if r else FracField.diff_none,
-            r.coverage_branches.diff_table()
-                if r else FracField.diff_none,
-            ' (%s)' % ', '.join(
-                    '+∞%' if t == +m.inf
-                    else '-∞%' if t == -m.inf
-                    else '%+.1f%%' % (100*t)
-                    for t in [line_ratio, branch_ratio])))
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
     else:
-        print(' %s %s %s %s %s %s%s' % (
-            diff_r.coverage_lines.diff_table()
-                if diff_r else FracField.diff_none,
-            diff_r.coverage_branches.diff_table()
-                if diff_r else FracField.diff_none,
-            r.coverage_lines.diff_table()
-                if r else FracField.diff_none,
-            r.coverage_branches.diff_table()
-                if r else FracField.diff_none,
-            FracField.diff_diff(
-                r.coverage_lines if r else None,
-                diff_r.coverage_lines if diff_r else None)
-                if r or diff_r else FracField.diff_none,
-            FracField.diff_diff(
-                r.coverage_branches if r else None,
-                diff_r.coverage_branches if diff_r else None)
-                if r or diff_r else FracField.diff_none,
-            ' (%s)' % ', '.join(
+        for k in fields:
+            line.append(getattr(diff_r, k).diff_table()
+                if getattr(diff_r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(types[k].diff_diff(
+                    getattr(r, k, None),
+                    getattr(diff_r, k, None)))
+    if diff_results is None:
+        line.append('')
+    elif percent:
+        line.append(' (%s)' % ', '.join(
             '+∞%' if t == +m.inf
             else '-∞%' if t == -m.inf
             else '%+.1f%%' % (100*t)
-                    for t in [line_ratio, branch_ratio]
+            for t in ratios))
+    else:
+        line.append(' (%s)' % ', '.join(
+                '+∞%' if t == +m.inf
+                else '-∞%' if t == -m.inf
+                else '%+.1f%%' % (100*t)
+                for t in ratios
                 if t)
-                if line_ratio or branch_ratio else ''))
+            if any(ratios) else '')
+    lines.append(line)
+
+    # find the best widths, note that column 0 contains the names and column -1
+    # the ratios, so those are handled a bit differently
+    widths = [
+        ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
+        for w, i in zip(
+            it.chain([23], it.repeat(7)),
+            range(len(lines[0])-1))]
+
+    # print our table
+    for line in lines:
+        print('%-*s  %s%s' % (
+            widths[0], line[0],
+            ' '.join('%*s' % (w, x)
+                for w, x in zip(widths[1:], line[1:-1])),
+            line[-1]))
 
 
-def annotate(paths, results, *,
+def annotate(Result, results, paths, *,
         annotate=False,
         lines=False,
         branches=False,
         build_dir=None,
         **args):
+    # if neither branches/lines specified, color both
+    if annotate and not lines and not branches:
+        lines, branches = True, True
+
     for path in paths:
         # map to source file
         src_path = re.sub('\.t\.a\.gcda$', '.c', path)
@@ -521,7 +537,7 @@ def annotate(paths, results, *,
                 src_path)
 
         # flatten to line info
-        results = fold(results, by=['file', 'line'])
+        results = fold(Result, results, by=['file', 'line'])
         table = {r.line: r for r in results if r.file == src_path}
 
         # calculate spans to show
@@ -529,10 +545,8 @@ def annotate(paths, results, *,
             spans = []
             last = None
             for line, r in sorted(table.items()):
-                if ((lines and int(r.coverage_hits) == 0)
-                        or (branches
-                            and r.coverage_branches.a
-                                < r.coverage_branches.b)):
+                if ((lines and int(r.hits) == 0)
+                        or (branches and r.branches.a < r.branches.b)):
                     if last is not None and line - last.stop <= args['context']:
                         last = range(
                             last.start,
@@ -568,24 +582,29 @@ def annotate(paths, results, *,
 
                 if i+1 in table:
                     r = table[i+1]
-                    line = '%-*s // %s hits, %s branches' % (
+                    line = '%-*s // %s hits%s' % (
                         args['width'],
                         line,
-                        r.coverage_hits,
-                        r.coverage_branches)
+                        r.hits,
+                        ', %s branches' % (r.branches,)
+                            if int(r.branches.b) else '')
 
                     if args['color']:
-                        if lines and int(r.coverage_hits) == 0:
+                        if lines and int(r.hits) == 0:
                             line = '\x1b[1;31m%s\x1b[m' % line
-                        elif (branches
-                                and r.coverage_branches.a
-                                    < r.coverage_branches.b):
+                        elif branches and r.branches.a < r.branches.b:
                             line = '\x1b[35m%s\x1b[m' % line
 
                 print(line)
 
 
-def main(gcda_paths, **args):
+def main(gcda_paths, *,
+        by=None,
+        fields=None,
+        defines=None,
+        sort=None,
+        hits=False,
+        **args):
     # figure out what color should be
     if args.get('color') == 'auto':
         args['color'] = sys.stdout.isatty()
@@ -606,7 +625,7 @@ def main(gcda_paths, **args):
                 paths.append(path)
 
         if not paths:
-            print('no .gcda files found in %r?' % gcda_paths)
+            print("error: no .gcda files found in %r?" % gcda_paths)
             sys.exit(-1)
 
         results = collect(paths, **args)
@@ -616,25 +635,38 @@ def main(gcda_paths, **args):
             reader = csv.DictReader(f, restval='')
             for r in reader:
                 try:
-                    results.append(CoverageResult(**{
-                        k: v for k, v in r.items()
-                        if k in CoverageResult._fields}))
+                    results.append(CoverageResult(
+                        **{k: r[k] for k in CoverageResult._by
+                            if k in r and r[k].strip()},
+                        **{k: r['coverage_'+k]
+                            for k in CoverageResult._fields
+                            if 'coverage_'+k in r
+                                and r['coverage_'+k].strip()}))
                 except TypeError:
                     pass
 
-    # fold to remove duplicates
-    results = fold(results)
+    # fold
+    results = fold(CoverageResult, results, by=by, defines=defines)
 
-    # sort because why not
+    # sort, note that python's sort is stable
     results.sort()
+    if sort:
+        for k, reverse in reversed(sort):
+            results.sort(key=lambda r: (getattr(r, k),)
+                if getattr(r, k) is not None else (),
+                reverse=reverse ^ (not k or k in CoverageResult._fields))
 
     # write results to CSV
     if args.get('output'):
         with openio(args['output'], 'w') as f:
-            writer = csv.DictWriter(f, CoverageResult._fields)
+            writer = csv.DictWriter(f, CoverageResult._by
+                + ['coverage_'+k for k in CoverageResult._fields])
             writer.writeheader()
             for r in results:
-                writer.writerow(r._asdict())
+                writer.writerow(
+                    {k: getattr(r, k) for k in CoverageResult._by}
+                    | {'coverage_'+k: getattr(r, k)
+                        for k in CoverageResult._fields})
 
     # find previous results?
     if args.get('diff'):
@@ -644,31 +676,39 @@ def main(gcda_paths, **args):
                 reader = csv.DictReader(f, restval='')
                 for r in reader:
                     try:
-                        diff_results.append(CoverageResult(**{
-                            k: v for k, v in r.items()
-                            if k in CoverageResult._fields}))
+                        diff_results.append(CoverageResult(
+                            **{k: r[k] for k in CoverageResult._by
+                                if k in r and r[k].strip()},
+                            **{k: r['coverage_'+k]
+                                for k in CoverageResult._fields
+                                if 'coverage_'+k in r
+                                    and r['coverage_'+k].strip()}))
                     except TypeError:
                         pass
         except FileNotFoundError:
             pass
 
-        # fold to remove duplicates
-        diff_results = fold(diff_results)
+        # fold
+        diff_results = fold(CoverageResult, diff_results,
+            by=by, defines=defines)
 
+    # print table
     if not args.get('quiet'):
         if (args.get('annotate')
                 or args.get('lines')
                 or args.get('branches')):
             # annotate sources
-            annotate(
-                paths,
-                results,
+            annotate(CoverageResult, results, paths,
                 **args)
         else:
             # print table
-            table(
-                results,
+            table(CoverageResult, results,
                 diff_results if args.get('diff') else None,
+                by=by if by is not None else ['function'],
+                fields=fields if fields is not None
+                    else ['lines', 'branches'] if not hits
+                    else ['calls', 'hits'],
+                sort=sort,
                 **args)
 
     # catch lack of coverage
@@ -717,33 +757,47 @@ if __name__ == "__main__":
         action='store_true',
         help="Only show percentage change, not a full diff.")
     parser.add_argument(
-        '-b', '--by-file',
-        action='store_true',
-        help="Group by file.")
+        '-b', '--by',
+        action='append',
+        choices=CoverageResult._by,
+        help="Group by this field.")
     parser.add_argument(
-        '--by-line',
-        action='store_true',
-        help="Group by line.")
+        '-f', '--field',
+        dest='fields',
+        action='append',
+        choices=CoverageResult._fields,
+        help="Show this field.")
     parser.add_argument(
-        '-s', '--line-sort',
-        action='store_true',
-        help="Sort by line coverage.")
+        '-D', '--define',
+        dest='defines',
+        action='append',
+        type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
+        help="Only include results where this field is this value.")
+    class AppendSort(argparse.Action):
+        def __call__(self, parser, namespace, value, option):
+            if namespace.sort is None:
+                namespace.sort = []
+            namespace.sort.append((value, True if option == '-S' else False))
     parser.add_argument(
-        '-S', '--reverse-line-sort',
-        action='store_true',
-        help="Sort by line coverage, but backwards.")
+        '-s', '--sort',
+        action=AppendSort,
+        help="Sort by this field.")
     parser.add_argument(
-        '--branch-sort',
-        action='store_true',
-        help="Sort by branch coverage.")
-    parser.add_argument(
-        '--reverse-branch-sort',
-        action='store_true',
-        help="Sort by branch coverage, but backwards.")
+        '-S', '--reverse-sort',
+        action=AppendSort,
+        help="Sort by this field, but backwards.")
     parser.add_argument(
         '-Y', '--summary',
         action='store_true',
-        help="Only show the total size.")
+        help="Only show the total.")
+    parser.add_argument(
+        '-A', '--everything',
+        action='store_true',
+        help="Include builtin and libc specific symbols.")
+    parser.add_argument(
+        '-H', '--hits',
+        action='store_true',
+        help="Show total hits instead of coverage.")
     parser.add_argument(
         '-l', '--annotate',
         action='store_true',
@@ -779,10 +833,6 @@ if __name__ == "__main__":
         '-E', '--error-on-branches',
         action='store_true',
         help="Error if any branches are not covered.")
-    parser.add_argument(
-        '-A', '--everything',
-        action='store_true',
-        help="Include builtin and libc specific symbols.")
     parser.add_argument(
         '--gcov-tool',
         default=GCOV_TOOL,

scripts/data.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Script to find data size at the function level. Basically just a bit wrapper
+# Script to find data size at the function level. Basically just a big wrapper
 # around nm with some extra conveniences for comparing builds. Heavily inspired
 # by Linux's Bloat-O-Meter.
 #
@@ -28,12 +28,11 @@ NM_TOOL = ['nm']
 TYPE = 'dDbB'
 
 
-
 # integer fields
-class IntField(co.namedtuple('IntField', 'x')):
+class Int(co.namedtuple('Int', 'x')):
     __slots__ = ()
     def __new__(cls, x=0):
-        if isinstance(x, IntField):
+        if isinstance(x, Int):
             return x
         if isinstance(x, str):
             try:
@@ -99,35 +98,30 @@ class IntField(co.namedtuple('IntField', 'x')):
             return (new-old) / old
 
     def __add__(self, other):
-        return IntField(self.x + other.x)
+        return self.__class__(self.x + other.x)
 
     def __sub__(self, other):
-        return IntField(self.x - other.x)
+        return self.__class__(self.x - other.x)
 
     def __mul__(self, other):
-        return IntField(self.x * other.x)
-
-    def __lt__(self, other):
-        return self.x < other.x
-
-    def __gt__(self, other):
-        return self.__class__.__lt__(other, self)
-
-    def __le__(self, other):
-        return not self.__gt__(other)
-
-    def __ge__(self, other):
-        return not self.__lt__(other)
+        return self.__class__(self.x * other.x)
 
 # data size results
-class DataResult(co.namedtuple('DataResult', 'file,function,data_size')):
+class DataResult(co.namedtuple('DataResult', [
+        'file', 'function',
+        'size'])):
+    _by = ['file', 'function']
+    _fields = ['size']
+    _types = {'size': Int}
+
     __slots__ = ()
-    def __new__(cls, file, function, data_size):
-        return super().__new__(cls, file, function, IntField(data_size))
+    def __new__(cls, file='', function='', size=0):
+        return super().__new__(cls, file, function,
+            Int(size))
 
     def __add__(self, other):
         return DataResult(self.file, self.function,
-            self.data_size + other.data_size)
+            self.size + other.size)
 
 
 def openio(path, mode='r'):
@@ -189,9 +183,27 @@ def collect(paths, *,
     return results
 
 
-def fold(results, *,
-        by=['file', 'function'],
+def fold(Result, results, *,
+        by=None,
+        defines=None,
         **_):
+    if by is None:
+        by = Result._by
+
+    for k in it.chain(by or [], (k for k, _ in defines or [])):
+        if k not in Result._by and k not in Result._fields:
+            print("error: could not find field %r?" % k)
+            sys.exit(-1)
+
+    # filter by matching defines
+    if defines is not None:
+        results_ = []
+        for r in results:
+            if all(getattr(r, k) in vs for k, vs in defines):
+                results_.append(r)
+        results = results_
+
+    # organize results into conflicts
     folding = co.OrderedDict()
     for r in results:
         name = tuple(getattr(r, k) for k in by)
@@ -199,157 +211,220 @@ def fold(results, *,
             folding[name] = []
         folding[name].append(r)
 
+    # merge conflicts
     folded = []
-    for rs in folding.values():
+    for name, rs in folding.items():
         folded.append(sum(rs[1:], start=rs[0]))
 
     return folded
 
-
-def table(results, diff_results=None, *,
-        by_file=False,
-        size_sort=False,
-        reverse_size_sort=False,
+def table(Result, results, diff_results=None, *,
+        by=None,
+        fields=None,
+        sort=None,
         summary=False,
         all=False,
         percent=False,
         **_):
     all_, all = all, __builtins__.all
 
-    # fold
-    results = fold(results, by=['file' if by_file else 'function'])
-    if diff_results is not None:
-        diff_results = fold(diff_results,
-            by=['file' if by_file else 'function'])
+    if by is None:
+        by = Result._by
+    if fields is None:
+        fields = Result._fields
+    types = Result._types
 
+    # fold again
+    results = fold(Result, results, by=by)
+    if diff_results is not None:
+        diff_results = fold(Result, diff_results, by=by)
+
+    # organize by name
     table = {
-        r.file if by_file else r.function: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in results}
     diff_table = {
-        r.file if by_file else r.function: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in diff_results or []}
-
-    # sort, note that python's sort is stable
     names = list(table.keys() | diff_table.keys())
+
+    # sort again, now with diff info, note that python's sort is stable
     names.sort()
     if diff_results is not None:
-        names.sort(key=lambda n: -IntField.ratio(
-            table[n].data_size if n in table else None,
-            diff_table[n].data_size if n in diff_table else None))
-    if size_sort:
-        names.sort(key=lambda n: (table[n].data_size,) if n in table else (),
+        names.sort(key=lambda n: tuple(
+            types[k].ratio(
+                getattr(table.get(n), k, None),
+                getattr(diff_table.get(n), k, None))
+            for k in fields),
             reverse=True)
-    elif reverse_size_sort:
-        names.sort(key=lambda n: (table[n].data_size,) if n in table else (),
-            reverse=False)
+    if sort:
+        for k, reverse in reversed(sort):
+            names.sort(key=lambda n: (getattr(table[n], k),)
+                if getattr(table.get(n), k, None) is not None else (),
+                reverse=reverse ^ (not k or k in Result._fields))
 
-    # print header
-    if not summary:
-        title = '%s%s' % (
-            'file' if by_file else 'function',
+
+    # build up our lines
+    lines = []
+
+    # header
+    line = []
+    line.append('%s%s' % (
+        ','.join(by),
         ' (%d added, %d removed)' % (
             sum(1 for n in table if n not in diff_table),
             sum(1 for n in diff_table if n not in table))
             if diff_results is not None and not percent else '')
-        name_width = max(it.chain([23, len(title)], (len(n) for n in names)))
-    else:
-        title = ''
-        name_width = 23
-    name_width = 4*((name_width+1+4-1)//4)-1
-
-    print('%-*s ' % (name_width, title), end='')
+        if not summary else '')
     if diff_results is None:
-        print(' %s' % ('size'.rjust(len(IntField.none))))
+        for k in fields:
+            line.append(k)
     elif percent:
-        print(' %s' % ('size'.rjust(len(IntField.diff_none))))
+        for k in fields:
+            line.append(k)
     else:
-        print(' %s %s %s' % (
-            'old'.rjust(len(IntField.diff_none)),
-            'new'.rjust(len(IntField.diff_none)),
-            'diff'.rjust(len(IntField.diff_none))))
+        for k in fields:
+            line.append('o'+k)
+        for k in fields:
+            line.append('n'+k)
+        for k in fields:
+            line.append('d'+k)
+    line.append('')
+    lines.append(line)
 
-    # print entries
+    # entries
     if not summary:
         for name in names:
             r = table.get(name)
             if diff_results is not None:
                 diff_r = diff_table.get(name)
-                ratio = IntField.ratio(
-                    r.data_size if r else None,
-                    diff_r.data_size if diff_r else None)
-                if not ratio and not all_:
+                ratios = [
+                    types[k].ratio(
+                        getattr(r, k, None),
+                        getattr(diff_r, k, None))
+                    for k in fields]
+                if not any(ratios) and not all_:
                     continue
 
-            print('%-*s ' % (name_width, name), end='')
+            line = []
+            line.append(name)
             if diff_results is None:
-                print(' %s' % (
-                    r.data_size.table()
-                        if r else IntField.none))
+                for k in fields:
+                    line.append(getattr(r, k).table()
+                        if getattr(r, k, None) is not None
+                        else types[k].none)
             elif percent:
-                print(' %s%s' % (
-                    r.data_size.diff_table()
-                        if r else IntField.diff_none,
-                    ' (%s)' % (
-                        '+∞%' if ratio == +m.inf
-                        else '-∞%' if ratio == -m.inf
-                        else '%+.1f%%' % (100*ratio))))
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
             else:
-                print(' %s %s %s%s' % (
-                    diff_r.data_size.diff_table()
-                        if diff_r else IntField.diff_none,
-                    r.data_size.diff_table()
-                        if r else IntField.diff_none,
-                    IntField.diff_diff(
-                        r.data_size if r else None,
-                        diff_r.data_size if diff_r else None)
-                        if r or diff_r else IntField.diff_none,
-                    ' (%s)' % (
-                        '+∞%' if ratio == +m.inf
-                        else '-∞%' if ratio == -m.inf
-                        else '%+.1f%%' % (100*ratio))
-                        if ratio else ''))
+                for k in fields:
+                    line.append(getattr(diff_r, k).diff_table()
+                        if getattr(diff_r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(types[k].diff_diff(
+                            getattr(r, k, None),
+                            getattr(diff_r, k, None)))
+            if diff_results is None:
+                line.append('')
+            elif percent:
+                line.append(' (%s)' % ', '.join(
+                    '+∞%' if t == +m.inf
+                    else '-∞%' if t == -m.inf
+                    else '%+.1f%%' % (100*t)
+                    for t in ratios))
+            else:
+                line.append(' (%s)' % ', '.join(
+                        '+∞%' if t == +m.inf
+                        else '-∞%' if t == -m.inf
+                        else '%+.1f%%' % (100*t)
+                        for t in ratios
+                        if t)
+                    if any(ratios) else '')
+            lines.append(line)
 
-    # print total
-    total = fold(results, by=[])
-    r = total[0] if total else None
+    # total
+    r = next(iter(fold(Result, results, by=[])), None)
     if diff_results is not None:
-        diff_total = fold(diff_results, by=[])
-        diff_r = diff_total[0] if diff_total else None
-        ratio = IntField.ratio(
-            r.data_size if r else None,
-            diff_r.data_size if diff_r else None)
+        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
+        ratios = [
+            types[k].ratio(
+                getattr(r, k, None),
+                getattr(diff_r, k, None))
+            for k in fields]
 
-    print('%-*s ' % (name_width, 'TOTAL'), end='')
+    line = []
+    line.append('TOTAL')
     if diff_results is None:
-        print(' %s' % (
-            r.data_size.table()
-                if r else IntField.none))
+        for k in fields:
+            line.append(getattr(r, k).table()
+                if getattr(r, k, None) is not None
+                else types[k].none)
     elif percent:
-        print(' %s%s' % (
-            r.data_size.diff_table()
-                if r else IntField.diff_none,
-            ' (%s)' % (
-                '+∞%' if ratio == +m.inf
-                else '-∞%' if ratio == -m.inf
-                else '%+.1f%%' % (100*ratio))))
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
     else:
-        print(' %s %s %s%s' % (
-            diff_r.data_size.diff_table()
-                if diff_r else IntField.diff_none,
-            r.data_size.diff_table()
-                if r else IntField.diff_none,
-            IntField.diff_diff(
-                r.data_size if r else None,
-                diff_r.data_size if diff_r else None)
-                if r or diff_r else IntField.diff_none,
-            ' (%s)' % (
-                '+∞%' if ratio == +m.inf
-                else '-∞%' if ratio == -m.inf
-                else '%+.1f%%' % (100*ratio))
-                if ratio else ''))
+        for k in fields:
+            line.append(getattr(diff_r, k).diff_table()
+                if getattr(diff_r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(types[k].diff_diff(
+                    getattr(r, k, None),
+                    getattr(diff_r, k, None)))
+    if diff_results is None:
+        line.append('')
+    elif percent:
+        line.append(' (%s)' % ', '.join(
+            '+∞%' if t == +m.inf
+            else '-∞%' if t == -m.inf
+            else '%+.1f%%' % (100*t)
+            for t in ratios))
+    else:
+        line.append(' (%s)' % ', '.join(
+                '+∞%' if t == +m.inf
+                else '-∞%' if t == -m.inf
+                else '%+.1f%%' % (100*t)
+                for t in ratios
+                if t)
+            if any(ratios) else '')
+    lines.append(line)
+
+    # find the best widths, note that column 0 contains the names and column -1
+    # the ratios, so those are handled a bit differently
+    widths = [
+        ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
+        for w, i in zip(
+            it.chain([23], it.repeat(7)),
+            range(len(lines[0])-1))]
+
+    # print our table
+    for line in lines:
+        print('%-*s  %s%s' % (
+            widths[0], line[0],
+            ' '.join('%*s' % (w, x)
+                for w, x in zip(widths[1:], line[1:-1])),
+            line[-1]))
 
 
-def main(obj_paths, **args):
+def main(obj_paths, *,
+        by=None,
+        fields=None,
+        defines=None,
+        sort=None,
+        **args):
     # find sizes
     if not args.get('use', None):
         # find .o files
@@ -362,7 +437,7 @@ def main(obj_paths, **args):
                 paths.append(path)
 
         if not paths:
-            print('no .obj files found in %r?' % obj_paths)
+            print("error: no .obj files found in %r?" % obj_paths)
             sys.exit(-1)
 
         results = collect(paths, **args)
@@ -372,25 +447,35 @@ def main(obj_paths, **args):
             reader = csv.DictReader(f, restval='')
             for r in reader:
                 try:
-                    results.append(DataResult(**{
-                        k: v for k, v in r.items()
-                        if k in DataResult._fields}))
+                    results.append(DataResult(
+                        **{k: r[k] for k in DataResult._by
+                            if k in r and r[k].strip()},
+                        **{k: r['data_'+k] for k in DataResult._fields
+                            if 'data_'+k in r and r['data_'+k].strip()}))
                 except TypeError:
                     pass
 
-    # fold to remove duplicates
-    results = fold(results)
+    # fold
+    results = fold(DataResult, results, by=by, defines=defines)
 
-    # sort because why not
+    # sort, note that python's sort is stable
     results.sort()
+    if sort:
+        for k, reverse in reversed(sort):
+            results.sort(key=lambda r: (getattr(r, k),)
+                if getattr(r, k) is not None else (),
+                reverse=reverse ^ (not k or k in DataResult._fields))
 
     # write results to CSV
     if args.get('output'):
         with openio(args['output'], 'w') as f:
-            writer = csv.DictWriter(f, DataResult._fields)
+            writer = csv.DictWriter(f, DataResult._by
+                + ['data_'+k for k in DataResult._fields])
             writer.writeheader()
             for r in results:
-                writer.writerow(r._asdict())
+                writer.writerow(
+                    {k: getattr(r, k) for k in DataResult._by}
+                    | {'data_'+k: getattr(r, k) for k in DataResult._fields})
 
     # find previous results?
     if args.get('diff'):
@@ -400,22 +485,26 @@ def main(obj_paths, **args):
                 reader = csv.DictReader(f, restval='')
                 for r in reader:
                     try:
-                        diff_results.append(DataResult(**{
-                            k: v for k, v in r.items()
-                            if k in DataResult._fields}))
+                        diff_results.append(DataResult(
+                            **{k: r[k] for k in DataResult._by
+                                if k in r and r[k].strip()},
+                            **{k: r['data_'+k] for k in DataResult._fields
+                                if 'data_'+k in r and r['data_'+k].strip()}))
                     except TypeError:
                         pass
         except FileNotFoundError:
             pass
 
-        # fold to remove duplicates
-        diff_results = fold(diff_results)
+        # fold
+        diff_results = fold(DataResult, diff_results, by=by, defines=defines)
 
     # print table
     if not args.get('quiet'):
-        table(
-            results,
+        table(DataResult, results,
             diff_results if args.get('diff') else None,
+            by=by if by is not None else ['function'],
+            fields=fields,
+            sort=sort,
             **args)
 
 
@@ -456,22 +545,39 @@ if __name__ == "__main__":
         action='store_true',
         help="Only show percentage change, not a full diff.")
     parser.add_argument(
-        '-b', '--by-file',
-        action='store_true',
-        help="Group by file. Note this does not include padding "
-            "so sizes may differ from other tools.")
+        '-b', '--by',
+        action='append',
+        choices=DataResult._by,
+        help="Group by this field.")
     parser.add_argument(
-        '-s', '--size-sort',
-        action='store_true',
-        help="Sort by size.")
+        '-f', '--field',
+        dest='fields',
+        action='append',
+        choices=DataResult._fields,
+        help="Show this field.")
     parser.add_argument(
-        '-S', '--reverse-size-sort',
-        action='store_true',
-        help="Sort by size, but backwards.")
+        '-D', '--define',
+        dest='defines',
+        action='append',
+        type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
+        help="Only include results where this field is this value.")
+    class AppendSort(argparse.Action):
+        def __call__(self, parser, namespace, value, option):
+            if namespace.sort is None:
+                namespace.sort = []
+            namespace.sort.append((value, True if option == '-S' else False))
+    parser.add_argument(
+        '-s', '--sort',
+        action=AppendSort,
+        help="Sort by this fields.")
+    parser.add_argument(
+        '-S', '--reverse-sort',
+        action=AppendSort,
+        help="Sort by this fields, but backwards.")
     parser.add_argument(
         '-Y', '--summary',
         action='store_true',
-        help="Only show the total size.")
+        help="Only show the total.")
     parser.add_argument(
         '-A', '--everything',
         action='store_true',

scripts/plot.py

@@ -438,9 +438,7 @@ def datasets(results, by=None, x=None, y=None, define=[]):
         y = co.OrderedDict()
         for r in results:
             for k, v in r.items():
-                if by is not None and k in by:
-                    continue
-                if y.get(k, True):
+                if (by is None or k not in by) and v.strip():
                     try:
                         dat(v)
                         y[k] = True
@@ -462,7 +460,7 @@ def datasets(results, by=None, x=None, y=None, define=[]):
             for y_ in y:
                 # hide x/y if there is only one field
                 k_x = x_ if len(x or []) > 1 else ''
-                k_y = y_ if len(y or []) > 1 else ''
+                k_y = y_ if len(y or []) > 1 or (not ks_ and not k_x) else ''
 
                 datasets[ks_ + (k_x, k_y)] = dataset(
                     results,
@@ -509,15 +507,15 @@ def main(csv_paths, *,
         ylim = (0, ylim[0])
 
     # separate out renames
-    renames = [k.split('=', 1)
-        for k in it.chain(by or [], x or [], y or [])
-        if '=' in k]
+    renames = list(it.chain.from_iterable(
+        ((k, v) for v in vs)
+        for k, vs in it.chain(by or [], x or [], y or [])))
     if by is not None:
-        by = [k.split('=', 1)[0] for k in by]
+        by = [k for k, _ in by]
     if x is not None:
-        x = [k.split('=', 1)[0] for k in x]
+        x = [k for k, _ in x]
     if y is not None:
-        y = [k.split('=', 1)[0] for k in y]
+        y = [k for k, _ in y]
 
     def draw(f):
         def writeln(s=''):
@@ -739,23 +737,31 @@ if __name__ == "__main__":
     parser.add_argument(
         '-b', '--by',
         action='append',
-        help="Fields to render as separate plots. All other fields will be "
-            "summed as needed. Can rename fields with new_name=old_name.")
+        type=lambda x: (
+            lambda k,v=None: (k, v.split(',') if v is not None else ())
+            )(*x.split('=', 1)),
+        help="Group by this field. Can rename fields with new_name=old_name.")
     parser.add_argument(
         '-x',
         action='append',
-        help="Fields to use for the x-axis. Can rename fields with "
+        type=lambda x: (
+            lambda k,v=None: (k, v.split(',') if v is not None else ())
+            )(*x.split('=', 1)),
+        help="Field to use for the x-axis. Can rename fields with "
             "new_name=old_name.")
     parser.add_argument(
         '-y',
         action='append',
-        help="Fields to use for the y-axis. Can rename fields with "
+        type=lambda x: (
+            lambda k,v=None: (k, v.split(',') if v is not None else ())
+            )(*x.split('=', 1)),
+        help="Field to use for the y-axis. Can rename fields with "
             "new_name=old_name.")
     parser.add_argument(
         '-D', '--define',
         type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
         action='append',
-        help="Only include rows where this field is this value. May include "
+        help="Only include results where this field is this value. May include "
             "comma-separated options.")
     parser.add_argument(
         '--color',

scripts/stack.py

@@ -23,10 +23,10 @@ CI_PATHS = ['*.ci']
 
 
 # integer fields
-class IntField(co.namedtuple('IntField', 'x')):
+class Int(co.namedtuple('Int', 'x')):
     __slots__ = ()
     def __new__(cls, x=0):
-        if isinstance(x, IntField):
+        if isinstance(x, Int):
             return x
         if isinstance(x, str):
             try:
@@ -92,38 +92,33 @@ class IntField(co.namedtuple('IntField', 'x')):
             return (new-old) / old
 
     def __add__(self, other):
-        return IntField(self.x + other.x)
+        return self.__class__(self.x + other.x)
 
     def __sub__(self, other):
-        return IntField(self.x - other.x)
+        return self.__class__(self.x - other.x)
 
     def __mul__(self, other):
-        return IntField(self.x * other.x)
-
-    def __lt__(self, other):
-        return self.x < other.x
-
-    def __gt__(self, other):
-        return self.__class__.__lt__(other, self)
-
-    def __le__(self, other):
-        return not self.__gt__(other)
-
-    def __ge__(self, other):
-        return not self.__lt__(other)
+        return self.__class__(self.x * other.x)
 
 # size results
-class StackResult(co.namedtuple('StackResult',
-        'file,function,stack_frame,stack_limit')):
+class StackResult(co.namedtuple('StackResult', [
+        'file', 'function', 'frame', 'limit', 'calls'])):
+    _by = ['file', 'function']
+    _fields = ['frame', 'limit']
+    _types = {'frame': Int, 'limit': Int}
+
     __slots__ = ()
-    def __new__(cls, file, function, stack_frame, stack_limit):
+    def __new__(cls, file='', function='',
+            frame=0, limit=0, calls=set()):
         return super().__new__(cls, file, function,
-            IntField(stack_frame), IntField(stack_limit))
+            Int(frame), Int(limit),
+            calls)
 
     def __add__(self, other):
         return StackResult(self.file, self.function,
-            self.stack_frame + other.stack_frame,
-            max(self.stack_limit, other.stack_limit))
+            self.frame + other.frame,
+            max(self.limit, other.limit),
+            self.calls | other.calls)
 
 
 def openio(path, mode='r'):
@@ -135,7 +130,6 @@ def openio(path, mode='r'):
     else:
         return open(path, mode)
 
-
 def collect(paths, *,
         everything=False,
         **args):
@@ -147,10 +141,10 @@ def collect(paths, *,
             node = []
             while True:
                 rest = rest.lstrip()
-                m = k_pattern.match(rest)
-                if not m:
+                m_ = k_pattern.match(rest)
+                if not m_:
                     return (node, rest)
-                k, rest = m.group(1), rest[m.end(0):]
+                k, rest = m_.group(1), rest[m_.end(0):]
 
                 rest = rest.lstrip()
                 if rest.startswith('{'):
@@ -159,9 +153,9 @@ def collect(paths, *,
                     rest = rest[1:]
                     node.append((k, v))
                 else:
-                    m = v_pattern.match(rest)
-                    assert m, "unexpected %r" % rest[0:1]
-                    v, rest = m.group(1) or m.group(2), rest[m.end(0):]
+                    m_ = v_pattern.match(rest)
+                    assert m_, "unexpected %r" % rest[0:1]
+                    v, rest = m_.group(1) or m_.group(2), rest[m_.end(0):]
                     node.append((k, v))
 
         node, rest = parse_vcg(rest)
@@ -181,13 +175,13 @@ def collect(paths, *,
             for k, info in graph:
                 if k == 'node':
                     info = dict(info)
-                    m = f_pattern.match(info['label'])
-                    if m:
-                        function, file, size, type = m.groups()
+                    m_ = f_pattern.match(info['label'])
+                    if m_:
+                        function, file, size, type = m_.groups()
                         if (not args.get('quiet')
                                 and 'static' not in type
                                 and 'bounded' not in type):
-                            print('warning: found non-static stack for %s (%s)'
+                            print("warning: found non-static stack for %s (%s)"
                                 % (function, type, size))
                         _, _, _, targets = callgraph[info['title']]
                         callgraph[info['title']] = (
@@ -217,7 +211,7 @@ def collect(paths, *,
         for target in targets:
             if target in seen:
                 # found a cycle
-                return float('inf')
+                return m.inf
             limit_ = find_limit(target, seen | {target})
             limit = max(limit, limit_)
 
@@ -233,19 +227,35 @@ def collect(paths, *,
 
     # build results
     results = []
-    calls = {}
     for source, (s_file, s_function, frame, targets) in callgraph.items():
         limit = find_limit(source)
-        cs = find_calls(targets)
-        results.append(StackResult(s_file, s_function, frame, limit))
-        calls[(s_file, s_function)] = cs
+        calls = find_calls(targets)
+        results.append(StackResult(s_file, s_function, frame, limit, calls))
 
-    return results, calls
+    return results
 
 
-def fold(results, *,
-        by=['file', 'function'],
+def fold(Result, results, *,
+        by=None,
+        defines=None,
         **_):
+    if by is None:
+        by = Result._by
+
+    for k in it.chain(by or [], (k for k, _ in defines or [])):
+        if k not in Result._by and k not in Result._fields:
+            print("error: could not find field %r?" % k)
+            sys.exit(-1)
+
+    # filter by matching defines
+    if defines is not None:
+        results_ = []
+        for r in results:
+            if all(getattr(r, k) in vs for k, vs in defines):
+                results_.append(r)
+        results = results_
+
+    # organize results into conflicts
     folding = co.OrderedDict()
     for r in results:
         name = tuple(getattr(r, k) for k in by)
@@ -253,36 +263,17 @@ def fold(results, *,
             folding[name] = []
         folding[name].append(r)
 
+    # merge conflicts
     folded = []
-    for rs in folding.values():
+    for name, rs in folding.items():
         folded.append(sum(rs[1:], start=rs[0]))
 
     return folded
 
-def fold_calls(calls, *,
-        by=['file', 'function'],
-        **_):
-    def by_(name):
-        file, function = name
-        return (((file,) if 'file' in by else ())
-            + ((function,) if 'function' in by else ()))
-
-    folded = {}
-    for name, cs in calls.items():
-        name = by_(name)
-        if name not in folded:
-            folded[name] = set()
-        folded[name] |= {by_(c) for c in cs}
-
-    return folded
-
-
-def table(results, calls, diff_results=None, *,
-        by_file=False,
-        limit_sort=False,
-        reverse_limit_sort=False,
-        frame_sort=False,
-        reverse_frame_sort=False,
+def table(Result, results, diff_results=None, *,
+        by=None,
+        fields=None,
+        sort=None,
         summary=False,
         all=False,
         percent=False,
@@ -291,209 +282,268 @@ def table(results, calls, diff_results=None, *,
         **_):
     all_, all = all, __builtins__.all
 
-    # tree doesn't really make sense with depth=0, assume depth=inf
-    if depth is None:
-        depth = float('inf') if tree else 0
+    if by is None:
+        by = Result._by
+    if fields is None:
+        fields = Result._fields
+    types = Result._types
 
-    # fold
-    results = fold(results, by=['file' if by_file else 'function'])
-    calls = fold_calls(calls, by=['file' if by_file else 'function'])
+    # fold again
+    results = fold(Result, results, by=by)
     if diff_results is not None:
-        diff_results = fold(diff_results,
-            by=['file' if by_file else 'function'])
+        diff_results = fold(Result, diff_results, by=by)
 
+    # organize by name
     table = {
-        r.file if by_file else r.function: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in results}
     diff_table = {
-        r.file if by_file else r.function: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in diff_results or []}
-
-    # sort, note that python's sort is stable
     names = list(table.keys() | diff_table.keys())
+
+    # sort again, now with diff info, note that python's sort is stable
     names.sort()
     if diff_results is not None:
-        names.sort(key=lambda n: -IntField.ratio(
-            table[n].stack_frame if n in table else None,
-            diff_table[n].stack_frame if n in diff_table else None))
-    if limit_sort:
-        names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (),
+        names.sort(key=lambda n: tuple(
+            types[k].ratio(
+                getattr(table.get(n), k, None),
+                getattr(diff_table.get(n), k, None))
+            for k in fields),
             reverse=True)
-    elif reverse_limit_sort:
-        names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (),
-            reverse=False)
-    elif frame_sort:
-        names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (),
-            reverse=True)
-    elif reverse_frame_sort:
-        names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (),
-            reverse=False)
+    if sort:
+        for k, reverse in reversed(sort):
+            names.sort(key=lambda n: (getattr(table[n], k),)
+                if getattr(table.get(n), k, None) is not None else (),
+                reverse=reverse ^ (not k or k in Result._fields))
 
-    # print header
-    if not summary:
-        title = '%s%s' % (
-            'file' if by_file else 'function',
+
+    # build up our lines
+    lines = []
+
+    # header
+    line = []
+    line.append('%s%s' % (
+        ','.join(by),
         ' (%d added, %d removed)' % (
             sum(1 for n in table if n not in diff_table),
             sum(1 for n in diff_table if n not in table))
             if diff_results is not None and not percent else '')
-        name_width = max(it.chain([23, len(title)], (len(n) for n in names)))
-    else:
-        title = ''
-        name_width = 23
-    name_width = 4*((name_width+1+4-1)//4)-1
-
-    # adjust the name width based on the expected call depth, note that we
-    # can't always find the depth due to recursion
-    if not m.isinf(depth):
-        name_width += 4*depth
-
-    if not tree:
-        print('%-*s ' % (name_width, title), end='')
+        if not summary else '')
     if diff_results is None:
-            print(' %s %s' % (
-                'frame'.rjust(len(IntField.none)),
-                'limit'.rjust(len(IntField.none))))
+        for k in fields:
+            line.append(k)
     elif percent:
-            print(' %s %s' % (
-                'frame'.rjust(len(IntField.diff_none)),
-                'limit'.rjust(len(IntField.diff_none))))
+        for k in fields:
+            line.append(k)
     else:
-            print(' %s %s %s %s %s %s' % (
-                'oframe'.rjust(len(IntField.diff_none)),
-                'olimit'.rjust(len(IntField.diff_none)),
-                'nframe'.rjust(len(IntField.diff_none)),
-                'nlimit'.rjust(len(IntField.diff_none)),
-                'dframe'.rjust(len(IntField.diff_none)),
-                'dlimit'.rjust(len(IntField.diff_none))))
+        for k in fields:
+            line.append('o'+k)
+        for k in fields:
+            line.append('n'+k)
+        for k in fields:
+            line.append('d'+k)
+    line.append('')
+    lines.append(line)
 
-    # print entries
+    # entries
     if not summary:
-        # print the tree recursively
-        def table_calls(names_, depth,
-                prefixes=('', '', '', '')):
-            for i, name in enumerate(names_):
+        for name in names:
             r = table.get(name)
             if diff_results is not None:
                 diff_r = diff_table.get(name)
-                    ratio = IntField.ratio(
-                        r.stack_limit if r else None,
-                        diff_r.stack_limit if diff_r else None)
-                    if not ratio and not all_:
+                ratios = [
+                    types[k].ratio(
+                        getattr(r, k, None),
+                        getattr(diff_r, k, None))
+                    for k in fields]
+                if not any(ratios) and not all_:
                     continue
 
-                is_last = (i == len(names_)-1)
-                print('%-*s ' % (name_width, prefixes[0+is_last]+name), end='')
-                if tree:
-                    print()
-                elif diff_results is None:
-                    print(' %s %s' % (
-                        r.stack_frame.table()
-                            if r else IntField.none,
-                        r.stack_limit.table()
-                            if r else IntField.none))
+            line = []
+            line.append(name)
+            if diff_results is None:
+                for k in fields:
+                    line.append(getattr(r, k).table()
+                        if getattr(r, k, None) is not None
+                        else types[k].none)
             elif percent:
-                    print(' %s %s%s' % (
-                        r.stack_frame.diff_table()
-                            if r else IntField.diff_none,
-                        r.stack_limit.diff_table()
-                            if r else IntField.diff_none,
-                        ' (%s)' % (
-                            '+∞%' if ratio == +m.inf
-                            else '-∞%' if ratio == -m.inf
-                            else '%+.1f%%' % (100*ratio))))
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
             else:
-                    print(' %s %s %s %s %s %s%s' % (
-                        diff_r.stack_frame.diff_table()
-                            if diff_r else IntField.diff_none,
-                        diff_r.stack_limit.diff_table()
-                            if diff_r else IntField.diff_none,
-                        r.stack_frame.diff_table()
-                            if r else IntField.diff_none,
-                        r.stack_limit.diff_table()
-                            if r else IntField.diff_none,
-                        IntField.diff_diff(
-                            r.stack_frame if r else None,
-                            diff_r.stack_frame if diff_r else None)
-                            if r or diff_r else IntField.diff_none,
-                        IntField.diff_diff(
-                            r.stack_limit if r else None,
-                            diff_r.stack_limit if diff_r else None)
-                            if r or diff_r else IntField.diff_none,
-                        ' (%s)' % (
-                            '+∞%' if ratio == +m.inf
-                            else '-∞%' if ratio == -m.inf
-                            else '%+.1f%%' % (100*ratio))
-                            if ratio else ''))
+                for k in fields:
+                    line.append(getattr(diff_r, k).diff_table()
+                        if getattr(diff_r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(types[k].diff_diff(
+                            getattr(r, k, None),
+                            getattr(diff_r, k, None)))
+            if diff_results is None:
+                line.append('')
+            elif percent:
+                line.append(' (%s)' % ', '.join(
+                    '+∞%' if t == +m.inf
+                    else '-∞%' if t == -m.inf
+                    else '%+.1f%%' % (100*t)
+                    for t in ratios))
+            else:
+                line.append(' (%s)' % ', '.join(
+                        '+∞%' if t == +m.inf
+                        else '-∞%' if t == -m.inf
+                        else '%+.1f%%' % (100*t)
+                        for t in ratios
+                        if t)
+                    if any(ratios) else '')
+            lines.append(line)
+
+    # total
+    r = next(iter(fold(Result, results, by=[])), None)
+    if diff_results is not None:
+        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
+        ratios = [
+            types[k].ratio(
+                getattr(r, k, None),
+                getattr(diff_r, k, None))
+            for k in fields]
+
+    line = []
+    line.append('TOTAL')
+    if diff_results is None:
+        for k in fields:
+            line.append(getattr(r, k).table()
+                if getattr(r, k, None) is not None
+                else types[k].none)
+    elif percent:
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
+    else:
+        for k in fields:
+            line.append(getattr(diff_r, k).diff_table()
+                if getattr(diff_r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(types[k].diff_diff(
+                    getattr(r, k, None),
+                    getattr(diff_r, k, None)))
+    if diff_results is None:
+        line.append('')
+    elif percent:
+        line.append(' (%s)' % ', '.join(
+            '+∞%' if t == +m.inf
+            else '-∞%' if t == -m.inf
+            else '%+.1f%%' % (100*t)
+            for t in ratios))
+    else:
+        line.append(' (%s)' % ', '.join(
+                '+∞%' if t == +m.inf
+                else '-∞%' if t == -m.inf
+                else '%+.1f%%' % (100*t)
+                for t in ratios
+                if t)
+            if any(ratios) else '')
+    lines.append(line)
+
+    # find the best widths, note that column 0 contains the names and column -1
+    # the ratios, so those are handled a bit differently
+    widths = [
+        ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
+        for w, i in zip(
+            it.chain([23], it.repeat(7)),
+            range(len(lines[0])-1))]
+
+    # adjust the name width based on the expected call depth, though
+    # note this doesn't really work with unbounded recursion
+    if not summary:
+        # it doesn't really make sense to not have a depth with tree,
+        # so assume depth=inf if tree by default
+        if depth is None:
+            depth = m.inf if tree else 0
+        elif depth == 0:
+            depth = m.inf
+
+        if not m.isinf(depth):
+            widths[0] += 4*depth
+
+    # print our table with optional call info
+    #
+    # note we try to adjust our name width based on expected call depth, but
+    # this doesn't work if there's unbounded recursion
+    if not tree:
+        print('%-*s  %s%s' % (
+            widths[0], lines[0][0],
+            ' '.join('%*s' % (w, x)
+                for w, x, in zip(widths[1:], lines[0][1:-1])),
+            lines[0][-1]))
+
+    # print the tree recursively
+    if not summary:
+        line_table = {n: l for n, l in zip(names, lines[1:-1])}
+
+        def recurse(names_, depth_, prefixes=('', '', '', '')):
+            for i, name in enumerate(names_):
+                if name not in line_table:
+                    continue
+                line = line_table[name]
+                is_last = (i == len(names_)-1)
+
+                print('%s%-*s ' % (
+                    prefixes[0+is_last],
+                    widths[0] - (
+                        len(prefixes[0+is_last])
+                        if not m.isinf(depth) else 0),
+                    line[0]),
+                    end='')
+                if not tree:
+                    print(' %s%s' % (
+                        ' '.join('%*s' % (w, x)
+                            for w, x, in zip(widths[1:], line[1:-1])),
+                        line[-1]),
+                        end='')
+                print() 
 
                 # recurse?
-                if depth > 0:
-                    cs = calls.get((name,), set())
-                    table_calls(
-                        [n for n in names if (n,) in cs],
-                        depth-1,
-                        (   prefixes[2+is_last] + "|-> ",
+                if name in table and depth_ > 0:
+                    calls = {
+                        ','.join(str(getattr(Result(*c), k) or '') for k in by)
+                        for c in table[name].calls}
+
+                    recurse(
+                        # note we're maintaining sort order
+                        [n for n in names if n in calls],
+                        depth_-1,
+                        (prefixes[2+is_last] + "|-> ",
                          prefixes[2+is_last] + "'-> ",
                          prefixes[2+is_last] + "|   ",
                          prefixes[2+is_last] + "    "))
+        recurse(names, depth)
 
-
-        table_calls(names, depth)
-
-    # print total
     if not tree:
-        total = fold(results, by=[])
-        r = total[0] if total else None
-        if diff_results is not None:
-            diff_total = fold(diff_results, by=[])
-            diff_r = diff_total[0] if diff_total else None
-            ratio = IntField.ratio(
-                r.stack_limit if r else None,
-                diff_r.stack_limit if diff_r else None)
-
-        print('%-*s ' % (name_width, 'TOTAL'), end='')
-        if diff_results is None:
-            print(' %s %s' % (
-                r.stack_frame.table()
-                    if r else IntField.none,
-                r.stack_limit.table()
-                    if r else IntField.none))
-        elif percent:
-            print(' %s %s%s' % (
-                r.stack_frame.diff_table()
-                    if r else IntField.diff_none,
-                r.stack_limit.diff_table()
-                    if r else IntField.diff_none,
-                ' (%s)' % (
-                    '+∞%' if ratio == +m.inf
-                    else '-∞%' if ratio == -m.inf
-                    else '%+.1f%%' % (100*ratio))))
-        else:
-            print(' %s %s %s %s %s %s%s' % (
-                diff_r.stack_frame.diff_table()
-                    if diff_r else IntField.diff_none,
-                diff_r.stack_limit.diff_table()
-                    if diff_r else IntField.diff_none,
-                r.stack_frame.diff_table()
-                    if r else IntField.diff_none,
-                r.stack_limit.diff_table()
-                    if r else IntField.diff_none,
-                IntField.diff_diff(
-                    r.stack_frame if r else None,
-                    diff_r.stack_frame if diff_r else None)
-                    if r or diff_r else IntField.diff_none,
-                IntField.diff_diff(
-                    r.stack_limit if r else None,
-                    diff_r.stack_limit if diff_r else None)
-                    if r or diff_r else IntField.diff_none,
-                ' (%s)' % (
-                    '+∞%' if ratio == +m.inf
-                    else '-∞%' if ratio == -m.inf
-                    else '%+.1f%%' % (100*ratio))
-                    if ratio else ''))
+        print('%-*s  %s%s' % (
+            widths[0], lines[-1][0],
+            ' '.join('%*s' % (w, x)
+                for w, x, in zip(widths[1:], lines[-1][1:-1])),
+            lines[-1][-1]))
 
 
-def main(ci_paths, **args):
+def main(ci_paths,
+        by=None,
+        fields=None,
+        defines=None,
+        sort=None,
+        **args):
     # find sizes
     if not args.get('use', None):
         # find .ci files
@@ -506,37 +556,45 @@ def main(ci_paths, **args):
                 paths.append(path)
 
         if not paths:
-            print('no .ci files found in %r?' % ci_paths)
+            print("error: no .ci files found in %r?" % ci_paths)
             sys.exit(-1)
 
-        results, calls = collect(paths, **args)
+        results = collect(paths, **args)
     else:
         results = []
         with openio(args['use']) as f:
             reader = csv.DictReader(f, restval='')
             for r in reader:
                 try:
-                    results.append(StackResult(**{
-                        k: v for k, v in r.items()
-                        if k in StackResult._fields}))
+                    results.append(StackResult(
+                        **{k: r[k] for k in StackResult._by
+                            if k in r and r[k].strip()},
+                        **{k: r['stack_'+k] for k in StackResult._fields
+                            if 'stack_'+k in r and r['stack_'+k].strip()}))
                 except TypeError:
                     pass
 
-        calls = {}
+    # fold
+    results = fold(StackResult, results, by=by, defines=defines)
 
-    # fold to remove duplicates
-    results = fold(results)
-
-    # sort because why not
+    # sort, note that python's sort is stable
     results.sort()
+    if sort:
+        for k, reverse in reversed(sort):
+            results.sort(key=lambda r: (getattr(r, k),)
+                if getattr(r, k) is not None else (),
+                reverse=reverse ^ (not k or k in StackResult._fields))
 
     # write results to CSV
     if args.get('output'):
         with openio(args['output'], 'w') as f:
-            writer = csv.DictWriter(f, StackResult._fields)
+            writer = csv.DictWriter(f, StackResult._by
+                + ['stack_'+k for k in StackResult._fields])
             writer.writeheader()
             for r in results:
-                writer.writerow(r._asdict())
+                writer.writerow(
+                    {k: getattr(r, k) for k in StackResult._by}
+                    | {'stack_'+k: getattr(r, k) for k in StackResult._fields})
 
     # find previous results?
     if args.get('diff'):
@@ -546,28 +604,31 @@ def main(ci_paths, **args):
                 reader = csv.DictReader(f, restval='')
                 for r in reader:
                     try:
-                        diff_results.append(StackResult(**{
-                            k: v for k, v in r.items()
-                            if k in StackResult._fields}))
+                        diff_results.append(StackResult(
+                            **{k: r[k] for k in StackResult._by
+                                if k in r and r[k].strip()},
+                            **{k: r['stack_'+k] for k in StackResult._fields
+                                if 'stack_'+k in r and r['stack_'+k].strip()}))
                     except TypeError:
-                        pass
+                        raise
         except FileNotFoundError:
             pass
 
-        # fold to remove duplicates
-        diff_results = fold(diff_results)
+        # fold
+        diff_results = fold(StackResult, diff_results, by=by, defines=defines)
 
     # print table
     if not args.get('quiet'):
-        table(
-            results,
-            calls,
+        table(StackResult, results,
             diff_results if args.get('diff') else None,
+            by=by if by is not None else ['function'],
+            fields=fields,
+            sort=sort,
             **args)
 
     # error on recursion
     if args.get('error_on_recursion') and any(
-            m.isinf(float(r.stack_limit)) for r in results):
+            m.isinf(float(r.limit)) for r in results):
         sys.exit(2)
 
 
@@ -608,47 +669,58 @@ if __name__ == "__main__":
         action='store_true',
         help="Only show percentage change, not a full diff.")
     parser.add_argument(
-        '-t', '--tree',
-        action='store_true',
-        help="Only show the function call tree.")
+        '-b', '--by',
+        action='append',
+        choices=StackResult._by,
+        help="Group by this field.")
     parser.add_argument(
-        '-b', '--by-file',
-        action='store_true',
-        help="Group by file.")
+        '-f', '--field',
+        dest='fields',
+        action='append',
+        choices=StackResult._fields,
+        help="Show this field.")
     parser.add_argument(
-        '-s', '--limit-sort',
-        action='store_true',
-        help="Sort by stack limit.")
+        '-D', '--define',
+        dest='defines',
+        action='append',
+        type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
+        help="Only include results where this field is this value.")
+    class AppendSort(argparse.Action):
+        def __call__(self, parser, namespace, value, option):
+            if namespace.sort is None:
+                namespace.sort = []
+            namespace.sort.append((value, True if option == '-S' else False))
     parser.add_argument(
-        '-S', '--reverse-limit-sort',
-        action='store_true',
-        help="Sort by stack limit, but backwards.")
+        '-s', '--sort',
+        action=AppendSort,
+        help="Sort by this fields.")
     parser.add_argument(
-        '--frame-sort',
-        action='store_true',
-        help="Sort by stack frame.")
-    parser.add_argument(
-        '--reverse-frame-sort',
-        action='store_true',
-        help="Sort by stack frame, but backwards.")
+        '-S', '--reverse-sort',
+        action=AppendSort,
+        help="Sort by this fields, but backwards.")
     parser.add_argument(
         '-Y', '--summary',
         action='store_true',
-        help="Only show the total size.")
-    parser.add_argument(
-        '-L', '--depth',
-        nargs='?',
-        type=lambda x: int(x, 0),
-        const=float('inf'),
-        help="Depth of function calls to show.")
-    parser.add_argument(
-        '-e', '--error-on-recursion',
-        action='store_true',
-        help="Error if any functions are recursive.")
+        help="Only show the total.")
     parser.add_argument(
         '-A', '--everything',
         action='store_true',
         help="Include builtin and libc specific symbols.")
+    parser.add_argument(
+        '--tree',
+        action='store_true',
+        help="Only show the function call tree.")
+    parser.add_argument(
+        '-L', '--depth',
+        nargs='?',
+        type=lambda x: int(x, 0),
+        const=0,
+        help="Depth of function calls to show. 0 show all calls but may not "
+            "terminate!")
+    parser.add_argument(
+        '-e', '--error-on-recursion',
+        action='store_true',
+        help="Error if any functions are recursive.")
     parser.add_argument(
         '--build-dir',
         help="Specify the relative build directory. Used to map object files "

scripts/struct_.py

@@ -24,11 +24,12 @@ OBJ_PATHS = ['*.o']
 OBJDUMP_TOOL = ['objdump']
 
 
+
 # integer fields
-class IntField(co.namedtuple('IntField', 'x')):
+class Int(co.namedtuple('Int', 'x')):
     __slots__ = ()
     def __new__(cls, x=0):
-        if isinstance(x, IntField):
+        if isinstance(x, Int):
             return x
         if isinstance(x, str):
             try:
@@ -94,35 +95,28 @@ class IntField(co.namedtuple('IntField', 'x')):
             return (new-old) / old
 
     def __add__(self, other):
-        return IntField(self.x + other.x)
+        return self.__class__(self.x + other.x)
 
     def __sub__(self, other):
-        return IntField(self.x - other.x)
+        return self.__class__(self.x - other.x)
 
     def __mul__(self, other):
-        return IntField(self.x * other.x)
-
-    def __lt__(self, other):
-        return self.x < other.x
-
-    def __gt__(self, other):
-        return self.__class__.__lt__(other, self)
-
-    def __le__(self, other):
-        return not self.__gt__(other)
-
-    def __ge__(self, other):
-        return not self.__lt__(other)
+        return self.__class__(self.x * other.x)
 
 # struct size results
-class StructResult(co.namedtuple('StructResult', 'file,struct,struct_size')):
+class StructResult(co.namedtuple('StructResult', ['file', 'struct', 'size'])):
+    _by = ['file', 'struct']
+    _fields = ['size']
+    _types = {'size': Int}
+
     __slots__ = ()
-    def __new__(cls, file, struct, struct_size):
-        return super().__new__(cls, file, struct, IntField(struct_size))
+    def __new__(cls, file='', struct='', size=0):
+        return super().__new__(cls, file, struct,
+            Int(size))
 
     def __add__(self, other):
         return StructResult(self.file, self.struct,
-            self.struct_size + other.struct_size)
+            self.size + other.size)
 
 
 def openio(path, mode='r'):
@@ -231,9 +225,27 @@ def collect(paths, *,
     return results
 
 
-def fold(results, *,
-        by=['file', 'struct'],
+def fold(Result, results, *,
+        by=None,
+        defines=None,
         **_):
+    if by is None:
+        by = Result._by
+
+    for k in it.chain(by or [], (k for k, _ in defines or [])):
+        if k not in Result._by and k not in Result._fields:
+            print("error: could not find field %r?" % k)
+            sys.exit(-1)
+
+    # filter by matching defines
+    if defines is not None:
+        results_ = []
+        for r in results:
+            if all(getattr(r, k) in vs for k, vs in defines):
+                results_.append(r)
+        results = results_
+
+    # organize results into conflicts
     folding = co.OrderedDict()
     for r in results:
         name = tuple(getattr(r, k) for k in by)
@@ -241,157 +253,220 @@ def fold(results, *,
             folding[name] = []
         folding[name].append(r)
 
+    # merge conflicts
     folded = []
-    for rs in folding.values():
+    for name, rs in folding.items():
         folded.append(sum(rs[1:], start=rs[0]))
 
     return folded
 
-
-def table(results, diff_results=None, *,
-        by_file=False,
-        size_sort=False,
-        reverse_size_sort=False,
+def table(Result, results, diff_results=None, *,
+        by=None,
+        fields=None,
+        sort=None,
         summary=False,
         all=False,
         percent=False,
         **_):
     all_, all = all, __builtins__.all
 
-    # fold
-    results = fold(results, by=['file' if by_file else 'struct'])
-    if diff_results is not None:
-        diff_results = fold(diff_results,
-            by=['file' if by_file else 'struct'])
+    if by is None:
+        by = Result._by
+    if fields is None:
+        fields = Result._fields
+    types = Result._types
 
+    # fold again
+    results = fold(Result, results, by=by)
+    if diff_results is not None:
+        diff_results = fold(Result, diff_results, by=by)
+
+    # organize by name
     table = {
-        r.file if by_file else r.struct: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in results}
     diff_table = {
-        r.file if by_file else r.struct: r
+        ','.join(str(getattr(r, k) or '') for k in by): r
         for r in diff_results or []}
-
-    # sort, note that python's sort is stable
     names = list(table.keys() | diff_table.keys())
+
+    # sort again, now with diff info, note that python's sort is stable
     names.sort()
     if diff_results is not None:
-        names.sort(key=lambda n: -IntField.ratio(
-            table[n].struct_size if n in table else None,
-            diff_table[n].struct_size if n in diff_table else None))
-    if size_sort:
-        names.sort(key=lambda n: (table[n].struct_size,) if n in table else (),
+        names.sort(key=lambda n: tuple(
+            types[k].ratio(
+                getattr(table.get(n), k, None),
+                getattr(diff_table.get(n), k, None))
+            for k in fields),
             reverse=True)
-    elif reverse_size_sort:
-        names.sort(key=lambda n: (table[n].struct_size,) if n in table else (),
-            reverse=False)
+    if sort:
+        for k, reverse in reversed(sort):
+            names.sort(key=lambda n: (getattr(table[n], k),)
+                if getattr(table.get(n), k, None) is not None else (),
+                reverse=reverse ^ (not k or k in Result._fields))
 
-    # print header
-    if not summary:
-        title = '%s%s' % (
-            'file' if by_file else 'struct',
+
+    # build up our lines
+    lines = []
+
+    # header
+    line = []
+    line.append('%s%s' % (
+        ','.join(by),
         ' (%d added, %d removed)' % (
             sum(1 for n in table if n not in diff_table),
             sum(1 for n in diff_table if n not in table))
             if diff_results is not None and not percent else '')
-        name_width = max(it.chain([23, len(title)], (len(n) for n in names)))
-    else:
-        title = ''
-        name_width = 23
-    name_width = 4*((name_width+1+4-1)//4)-1
-
-    print('%-*s ' % (name_width, title), end='')
+        if not summary else '')
     if diff_results is None:
-        print(' %s' % ('size'.rjust(len(IntField.none))))
+        for k in fields:
+            line.append(k)
     elif percent:
-        print(' %s' % ('size'.rjust(len(IntField.diff_none))))
+        for k in fields:
+            line.append(k)
     else:
-        print(' %s %s %s' % (
-            'old'.rjust(len(IntField.diff_none)),
-            'new'.rjust(len(IntField.diff_none)),
-            'diff'.rjust(len(IntField.diff_none))))
+        for k in fields:
+            line.append('o'+k)
+        for k in fields:
+            line.append('n'+k)
+        for k in fields:
+            line.append('d'+k)
+    line.append('')
+    lines.append(line)
 
-    # print entries
+    # entries
     if not summary:
         for name in names:
             r = table.get(name)
             if diff_results is not None:
                 diff_r = diff_table.get(name)
-                ratio = IntField.ratio(
-                    r.struct_size if r else None,
-                    diff_r.struct_size if diff_r else None)
-                if not ratio and not all_:
+                ratios = [
+                    types[k].ratio(
+                        getattr(r, k, None),
+                        getattr(diff_r, k, None))
+                    for k in fields]
+                if not any(ratios) and not all_:
                     continue
 
-            print('%-*s ' % (name_width, name), end='')
+            line = []
+            line.append(name)
             if diff_results is None:
-                print(' %s' % (
-                    r.struct_size.table()
-                        if r else IntField.none))
+                for k in fields:
+                    line.append(getattr(r, k).table()
+                        if getattr(r, k, None) is not None
+                        else types[k].none)
             elif percent:
-                print(' %s%s' % (
-                    r.struct_size.diff_table()
-                        if r else IntField.diff_none,
-                    ' (%s)' % (
-                        '+∞%' if ratio == +m.inf
-                        else '-∞%' if ratio == -m.inf
-                        else '%+.1f%%' % (100*ratio))))
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
             else:
-                print(' %s %s %s%s' % (
-                    diff_r.struct_size.diff_table()
-                        if diff_r else IntField.diff_none,
-                    r.struct_size.diff_table()
-                        if r else IntField.diff_none,
-                    IntField.diff_diff(
-                        r.struct_size if r else None,
-                        diff_r.struct_size if diff_r else None)
-                        if r or diff_r else IntField.diff_none,
-                    ' (%s)' % (
-                        '+∞%' if ratio == +m.inf
-                        else '-∞%' if ratio == -m.inf
-                        else '%+.1f%%' % (100*ratio))
-                        if ratio else ''))
+                for k in fields:
+                    line.append(getattr(diff_r, k).diff_table()
+                        if getattr(diff_r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(getattr(r, k).diff_table()
+                        if getattr(r, k, None) is not None
+                        else types[k].diff_none)
+                for k in fields:
+                    line.append(types[k].diff_diff(
+                            getattr(r, k, None),
+                            getattr(diff_r, k, None)))
+            if diff_results is None:
+                line.append('')
+            elif percent:
+                line.append(' (%s)' % ', '.join(
+                    '+∞%' if t == +m.inf
+                    else '-∞%' if t == -m.inf
+                    else '%+.1f%%' % (100*t)
+                    for t in ratios))
+            else:
+                line.append(' (%s)' % ', '.join(
+                        '+∞%' if t == +m.inf
+                        else '-∞%' if t == -m.inf
+                        else '%+.1f%%' % (100*t)
+                        for t in ratios
+                        if t)
+                    if any(ratios) else '')
+            lines.append(line)
 
-    # print total
-    total = fold(results, by=[])
-    r = total[0] if total else None
+    # total
+    r = next(iter(fold(Result, results, by=[])), None)
     if diff_results is not None:
-        diff_total = fold(diff_results, by=[])
-        diff_r = diff_total[0] if diff_total else None
-        ratio = IntField.ratio(
-            r.struct_size if r else None,
-            diff_r.struct_size if diff_r else None)
+        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
+        ratios = [
+            types[k].ratio(
+                getattr(r, k, None),
+                getattr(diff_r, k, None))
+            for k in fields]
 
-    print('%-*s ' % (name_width, 'TOTAL'), end='')
+    line = []
+    line.append('TOTAL')
     if diff_results is None:
-        print(' %s' % (
-            r.struct_size.table()
-                if r else IntField.none))
+        for k in fields:
+            line.append(getattr(r, k).table()
+                if getattr(r, k, None) is not None
+                else types[k].none)
     elif percent:
-        print(' %s%s' % (
-            r.struct_size.diff_table()
-                if r else IntField.diff_none,
-            ' (%s)' % (
-                '+∞%' if ratio == +m.inf
-                else '-∞%' if ratio == -m.inf
-                else '%+.1f%%' % (100*ratio))))
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
     else:
-        print(' %s %s %s%s' % (
-            diff_r.struct_size.diff_table()
-                if diff_r else IntField.diff_none,
-            r.struct_size.diff_table()
-                if r else IntField.diff_none,
-            IntField.diff_diff(
-                r.struct_size if r else None,
-                diff_r.struct_size if diff_r else None)
-                if r or diff_r else IntField.diff_none,
-            ' (%s)' % (
-                '+∞%' if ratio == +m.inf
-                else '-∞%' if ratio == -m.inf
-                else '%+.1f%%' % (100*ratio))
-                if ratio else ''))
+        for k in fields:
+            line.append(getattr(diff_r, k).diff_table()
+                if getattr(diff_r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(getattr(r, k).diff_table()
+                if getattr(r, k, None) is not None
+                else types[k].diff_none)
+        for k in fields:
+            line.append(types[k].diff_diff(
+                    getattr(r, k, None),
+                    getattr(diff_r, k, None)))
+    if diff_results is None:
+        line.append('')
+    elif percent:
+        line.append(' (%s)' % ', '.join(
+            '+∞%' if t == +m.inf
+            else '-∞%' if t == -m.inf
+            else '%+.1f%%' % (100*t)
+            for t in ratios))
+    else:
+        line.append(' (%s)' % ', '.join(
+                '+∞%' if t == +m.inf
+                else '-∞%' if t == -m.inf
+                else '%+.1f%%' % (100*t)
+                for t in ratios
+                if t)
+            if any(ratios) else '')
+    lines.append(line)
+
+    # find the best widths, note that column 0 contains the names and column -1
+    # the ratios, so those are handled a bit differently
+    widths = [
+        ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1
+        for w, i in zip(
+            it.chain([23], it.repeat(7)),
+            range(len(lines[0])-1))]
+
+    # print our table
+    for line in lines:
+        print('%-*s  %s%s' % (
+            widths[0], line[0],
+            ' '.join('%*s' % (w, x)
+                for w, x in zip(widths[1:], line[1:-1])),
+            line[-1]))
 
 
-def main(obj_paths, **args):
+def main(obj_paths, *,
+        by=None,
+        fields=None,
+        defines=None,
+        sort=None,
+        **args):
     # find sizes
     if not args.get('use', None):
         # find .o files
@@ -404,7 +479,7 @@ def main(obj_paths, **args):
                 paths.append(path)
 
         if not paths:
-            print('no .obj files found in %r?' % obj_paths)
+            print("error: no .obj files found in %r?" % obj_paths)
             sys.exit(-1)
 
         results = collect(paths, **args)
@@ -414,25 +489,38 @@ def main(obj_paths, **args):
             reader = csv.DictReader(f, restval='')
             for r in reader:
                 try:
-                    results.append(StructResult(**{
-                        k: v for k, v in r.items()
-                        if k in StructResult._fields}))
+                    results.append(StructResult(
+                        **{k: r[k] for k in StructResult._by
+                            if k in r and r[k].strip()},
+                        **{k: r['struct_'+k]
+                            for k in StructResult._fields
+                            if 'struct_'+k in r
+                                and r['struct_'+k].strip()}))
                 except TypeError:
                     pass
 
-    # fold to remove duplicates
-    results = fold(results)
+    # fold
+    results = fold(StructResult, results, by=by, defines=defines)
 
-    # sort because why not
+    # sort, note that python's sort is stable
     results.sort()
+    if sort:
+        for k, reverse in reversed(sort):
+            results.sort(key=lambda r: (getattr(r, k),)
+                if getattr(r, k) is not None else (),
+                reverse=reverse ^ (not k or k in StructResult._fields))
 
     # write results to CSV
     if args.get('output'):
         with openio(args['output'], 'w') as f:
-            writer = csv.DictWriter(f, StructResult._fields)
+            writer = csv.DictWriter(f, StructResult._by
+                + ['struct_'+k for k in StructResult._fields])
             writer.writeheader()
             for r in results:
-                writer.writerow(r._asdict())
+                writer.writerow(
+                    {k: getattr(r, k) for k in StructResult._by}
+                    | {'struct_'+k: getattr(r, k)
+                        for k in StructResult._fields})
 
     # find previous results?
     if args.get('diff'):
@@ -442,22 +530,28 @@ def main(obj_paths, **args):
                 reader = csv.DictReader(f, restval='')
                 for r in reader:
                     try:
-                        diff_results.append(StructResult(**{
-                            k: v for k, v in r.items()
-                            if k in StructResult._fields}))
+                        diff_results.append(StructResult(
+                            **{k: r[k] for k in StructResult._by
+                                if k in r and r[k].strip()},
+                            **{k: r['struct_'+k]
+                                for k in StructResult._fields
+                                if 'struct_'+k in r
+                                    and r['struct_'+k].strip()}))
                     except TypeError:
                         pass
         except FileNotFoundError:
             pass
 
-        # fold to remove duplicates
-        diff_results = fold(diff_results)
+        # fold
+        diff_results = fold(StructResult, diff_results, by=by, defines=defines)
 
     # print table
     if not args.get('quiet'):
-        table(
-            results,
+        table(StructResult, results,
             diff_results if args.get('diff') else None,
+            by=by if by is not None else ['struct'],
+            fields=fields,
+            sort=sort,
             **args)
 
 
@@ -498,22 +592,39 @@ if __name__ == "__main__":
         action='store_true',
         help="Only show percentage change, not a full diff.")
     parser.add_argument(
-        '-b', '--by-file',
-        action='store_true',
-        help="Group by file. Note this does not include padding "
-            "so sizes may differ from other tools.")
+        '-b', '--by',
+        action='append',
+        choices=StructResult._by,
+        help="Group by this field.")
     parser.add_argument(
-        '-s', '--size-sort',
-        action='store_true',
-        help="Sort by size.")
+        '-f', '--field',
+        dest='fields',
+        action='append',
+        choices=StructResult._fields,
+        help="Show this field.")
     parser.add_argument(
-        '-S', '--reverse-size-sort',
-        action='store_true',
-        help="Sort by size, but backwards.")
+        '-D', '--define',
+        dest='defines',
+        action='append',
+        type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
+        help="Only include results where this field is this value.")
+    class AppendSort(argparse.Action):
+        def __call__(self, parser, namespace, value, option):
+            if namespace.sort is None:
+                namespace.sort = []
+            namespace.sort.append((value, True if option == '-S' else False))
+    parser.add_argument(
+        '-s', '--sort',
+        action=AppendSort,
+        help="Sort by this field.")
+    parser.add_argument(
+        '-S', '--reverse-sort',
+        action=AppendSort,
+        help="Sort by this field, but backwards.")
     parser.add_argument(
         '-Y', '--summary',
         action='store_true',
-        help="Only show the total size.")
+        help="Only show the total.")
     parser.add_argument(
         '-A', '--everything',
         action='store_true',