scripts: Moved recursive entry generation before table rendering

This fixes an issue where mixing recursive renderers (-t/--hot or
-z/--depth) with defines (-Dfunction=lfsr_mount) would not account for
children entry widths. An unexpected side-effect of no longer filtering
the children entries.

We could continue to try to estimate the width without table rendering,
but it would basically need two full recursive pass at this point...
Instead, I've just moved the recursive stuff before table rendering,
which should remove any issues with width calculation while also
deduplicating the recursive passes.

It's invasive for a small change, but probably worthwhile long term.

The downside is this does mean our recursive scripts now build the full
table (including all recursive calls!) before they start printing. When
mixed with unbounded recursive depth (-z0 or --depth=0) this can get
quite large and cause quite a slow start.

But I guess that was the tradeoff in adopting this sort of intermediate
table rendering... At least it does make the code simpler and less bug
prone...

scripts/code.py

@@ -430,6 +430,7 @@ def table(Result, results, diff_results=None, *,
             header.append('d'+k)
     lines.append(header)
 
+    # entry helper
     def table_entry(name, r, diff_r=None):
         entry = [name]
         if diff_results is None:

scripts/cov.py

@@ -418,6 +418,7 @@ def table(Result, results, diff_results=None, *,
             header.append('d'+k)
     lines.append(header)
 
+    # entry helper
     def table_entry(name, r, diff_r=None):
         entry = [name]
         if diff_results is None:

scripts/data.py

@@ -430,6 +430,7 @@ def table(Result, results, diff_results=None, *,
             header.append('d'+k)
     lines.append(header)
 
+    # entry helper
     def table_entry(name, r, diff_r=None):
         entry = [name]
         if diff_results is None:

scripts/perf.py

@@ -746,6 +746,7 @@ def table(Result, results, diff_results=None, *,
             header.append('d'+k)
     lines.append(header)
 
+    # entry helper
     def table_entry(name, r, diff_r=None):
         entry = [name]
         if diff_results is None:
@@ -790,76 +791,7 @@ def table(Result, results, diff_results=None, *,
                                 getattr(diff_r, k, None)))))
         return entry
 
-    # entries
+    # recursive entry helpers
-    if not summary:
-        for name in names:
-            r = table.get(name)
-            if diff_results is None:
-                diff_r = None
-            else:
-                diff_r = diff_table.get(name)
-            lines.append(table_entry(name, r, diff_r))
-
-    # total
-    r = next(iter(fold(Result, results, by=[])), None)
-    if diff_results is None:
-        diff_r = None
-    else:
-        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
-    lines.append(table_entry('TOTAL', r, diff_r))
-
-    # homogenize
-    lines = [
-        [x if isinstance(x, tuple) else (x, []) for x in line]
-        for line in lines]
-
-    # find the best widths, note that column 0 contains the names and is
-    # handled a bit differently
-    widths = co.defaultdict(lambda: 7, {0: 23})
-    notes = co.defaultdict(lambda: 0)
-    for line in lines:
-        for i, x in enumerate(line):
-            widths[i] = max(widths[i], ((len(x[0])+1+4-1)//4)*4-1)
-            notes[i] = max(notes[i], 1+2*len(x[1])+sum(len(n) for n in x[1]))
-
-    if not summary:
-        # find the actual depth
-        depth_ = depth
-        if hot:
-            depth_ = 2
-        elif m.isinf(depth_):
-            def rec_depth(results_, seen=set()):
-                # build the children table at each layer
-                results_ = fold(Result, results_, by=by)
-                table_ = {
-                    ','.join(str(getattr(r, k) or '') for k in by): r
-                    for r in results_}
-                names_ = list(table_.keys())
-
-                return max(
-                    (rec_depth(table_[name].children, seen | {name})
-                        for name in names_
-                        if name not in seen),
-                    default=-1) + 1
-
-            depth_ = max(
-                (rec_depth(table[name].children, {name})
-                    for name in names
-                    if name in table),
-                default=-1) + 1
-
-        # adjust the name width based on the call depth
-        widths[0] += 4*max(depth_-1, 0)
-
-    # print the tree recursively
-    print('%-*s  %s' % (
-        widths[0], lines[0][0][0],
-        ' '.join('%*s%-*s' % (
-                widths[i], x[0],
-                notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
-            for i, x in enumerate(lines[0][1:], 1))))
-
-    if not summary:
     if hot:
         def recurse(results_, depth_, seen=set(),
                 prefixes=('', '', '', '')):
@@ -892,20 +824,13 @@ def table(Result, results, diff_results=None, *,
             is_last = not r.children
 
             line = table_entry(name, r)
-                line = [x if isinstance(x, tuple) else (x, [])
+            line = [x if isinstance(x, tuple) else (x, []) for x in line]
-                    for x in line]
+            # add prefixes
-                print('%s%-*s  %s' % (
+            line[0] = (prefixes[0+is_last] + line[0][0], line[0][1])
-                    prefixes[0+is_last],
+            # add cycle detection
-                    widths[0] - len(prefixes[0+is_last]), line[0][0],
+            if name in seen:
-                    ' '.join('%*s%-*s' % (
+                line[-1] = (line[-1][0], line[-1][1] + ['cycle detected'])
-                            widths[i], x[0],
+            lines.append(line)
-                            notes[i],
-                                ' (%s)' % ', '.join(it.chain(
-                                        x[1], ['cycle detected']))
-                                    if i == len(widths)-1 and name in seen
-                                else ' (%s)' % ', '.join(x[1]) if x[1]
-                                else '')
-                        for i, x in enumerate(line[1:], 1))))
 
             # found a cycle?
             if name in seen:
@@ -947,20 +872,13 @@ def table(Result, results, diff_results=None, *,
                 is_last = (i == len(names_)-1)
 
                 line = table_entry(name, r)
-                    line = [x if isinstance(x, tuple) else (x, [])
+                line = [x if isinstance(x, tuple) else (x, []) for x in line]
-                        for x in line]
+                # add prefixes
-                    print('%s%-*s  %s' % (
+                line[0] = (prefixes[0+is_last] + line[0][0], line[0][1])
-                        prefixes[0+is_last],
+                # add cycle detection
-                        widths[0] - len(prefixes[0+is_last]), line[0][0],
+                if name in seen:
-                        ' '.join('%*s%-*s' % (
+                    line[-1] = (line[-1][0], line[-1][1] + ['cycle detected'])
-                                widths[i], x[0],
+                lines.append(line)
-                                notes[i],
-                                    ' (%s)' % ', '.join(it.chain(
-                                            x[1], ['cycle detected']))
-                                        if i == len(widths)-1 and name in seen
-                                    else ' (%s)' % ', '.join(x[1]) if x[1]
-                                    else '')
-                            for i, x in enumerate(line[1:], 1))))
 
                 # found a cycle?
                 if name in seen:
@@ -977,15 +895,17 @@ def table(Result, results, diff_results=None, *,
                          prefixes[2+is_last] + "|   ",
                          prefixes[2+is_last] + "    "))
 
-        # the top layer is a bit of a special case
+    # entries
-        for name, line in zip(names, lines[1:-1]):
+    if not summary:
-            print('%-*s  %s' % (
+        for name in names:
-                widths[0], line[0][0],
+            r = table.get(name)
-                ' '.join('%*s%-*s' % (
+            if diff_results is None:
-                        widths[i], x[0],
+                diff_r = None
-                        notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
+            else:
-                    for i, x in enumerate(line[1:], 1))))
+                diff_r = diff_table.get(name)
+            lines.append(table_entry(name, r, diff_r))
 
+            # recursive entries
             if name in table and depth > 1:
                 recurse(
                     table[name].children,
@@ -996,12 +916,36 @@ def table(Result, results, diff_results=None, *,
                      "|   ",
                      "    "))
 
+    # total
+    r = next(iter(fold(Result, results, by=[])), None)
+    if diff_results is None:
+        diff_r = None
+    else:
+        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
+    lines.append(table_entry('TOTAL', r, diff_r))
+
+    # homogenize
+    lines = [
+        [x if isinstance(x, tuple) else (x, []) for x in line]
+        for line in lines]
+
+    # find the best widths, note that column 0 contains the names and is
+    # handled a bit differently
+    widths = co.defaultdict(lambda: 7, {0: 23})
+    notes = co.defaultdict(lambda: 0)
+    for line in lines:
+        for i, x in enumerate(line):
+            widths[i] = max(widths[i], ((len(x[0])+1+4-1)//4)*4-1)
+            notes[i] = max(notes[i], 1+2*len(x[1])+sum(len(n) for n in x[1]))
+
+    # print our table
+    for line in lines:
         print('%-*s  %s' % (
-        widths[0], lines[-1][0][0],
+            widths[0], line[0][0],
             ' '.join('%*s%-*s' % (
                     widths[i], x[0],
                     notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
-            for i, x in enumerate(lines[-1][1:], 1))))
+                for i, x in enumerate(line[1:], 1))))
 
 
 def annotate(Result, results, *,

scripts/perfbd.py

@@ -710,6 +710,7 @@ def table(Result, results, diff_results=None, *,
             header.append('d'+k)
     lines.append(header)
 
+    # entry helper
     def table_entry(name, r, diff_r=None):
         entry = [name]
         if diff_results is None:
@@ -754,76 +755,7 @@ def table(Result, results, diff_results=None, *,
                                 getattr(diff_r, k, None)))))
         return entry
 
-    # entries
+    # recursive entry helpers
-    if not summary:
-        for name in names:
-            r = table.get(name)
-            if diff_results is None:
-                diff_r = None
-            else:
-                diff_r = diff_table.get(name)
-            lines.append(table_entry(name, r, diff_r))
-
-    # total
-    r = next(iter(fold(Result, results, by=[])), None)
-    if diff_results is None:
-        diff_r = None
-    else:
-        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
-    lines.append(table_entry('TOTAL', r, diff_r))
-
-    # homogenize
-    lines = [
-        [x if isinstance(x, tuple) else (x, []) for x in line]
-        for line in lines]
-
-    # find the best widths, note that column 0 contains the names and is
-    # handled a bit differently
-    widths = co.defaultdict(lambda: 7, {0: 23})
-    notes = co.defaultdict(lambda: 0)
-    for line in lines:
-        for i, x in enumerate(line):
-            widths[i] = max(widths[i], ((len(x[0])+1+4-1)//4)*4-1)
-            notes[i] = max(notes[i], 1+2*len(x[1])+sum(len(n) for n in x[1]))
-
-    if not summary:
-        # find the actual depth
-        depth_ = depth
-        if hot:
-            depth_ = 2
-        elif m.isinf(depth_):
-            def rec_depth(results_, seen=set()):
-                # build the children table at each layer
-                results_ = fold(Result, results_, by=by)
-                table_ = {
-                    ','.join(str(getattr(r, k) or '') for k in by): r
-                    for r in results_}
-                names_ = list(table_.keys())
-
-                return max(
-                    (rec_depth(table_[name].children, seen | {name})
-                        for name in names_
-                        if name not in seen),
-                    default=-1) + 1
-
-            depth_ = max(
-                (rec_depth(table[name].children, {name})
-                    for name in names
-                    if name in table),
-                default=-1) + 1
-
-        # adjust the name width based on the call depth
-        widths[0] += 4*max(depth_-1, 0)
-
-    # print the tree recursively
-    print('%-*s  %s' % (
-        widths[0], lines[0][0][0],
-        ' '.join('%*s%-*s' % (
-                widths[i], x[0],
-                notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
-            for i, x in enumerate(lines[0][1:], 1))))
-
-    if not summary:
     if hot:
         def recurse(results_, depth_, seen=set(),
                 prefixes=('', '', '', '')):
@@ -856,20 +788,13 @@ def table(Result, results, diff_results=None, *,
             is_last = not r.children
 
             line = table_entry(name, r)
-                line = [x if isinstance(x, tuple) else (x, [])
+            line = [x if isinstance(x, tuple) else (x, []) for x in line]
-                    for x in line]
+            # add prefixes
-                print('%s%-*s  %s' % (
+            line[0] = (prefixes[0+is_last] + line[0][0], line[0][1])
-                    prefixes[0+is_last],
+            # add cycle detection
-                    widths[0] - len(prefixes[0+is_last]), line[0][0],
+            if name in seen:
-                    ' '.join('%*s%-*s' % (
+                line[-1] = (line[-1][0], line[-1][1] + ['cycle detected'])
-                            widths[i], x[0],
+            lines.append(line)
-                            notes[i],
-                                ' (%s)' % ', '.join(it.chain(
-                                        x[1], ['cycle detected']))
-                                    if i == len(widths)-1 and name in seen
-                                else ' (%s)' % ', '.join(x[1]) if x[1]
-                                else '')
-                        for i, x in enumerate(line[1:], 1))))
 
             # found a cycle?
             if name in seen:
@@ -911,20 +836,13 @@ def table(Result, results, diff_results=None, *,
                 is_last = (i == len(names_)-1)
 
                 line = table_entry(name, r)
-                    line = [x if isinstance(x, tuple) else (x, [])
+                line = [x if isinstance(x, tuple) else (x, []) for x in line]
-                        for x in line]
+                # add prefixes
-                    print('%s%-*s  %s' % (
+                line[0] = (prefixes[0+is_last] + line[0][0], line[0][1])
-                        prefixes[0+is_last],
+                # add cycle detection
-                        widths[0] - len(prefixes[0+is_last]), line[0][0],
+                if name in seen:
-                        ' '.join('%*s%-*s' % (
+                    line[-1] = (line[-1][0], line[-1][1] + ['cycle detected'])
-                                widths[i], x[0],
+                lines.append(line)
-                                notes[i],
-                                    ' (%s)' % ', '.join(it.chain(
-                                            x[1], ['cycle detected']))
-                                        if i == len(widths)-1 and name in seen
-                                    else ' (%s)' % ', '.join(x[1]) if x[1]
-                                    else '')
-                            for i, x in enumerate(line[1:], 1))))
 
                 # found a cycle?
                 if name in seen:
@@ -941,15 +859,17 @@ def table(Result, results, diff_results=None, *,
                          prefixes[2+is_last] + "|   ",
                          prefixes[2+is_last] + "    "))
 
-        # the top layer is a bit of a special case
+    # entries
-        for name, line in zip(names, lines[1:-1]):
+    if not summary:
-            print('%-*s  %s' % (
+        for name in names:
-                widths[0], line[0][0],
+            r = table.get(name)
-                ' '.join('%*s%-*s' % (
+            if diff_results is None:
-                        widths[i], x[0],
+                diff_r = None
-                        notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
+            else:
-                    for i, x in enumerate(line[1:], 1))))
+                diff_r = diff_table.get(name)
+            lines.append(table_entry(name, r, diff_r))
 
+            # recursive entries
             if name in table and depth > 1:
                 recurse(
                     table[name].children,
@@ -960,12 +880,36 @@ def table(Result, results, diff_results=None, *,
                      "|   ",
                      "    "))
 
+    # total
+    r = next(iter(fold(Result, results, by=[])), None)
+    if diff_results is None:
+        diff_r = None
+    else:
+        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
+    lines.append(table_entry('TOTAL', r, diff_r))
+
+    # homogenize
+    lines = [
+        [x if isinstance(x, tuple) else (x, []) for x in line]
+        for line in lines]
+
+    # find the best widths, note that column 0 contains the names and is
+    # handled a bit differently
+    widths = co.defaultdict(lambda: 7, {0: 23})
+    notes = co.defaultdict(lambda: 0)
+    for line in lines:
+        for i, x in enumerate(line):
+            widths[i] = max(widths[i], ((len(x[0])+1+4-1)//4)*4-1)
+            notes[i] = max(notes[i], 1+2*len(x[1])+sum(len(n) for n in x[1]))
+
+    # print our table
+    for line in lines:
         print('%-*s  %s' % (
-        widths[0], lines[-1][0][0],
+            widths[0], line[0][0],
             ' '.join('%*s%-*s' % (
                     widths[i], x[0],
                     notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
-            for i, x in enumerate(lines[-1][1:], 1))))
+                for i, x in enumerate(line[1:], 1))))
 
 
 def annotate(Result, results, *,

scripts/stack.py

@@ -395,6 +395,7 @@ def table(Result, results, diff_results=None, *,
             header.append('d'+k)
     lines.append(header)
 
+    # entry helper
     def table_entry(name, r, diff_r=None):
         entry = [name]
         if diff_results is None:
@@ -439,76 +440,7 @@ def table(Result, results, diff_results=None, *,
                                 getattr(diff_r, k, None)))))
         return entry
 
-    # entries
+    # recursive entry helpers
-    if not summary:
-        for name in names:
-            r = table.get(name)
-            if diff_results is None:
-                diff_r = None
-            else:
-                diff_r = diff_table.get(name)
-            lines.append(table_entry(name, r, diff_r))
-
-    # total
-    r = next(iter(fold(Result, results, by=[])), None)
-    if diff_results is None:
-        diff_r = None
-    else:
-        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
-    lines.append(table_entry('TOTAL', r, diff_r))
-
-    # homogenize
-    lines = [
-        [x if isinstance(x, tuple) else (x, []) for x in line]
-        for line in lines]
-
-    # find the best widths, note that column 0 contains the names and is
-    # handled a bit differently
-    widths = co.defaultdict(lambda: 7, {0: 23})
-    notes = co.defaultdict(lambda: 0)
-    for line in lines:
-        for i, x in enumerate(line):
-            widths[i] = max(widths[i], ((len(x[0])+1+4-1)//4)*4-1)
-            notes[i] = max(notes[i], 1+2*len(x[1])+sum(len(n) for n in x[1]))
-
-    if not summary:
-        # find the actual depth
-        depth_ = depth
-        if hot:
-            depth_ = 2
-        elif m.isinf(depth_):
-            def rec_depth(results_, seen=set()):
-                # build the children table at each layer
-                results_ = fold(Result, results_, by=by)
-                table_ = {
-                    ','.join(str(getattr(r, k) or '') for k in by): r
-                    for r in results_}
-                names_ = list(table_.keys())
-
-                return max(
-                    (rec_depth(table_[name].children, seen | {name})
-                        for name in names_
-                        if name not in seen),
-                    default=-1) + 1
-
-            depth_ = max(
-                (rec_depth(table[name].children, {name})
-                    for name in names
-                    if name in table),
-                default=-1) + 1
-
-        # adjust the name width based on the call depth
-        widths[0] += 4*max(depth_-1, 0)
-
-    # print the tree recursively
-    print('%-*s  %s' % (
-        widths[0], lines[0][0][0],
-        ' '.join('%*s%-*s' % (
-                widths[i], x[0],
-                notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
-            for i, x in enumerate(lines[0][1:], 1))))
-
-    if not summary:
     if hot:
         def recurse(results_, depth_, seen=set(),
                 prefixes=('', '', '', '')):
@@ -541,20 +473,13 @@ def table(Result, results, diff_results=None, *,
             is_last = not r.children
 
             line = table_entry(name, r)
-                line = [x if isinstance(x, tuple) else (x, [])
+            line = [x if isinstance(x, tuple) else (x, []) for x in line]
-                    for x in line]
+            # add prefixes
-                print('%s%-*s  %s' % (
+            line[0] = (prefixes[0+is_last] + line[0][0], line[0][1])
-                    prefixes[0+is_last],
+            # add cycle detection
-                    widths[0] - len(prefixes[0+is_last]), line[0][0],
+            if name in seen:
-                    ' '.join('%*s%-*s' % (
+                line[-1] = (line[-1][0], line[-1][1] + ['cycle detected'])
-                            widths[i], x[0],
+            lines.append(line)
-                            notes[i],
-                                ' (%s)' % ', '.join(it.chain(
-                                        x[1], ['cycle detected']))
-                                    if i == len(widths)-1 and name in seen
-                                else ' (%s)' % ', '.join(x[1]) if x[1]
-                                else '')
-                        for i, x in enumerate(line[1:], 1))))
 
             # found a cycle?
             if name in seen:
@@ -596,20 +521,13 @@ def table(Result, results, diff_results=None, *,
                 is_last = (i == len(names_)-1)
 
                 line = table_entry(name, r)
-                    line = [x if isinstance(x, tuple) else (x, [])
+                line = [x if isinstance(x, tuple) else (x, []) for x in line]
-                        for x in line]
+                # add prefixes
-                    print('%s%-*s  %s' % (
+                line[0] = (prefixes[0+is_last] + line[0][0], line[0][1])
-                        prefixes[0+is_last],
+                # add cycle detection
-                        widths[0] - len(prefixes[0+is_last]), line[0][0],
+                if name in seen:
-                        ' '.join('%*s%-*s' % (
+                    line[-1] = (line[-1][0], line[-1][1] + ['cycle detected'])
-                                widths[i], x[0],
+                lines.append(line)
-                                notes[i],
-                                    ' (%s)' % ', '.join(it.chain(
-                                            x[1], ['cycle detected']))
-                                        if i == len(widths)-1 and name in seen
-                                    else ' (%s)' % ', '.join(x[1]) if x[1]
-                                    else '')
-                            for i, x in enumerate(line[1:], 1))))
 
                 # found a cycle?
                 if name in seen:
@@ -626,15 +544,17 @@ def table(Result, results, diff_results=None, *,
                          prefixes[2+is_last] + "|   ",
                          prefixes[2+is_last] + "    "))
 
-        # the top layer is a bit of a special case
+    # entries
-        for name, line in zip(names, lines[1:-1]):
+    if not summary:
-            print('%-*s  %s' % (
+        for name in names:
-                widths[0], line[0][0],
+            r = table.get(name)
-                ' '.join('%*s%-*s' % (
+            if diff_results is None:
-                        widths[i], x[0],
+                diff_r = None
-                        notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
+            else:
-                    for i, x in enumerate(line[1:], 1))))
+                diff_r = diff_table.get(name)
+            lines.append(table_entry(name, r, diff_r))
 
+            # recursive entries
             if name in table and depth > 1:
                 recurse(
                     table[name].children,
@@ -645,12 +565,36 @@ def table(Result, results, diff_results=None, *,
                      "|   ",
                      "    "))
 
+    # total
+    r = next(iter(fold(Result, results, by=[])), None)
+    if diff_results is None:
+        diff_r = None
+    else:
+        diff_r = next(iter(fold(Result, diff_results, by=[])), None)
+    lines.append(table_entry('TOTAL', r, diff_r))
+
+    # homogenize
+    lines = [
+        [x if isinstance(x, tuple) else (x, []) for x in line]
+        for line in lines]
+
+    # find the best widths, note that column 0 contains the names and is
+    # handled a bit differently
+    widths = co.defaultdict(lambda: 7, {0: 23})
+    notes = co.defaultdict(lambda: 0)
+    for line in lines:
+        for i, x in enumerate(line):
+            widths[i] = max(widths[i], ((len(x[0])+1+4-1)//4)*4-1)
+            notes[i] = max(notes[i], 1+2*len(x[1])+sum(len(n) for n in x[1]))
+
+    # print our table
+    for line in lines:
         print('%-*s  %s' % (
-        widths[0], lines[-1][0][0],
+            widths[0], line[0][0],
             ' '.join('%*s%-*s' % (
                     widths[i], x[0],
                     notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
-            for i, x in enumerate(lines[-1][1:], 1))))
+                for i, x in enumerate(line[1:], 1))))
 
 
 def main(ci_paths,

scripts/structs.py

@@ -379,6 +379,7 @@ def table(Result, results, diff_results=None, *,
             header.append('d'+k)
     lines.append(header)
 
+    # entry helper
     def table_entry(name, r, diff_r=None):
         entry = [name]
         if diff_results is None:

scripts/summary.py

@@ -510,6 +510,7 @@ def table(Result, results, diff_results=None, *,
             header.append('d'+k)
     lines.append(header)
 
+    # entry helper
     def table_entry(name, r, diff_r=None):
         entry = [name]
         if diff_results is None: