From e32af5cd8aacf36140771411e940ec4036cf6827 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 3 Nov 2024 16:32:38 -0600 Subject: [PATCH] scripts: Added -t/--hot to recursive scripts, stack.py, etc This is mainly useful for stack.py, where -t/--hot lets you quickly see everything that contributes to the stack limit for each function. This was (and still is) possible with -s + -z, but it was pretty annoying to use: - The stack trace rendered _diagonally_ as a consequence of -z, which is probably the worst use of screen real estate. - This trick only really worked with -s, which was the opposite order of what you usually want on the command line: -S. Adding a special for-purpose -t/--hot flag makes looking at the hot path much easier, at the cost of more hacky python code (and I _mean_ hacky, making the hot path selection useful while following exising sort rules was annoyingly complicated). Also added -t/--hot to perf.py and perfbd.py for consistency, though it makes a bit less sense there. Also also reworked related code in all three scripts: stack.py, perf.py, perfbd.py. The logic should be a bit more equivalent, and perf.py/perfbd.py detect cycles now. --- scripts/perf.py | 154 +++++++++++++++++++++++++++++---------- scripts/perfbd.py | 154 +++++++++++++++++++++++++++++---------- scripts/stack.py | 181 +++++++++++++++++++++++++++++++++------------- 3 files changed, 364 insertions(+), 125 deletions(-) diff --git a/scripts/perf.py b/scripts/perf.py index 2ad30e02..7957c599 100755 --- a/scripts/perf.py +++ b/scripts/perf.py @@ -675,7 +675,8 @@ def table(Result, results, diff_results=None, *, summary=False, all=False, percent=False, - depth=1, + depth=None, + hot=False, **_): all_, all = all, __builtins__.all @@ -830,26 +831,27 @@ def table(Result, results, diff_results=None, *, if not summary: # find the actual depth - # - # note unlike stack.py we can't end up with cycles here depth_ = depth - if m.isinf(depth_): - def rec_depth(results_): + if hot: + depth_ = 2 + elif m.isinf(depth_): + def rec_depth(results_, seen=set()): # rebuild our tables at each layer table_ = { ','.join(str(getattr(r, k) or '') for k in by): r for r in results_} names_ = list(table_.keys()) - return max(( - rec_depth(table_[name].children) - for name in names_), + return max( + (rec_depth(table_[name].children, seen | {name}) + for name in names_ + if name not in seen), default=-1) + 1 - depth_ = max(( - rec_depth(table[name].children) - for name in names - if name in table), + depth_ = max( + (rec_depth(table[name].children, {name}) + for name in names + if name in table), default=-1) + 1 # adjust the name width based on the call depth @@ -864,50 +866,121 @@ def table(Result, results, diff_results=None, *, for i, x in enumerate(lines[0][1:], 1)))) if not summary: - def recurse(results_, depth_, prefixes=('', '', '', '')): - # rebuild our tables at each layer - table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r - for r in results_} - names_ = list(table_.keys()) + if hot: + def recurse(results_, depth_, seen=set(), + prefixes=('', '', '', '')): + # rebuild our tables at each layer + table_ = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results_} + names_ = list(table_.keys()) + if not names_: + return - # sort again at each layer, keep in mind the numbers are - # changing as we descend - names_.sort() - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( + # find the "hottest" path at each step, we use + # the sort field if requested, but ignore reversedness + name = max(names_, + key=lambda n: tuple( + tuple( + # make sure to use the rebuilt table (getattr(table_[n], k),) if getattr(table_.get(n), k, None) is not None else () for k in ([k] if k else [ - k for k in Result._sort if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + k for k in Result._sort if k in fields]) + if k in fields) + for k, reverse in it.chain( + sort or [], + [(None, False)]))) - for i, name in enumerate(names_): r = table_[name] - is_last = (i == len(names_)-1) + is_last = not r.children line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] + line = [x if isinstance(x, tuple) else (x, []) + for x in line] print('%s%-*s %s' % ( prefixes[0+is_last], widths[0] - len(prefixes[0+is_last]), line[0][0], ' '.join('%*s%-*s' % ( widths[i], x[0], - notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') + notes[i], + ' (%s)' % ', '.join(it.chain( + x[1], ['cycle detected'])) + if i == len(widths)-1 and name in seen + else ' (%s)' % ', '.join(x[1]) if x[1] + else '') for i, x in enumerate(line[1:], 1)))) + # found a cycle? + if name in seen: + return + # recurse? if depth_ > 1: recurse( r.children, depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + seen | {name}, + prefixes) + + else: + def recurse(results_, depth_, seen=set(), + prefixes=('', '', '', '')): + # rebuild our tables at each layer + table_ = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results_} + names_ = list(table_.keys()) + + # sort again at each layer, keep in mind the numbers are + # changing as we descend + names_.sort() + if sort: + for k, reverse in reversed(sort): + names_.sort( + key=lambda n: tuple( + (getattr(table_[n], k),) + if getattr(table_.get(n), k, None) is not None + else () + for k in ([k] if k else [ + k for k in Result._sort if k in fields])), + reverse=reverse ^ (not k or k in Result._fields)) + + for i, name in enumerate(names_): + r = table_[name] + is_last = (i == len(names_)-1) + + line = table_entry(name, r) + line = [x if isinstance(x, tuple) else (x, []) + for x in line] + print('%s%-*s %s' % ( + prefixes[0+is_last], + widths[0] - len(prefixes[0+is_last]), line[0][0], + ' '.join('%*s%-*s' % ( + widths[i], x[0], + notes[i], + ' (%s)' % ', '.join(it.chain( + x[1], ['cycle detected'])) + if i == len(widths)-1 and name in seen + else ' (%s)' % ', '.join(x[1]) if x[1] + else '') + for i, x in enumerate(line[1:], 1)))) + + # found a cycle? + if name in seen: + continue + + # recurse? + if depth_ > 1: + recurse( + r.children, + depth_-1, + seen | {name}, + (prefixes[2+is_last] + "|-> ", + prefixes[2+is_last] + "'-> ", + prefixes[2+is_last] + "| ", + prefixes[2+is_last] + " ")) # we have enough going on with diffing to make the top layer # a special case @@ -923,6 +996,7 @@ def table(Result, results, diff_results=None, *, recurse( table[name].children, depth-1, + {name}, ("|-> ", "'-> ", "| ", @@ -1049,8 +1123,10 @@ def report(perf_paths, *, else: args['color'] = False - # depth of 0 == m.inf - if args.get('depth') == 0: + # figure out depth + if args.get('depth') is None: + args['depth'] = m.inf if args.get('hot') else 1 + elif args.get('depth') == 0: args['depth'] = m.inf # find sizes @@ -1278,6 +1354,10 @@ if __name__ == "__main__": const=0, help="Depth of function calls to show. 0 shows all calls but may not " "terminate!") + parser.add_argument( + '-t', '--hot', + action='store_true', + help="Show only the hot path for each function call.") parser.add_argument( '-A', '--annotate', action='store_true', diff --git a/scripts/perfbd.py b/scripts/perfbd.py index 25f96921..e8c134da 100755 --- a/scripts/perfbd.py +++ b/scripts/perfbd.py @@ -639,7 +639,8 @@ def table(Result, results, diff_results=None, *, summary=False, all=False, percent=False, - depth=1, + depth=None, + hot=False, **_): all_, all = all, __builtins__.all @@ -794,26 +795,27 @@ def table(Result, results, diff_results=None, *, if not summary: # find the actual depth - # - # note unlike stack.py we can't end up with cycles here depth_ = depth - if m.isinf(depth_): - def rec_depth(results_): + if hot: + depth_ = 2 + elif m.isinf(depth_): + def rec_depth(results_, seen=set()): # rebuild our tables at each layer table_ = { ','.join(str(getattr(r, k) or '') for k in by): r for r in results_} names_ = list(table_.keys()) - return max(( - rec_depth(table_[name].children) - for name in names_), + return max( + (rec_depth(table_[name].children, seen | {name}) + for name in names_ + if name not in seen), default=-1) + 1 - depth_ = max(( - rec_depth(table[name].children) - for name in names - if name in table), + depth_ = max( + (rec_depth(table[name].children, {name}) + for name in names + if name in table), default=-1) + 1 # adjust the name width based on the call depth @@ -828,50 +830,121 @@ def table(Result, results, diff_results=None, *, for i, x in enumerate(lines[0][1:], 1)))) if not summary: - def recurse(results_, depth_, prefixes=('', '', '', '')): - # rebuild our tables at each layer - table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r - for r in results_} - names_ = list(table_.keys()) + if hot: + def recurse(results_, depth_, seen=set(), + prefixes=('', '', '', '')): + # rebuild our tables at each layer + table_ = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results_} + names_ = list(table_.keys()) + if not names_: + return - # sort again at each layer, keep in mind the numbers are - # changing as we descend - names_.sort() - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( + # find the "hottest" path at each step, we use + # the sort field if requested, but ignore reversedness + name = max(names_, + key=lambda n: tuple( + tuple( + # make sure to use the rebuilt table (getattr(table_[n], k),) if getattr(table_.get(n), k, None) is not None else () for k in ([k] if k else [ - k for k in Result._sort if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + k for k in Result._sort if k in fields]) + if k in fields) + for k, reverse in it.chain( + sort or [], + [(None, False)]))) - for i, name in enumerate(names_): r = table_[name] - is_last = (i == len(names_)-1) + is_last = not r.children line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] + line = [x if isinstance(x, tuple) else (x, []) + for x in line] print('%s%-*s %s' % ( prefixes[0+is_last], widths[0] - len(prefixes[0+is_last]), line[0][0], ' '.join('%*s%-*s' % ( widths[i], x[0], - notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') + notes[i], + ' (%s)' % ', '.join(it.chain( + x[1], ['cycle detected'])) + if i == len(widths)-1 and name in seen + else ' (%s)' % ', '.join(x[1]) if x[1] + else '') for i, x in enumerate(line[1:], 1)))) + # found a cycle? + if name in seen: + return + # recurse? if depth_ > 1: recurse( r.children, depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + seen | {name}, + prefixes) + + else: + def recurse(results_, depth_, seen=set(), + prefixes=('', '', '', '')): + # rebuild our tables at each layer + table_ = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results_} + names_ = list(table_.keys()) + + # sort again at each layer, keep in mind the numbers are + # changing as we descend + names_.sort() + if sort: + for k, reverse in reversed(sort): + names_.sort( + key=lambda n: tuple( + (getattr(table_[n], k),) + if getattr(table_.get(n), k, None) is not None + else () + for k in ([k] if k else [ + k for k in Result._sort if k in fields])), + reverse=reverse ^ (not k or k in Result._fields)) + + for i, name in enumerate(names_): + r = table_[name] + is_last = (i == len(names_)-1) + + line = table_entry(name, r) + line = [x if isinstance(x, tuple) else (x, []) + for x in line] + print('%s%-*s %s' % ( + prefixes[0+is_last], + widths[0] - len(prefixes[0+is_last]), line[0][0], + ' '.join('%*s%-*s' % ( + widths[i], x[0], + notes[i], + ' (%s)' % ', '.join(it.chain( + x[1], ['cycle detected'])) + if i == len(widths)-1 and name in seen + else ' (%s)' % ', '.join(x[1]) if x[1] + else '') + for i, x in enumerate(line[1:], 1)))) + + # found a cycle? + if name in seen: + continue + + # recurse? + if depth_ > 1: + recurse( + r.children, + depth_-1, + seen | {name}, + (prefixes[2+is_last] + "|-> ", + prefixes[2+is_last] + "'-> ", + prefixes[2+is_last] + "| ", + prefixes[2+is_last] + " ")) # we have enough going on with diffing to make the top layer # a special case @@ -887,6 +960,7 @@ def table(Result, results, diff_results=None, *, recurse( table[name].children, depth-1, + {name}, ("|-> ", "'-> ", "| ", @@ -1027,8 +1101,10 @@ def report(obj_path='', trace_paths=[], *, else: args['color'] = False - # depth of 0 == m.inf - if args.get('depth') == 0: + # figure out depth + if args.get('depth') is None: + args['depth'] = m.inf if args.get('hot') else 1 + elif args.get('depth') == 0: args['depth'] = m.inf # find sizes @@ -1239,6 +1315,10 @@ if __name__ == "__main__": const=0, help="Depth of function calls to show. 0 shows all calls but may not " "terminate!") + parser.add_argument( + '-t', '--hot', + action='store_true', + help="Show only the hot path for each function call.") parser.add_argument( '-A', '--annotate', action='store_true', diff --git a/scripts/stack.py b/scripts/stack.py index 7194a5c7..d0d0b232 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -321,7 +321,8 @@ def table(Result, results, diff_results=None, *, summary=False, all=False, percent=False, - depth=1, + depth=None, + hot=False, **_): all_, all = all, __builtins__.all @@ -477,27 +478,26 @@ def table(Result, results, diff_results=None, *, if not summary: # find the actual depth depth_ = depth - if m.isinf(depth_): - def rec_depth(names_, seen=set()): - depth_ = -1 - for name in names_: - # found a cycle? - if name in seen: - continue + if hot: + depth_ = 2 + elif m.isinf(depth_): + def rec_depth(children_, seen=set()): + names_ = { + ','.join(str(getattr(Result(*c), k) or '') + for k in by) + for c in children_} - # recurse? - if name in table: - children = { - ','.join(str(getattr(Result(*c), k) or '') - for k in by) - for c in table[name].children} - depth_ = max(depth_, - rec_depth( - [n for n in names if n in children], - seen | {name})) - return depth_ + 1 + return max( + (rec_depth(table[name].children, seen | {name}) + for name in names_ + if name not in seen), + default=-1) + 1 - depth_ = rec_depth(names) + depth_ = max( + (rec_depth(table[name].children, {name}) + for name in names + if name in table), + default=-1) + 1 # adjust the name width based on the call depth widths[0] += 4*max(depth_-1, 0) @@ -513,46 +513,120 @@ def table(Result, results, diff_results=None, *, if not summary: line_table = {n: l for n, l in zip(names, lines[1:-1])} - def recurse(names_, depth_, seen=set(), prefixes=('', '', '', '')): - for i, name in enumerate(names_): - if name not in line_table: - continue - line = line_table[name] - is_last = (i == len(names_)-1) + if hot: + def recurse(children_, depth_, seen=set(), + prefixes=('', '', '', '')): + names_ = {','.join(str(getattr(Result(*c), k) or '') + for k in by) + for c in children_} + if not names_: + return - print('%s%-*s %s' % ( - prefixes[0+is_last], - widths[0] - len(prefixes[0+is_last]), line[0][0], - ' '.join('%*s%-*s' % ( - widths[i], x[0], - notes[i], - ' (%s)' % ', '.join(it.chain( - x[1], ['cycle detected'])) - if i == len(widths)-1 and name in seen - else ' (%s)' % ', '.join(x[1]) if x[1] - else '') - for i, x in enumerate(line[1:], 1)))) + # find the "hottest" path at each step, we use + # the sort field if requested, but ignore reversedness + name = max(names_, + key=lambda n: tuple( + tuple( + (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None + else () + for k in ([k] if k else [ + k for k in Result._sort if k in fields]) + if k in fields) + for k, reverse in it.chain( + sort or [], + [(None, False)]))) + + if name in line_table: + line = line_table[name] + is_last = not table[name].children + + print('%s%-*s %s' % ( + prefixes[0+is_last], + widths[0] - len(prefixes[0+is_last]), line[0][0], + ' '.join('%*s%-*s' % ( + widths[i], x[0], + notes[i], + ' (%s)' % ', '.join(it.chain( + x[1], ['cycle detected'])) + if i == len(widths)-1 and name in seen + else ' (%s)' % ', '.join(x[1]) if x[1] + else '') + for i, x in enumerate(line[1:], 1)))) # found a cycle? if name in seen: - continue + return # recurse? - if name in table and depth_ > 1: - children = { - ','.join(str(getattr(Result(*c), k) or '') for k in by) - for c in table[name].children} + if depth_ > 1: recurse( - # note we're maintaining sort order - [n for n in names if n in children], + table[name].children, depth_-1, seen | {name}, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + prefixes) - recurse(names, depth) + else: + def recurse(children_, depth_, seen=set(), + prefixes=('', '', '', '')): + # note we're maintaining sort order + names_ = {','.join(str(getattr(Result(*c), k) or '') + for k in by) + for c in children_} + names_ = [n for n in names if n in names_] + + for i, name in enumerate(names_): + if name not in line_table: + continue + line = line_table[name] + is_last = (i == len(names_)-1) + + print('%s%-*s %s' % ( + prefixes[0+is_last], + widths[0] - len(prefixes[0+is_last]), line[0][0], + ' '.join('%*s%-*s' % ( + widths[i], x[0], + notes[i], + ' (%s)' % ', '.join(it.chain( + x[1], ['cycle detected'])) + if i == len(widths)-1 and name in seen + else ' (%s)' % ', '.join(x[1]) if x[1] + else '') + for i, x in enumerate(line[1:], 1)))) + + # found a cycle? + if name in seen: + continue + + # recurse? + if depth_ > 1: + recurse( + table[name].children, + depth_-1, + seen | {name}, + (prefixes[2+is_last] + "|-> ", + prefixes[2+is_last] + "'-> ", + prefixes[2+is_last] + "| ", + prefixes[2+is_last] + " ")) + + # make the top layer a special case + for name, line in zip(names, lines[1:-1]): + print('%-*s %s' % ( + widths[0], line[0][0], + ' '.join('%*s%-*s' % ( + widths[i], x[0], + notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') + for i, x in enumerate(line[1:], 1)))) + + if name in table and depth > 1: + recurse( + table[name].children, + depth-1, + {name}, + ("|-> ", + "'-> ", + "| ", + " ")) print('%-*s %s' % ( widths[0], lines[-1][0][0], @@ -568,8 +642,9 @@ def main(ci_paths, defines=[], sort=None, **args): + # figure out depth if args.get('depth') is None: - args['depth'] = 1 + args['depth'] = m.inf if args.get('hot') else 1 elif args.get('depth') == 0: args['depth'] = m.inf @@ -760,6 +835,10 @@ if __name__ == "__main__": const=0, help="Depth of function calls to show. 0 shows all calls but may not " "terminate!") + parser.add_argument( + '-t', '--hot', + action='store_true', + help="Show only the hot path for each function call.") parser.add_argument( '-e', '--error-on-recursion', action='store_true',