From 0c3868f92c74ad42e47532ac15c5647b2d086d32 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 4 Nov 2024 15:39:50 -0600 Subject: [PATCH] scripts: Fully connected graph in stack.py, no more recursive folding This makes -D/--define more useful in stack.py/perf.py/perfbd.py by no longer hiding undfined children entries. For example: $ ./scripts/stack.py lfs.ci lfs_util.ci -Dfunction=lfsr_mount -t function frame limit lfsr_mount 96 2816 |-> lfsr_fs_gc 80 2720 |-> lfsr_mtree_gc 176 2640 |-> lfsr_mdir_commit 576 2464 ... snip ... Now shows all functions in the hot path of lfsr_mount, where before it would only show functions in the hot path of lfsr_mount that were also _named_ lfsr_mount. The previous behavior was technically not wrong... but not very useful (and confusing). --- This was actually quite a bit annoying to get working because of the possibility of function call cycles. I ended up turning stack.py's result type into a fully connected graph, which only works because Python has a cycle detector. (Actually this script is so short-lived we probably wouldn't care if this leaked memory.) A nice side effect of this is now all the recursive scripts (stack.py, perf.py, and perfbd.py) share the same internal result representation and recursive printing logic, which is probably a good thing. --- scripts/perf.py | 24 +++------ scripts/perfbd.py | 24 +++------ scripts/stack.py | 133 +++++++++++++++++++++++++++------------------- 3 files changed, 93 insertions(+), 88 deletions(-) diff --git a/scripts/perf.py b/scripts/perf.py index 7edd81c5..8086f1df 100755 --- a/scripts/perf.py +++ b/scripts/perf.py @@ -657,15 +657,6 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) - # fold recursively - folded_ = [] - for r in folded: - folded_.append(r._replace(children=fold( - Result, r.children, - by=by, - defines=defines))) - folded = folded_ - return folded def table(Result, results, diff_results=None, *, @@ -838,7 +829,8 @@ def table(Result, results, diff_results=None, *, depth_ = 2 elif m.isinf(depth_): def rec_depth(results_, seen=set()): - # rebuild our tables at each layer + # build the children table at each layer + results_ = fold(Result, results_, by=by) table_ = { ','.join(str(getattr(r, k) or '') for k in by): r for r in results_} @@ -871,7 +863,8 @@ def table(Result, results, diff_results=None, *, if hot: def recurse(results_, depth_, seen=set(), prefixes=('', '', '', '')): - # rebuild our tables at each layer + # build the children table at each layer + results_ = fold(Result, results_, by=by) table_ = { ','.join(str(getattr(r, k) or '') for k in by): r for r in results_} @@ -929,14 +922,14 @@ def table(Result, results, diff_results=None, *, else: def recurse(results_, depth_, seen=set(), prefixes=('', '', '', '')): - # rebuild our tables at each layer + # build the children table at each layer + results_ = fold(Result, results_, by=by) table_ = { ','.join(str(getattr(r, k) or '') for k in by): r for r in results_} names_ = list(table_.keys()) - # sort again at each layer, keep in mind the numbers are - # changing as we descend + # sort the children layer names_.sort() if sort: for k, reverse in reversed(sort): @@ -984,8 +977,7 @@ def table(Result, results, diff_results=None, *, prefixes[2+is_last] + "| ", prefixes[2+is_last] + " ")) - # we have enough going on with diffing to make the top layer - # a special case + # the top layer is a bit of a special case for name, line in zip(names, lines[1:-1]): print('%-*s %s' % ( widths[0], line[0][0], diff --git a/scripts/perfbd.py b/scripts/perfbd.py index 0b9fc82c..cfe2e8f1 100755 --- a/scripts/perfbd.py +++ b/scripts/perfbd.py @@ -621,15 +621,6 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) - # fold recursively - folded_ = [] - for r in folded: - folded_.append(r._replace(children=fold( - Result, r.children, - by=by, - defines=defines))) - folded = folded_ - return folded def table(Result, results, diff_results=None, *, @@ -802,7 +793,8 @@ def table(Result, results, diff_results=None, *, depth_ = 2 elif m.isinf(depth_): def rec_depth(results_, seen=set()): - # rebuild our tables at each layer + # build the children table at each layer + results_ = fold(Result, results_, by=by) table_ = { ','.join(str(getattr(r, k) or '') for k in by): r for r in results_} @@ -835,7 +827,8 @@ def table(Result, results, diff_results=None, *, if hot: def recurse(results_, depth_, seen=set(), prefixes=('', '', '', '')): - # rebuild our tables at each layer + # build the children table at each layer + results_ = fold(Result, results_, by=by) table_ = { ','.join(str(getattr(r, k) or '') for k in by): r for r in results_} @@ -893,14 +886,14 @@ def table(Result, results, diff_results=None, *, else: def recurse(results_, depth_, seen=set(), prefixes=('', '', '', '')): - # rebuild our tables at each layer + # build the children table at each layer + results_ = fold(Result, results_, by=by) table_ = { ','.join(str(getattr(r, k) or '') for k in by): r for r in results_} names_ = list(table_.keys()) - # sort again at each layer, keep in mind the numbers are - # changing as we descend + # sort the children layer names_.sort() if sort: for k, reverse in reversed(sort): @@ -948,8 +941,7 @@ def table(Result, results, diff_results=None, *, prefixes[2+is_last] + "| ", prefixes[2+is_last] + " ")) - # we have enough going on with diffing to make the top layer - # a special case + # the top layer is a bit of a special case for name, line in zip(names, lines[1:-1]): print('%-*s %s' % ( widths[0], line[0][0], diff --git a/scripts/stack.py b/scripts/stack.py index 48f01146..6390721b 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -104,7 +104,8 @@ class StackResult(co.namedtuple('StackResult', [ __slots__ = () def __new__(cls, file='', function='', - frame=0, limit=0, children=set()): + frame=0, limit=0, + children=[]): return super().__new__(cls, file, function, RInt(frame), RInt(limit), children) @@ -113,7 +114,7 @@ class StackResult(co.namedtuple('StackResult', [ return StackResult(self.file, self.function, self.frame + other.frame, max(self.limit, other.limit), - self.children | other.children) + self.children + other.children) def openio(path, mode='r', buffering=-1): @@ -257,22 +258,21 @@ def collect(ci_paths, *, find_limit.cache[source] = frame + limit return frame + limit - def find_children(targets): - children = set() - for target in targets: - if target in callgraph: - t_file, t_function, _, _ = callgraph[target] - children.add((t_file, t_function)) - return children - # build results - results = [] - for source, (s_file, s_function, frame, targets) in callgraph.items(): + results = {} + for source, (s_file, s_function, frame, _) in callgraph.items(): limit = find_limit(source) - children = find_children(targets) - results.append(StackResult(s_file, s_function, frame, limit, children)) + results[source] = StackResult(s_file, s_function, frame, limit, []) - return results + # connect parents to their children, this may create a fully cyclic graph + # in the case of recursion + for source, (_, _, _, targets) in callgraph.items(): + results[source].children.extend( + results[target] + for target in targets + if target in results) + + return list(results.values()) def fold(Result, results, by=None, defines=[]): @@ -477,14 +477,16 @@ def table(Result, results, diff_results=None, *, if hot: depth_ = 2 elif m.isinf(depth_): - def rec_depth(children_, seen=set()): - names_ = { - ','.join(str(getattr(Result(*c), k) or '') - for k in by) - for c in children_} + def rec_depth(results_, seen=set()): + # build the children table at each layer + results_ = fold(Result, results_, by=by) + table_ = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results_} + names_ = list(table_.keys()) return max( - (rec_depth(table[name].children, seen | {name}) + (rec_depth(table_[name].children, seen | {name}) for name in names_ if name not in seen), default=-1) + 1 @@ -507,14 +509,15 @@ def table(Result, results, diff_results=None, *, for i, x in enumerate(lines[0][1:], 1)))) if not summary: - line_table = {n: l for n, l in zip(names, lines[1:-1])} - if hot: - def recurse(children_, depth_, seen=set(), + def recurse(results_, depth_, seen=set(), prefixes=('', '', '', '')): - names_ = {','.join(str(getattr(Result(*c), k) or '') - for k in by) - for c in children_} + # build the children table at each layer + results_ = fold(Result, results_, by=by) + table_ = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results_} + names_ = list(table_.keys()) if not names_: return @@ -523,8 +526,9 @@ def table(Result, results, diff_results=None, *, name = max(names_, key=lambda n: tuple( tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None + # make sure to use the rebuilt table + (getattr(table_[n], k),) + if getattr(table_.get(n), k, None) is not None else () for k in ([k] if k else [ k for k in Result._sort if k in fields]) @@ -533,22 +537,24 @@ def table(Result, results, diff_results=None, *, sort or [], [(None, False)]))) - if name in line_table: - line = line_table[name] - is_last = not table[name].children + r = table_[name] + is_last = not r.children - print('%s%-*s %s' % ( - prefixes[0+is_last], - widths[0] - len(prefixes[0+is_last]), line[0][0], - ' '.join('%*s%-*s' % ( - widths[i], x[0], - notes[i], - ' (%s)' % ', '.join(it.chain( - x[1], ['cycle detected'])) - if i == len(widths)-1 and name in seen - else ' (%s)' % ', '.join(x[1]) if x[1] - else '') - for i, x in enumerate(line[1:], 1)))) + line = table_entry(name, r) + line = [x if isinstance(x, tuple) else (x, []) + for x in line] + print('%s%-*s %s' % ( + prefixes[0+is_last], + widths[0] - len(prefixes[0+is_last]), line[0][0], + ' '.join('%*s%-*s' % ( + widths[i], x[0], + notes[i], + ' (%s)' % ', '.join(it.chain( + x[1], ['cycle detected'])) + if i == len(widths)-1 and name in seen + else ' (%s)' % ', '.join(x[1]) if x[1] + else '') + for i, x in enumerate(line[1:], 1)))) # found a cycle? if name in seen: @@ -557,26 +563,41 @@ def table(Result, results, diff_results=None, *, # recurse? if depth_ > 1: recurse( - table[name].children, + r.children, depth_-1, seen | {name}, prefixes) else: - def recurse(children_, depth_, seen=set(), + def recurse(results_, depth_, seen=set(), prefixes=('', '', '', '')): - # note we're maintaining sort order - names_ = {','.join(str(getattr(Result(*c), k) or '') - for k in by) - for c in children_} - names_ = [n for n in names if n in names_] + # build the children table at each layer + results_ = fold(Result, results_, by=by) + table_ = { + ','.join(str(getattr(r, k) or '') for k in by): r + for r in results_} + names_ = list(table_.keys()) + + # sort the children layer + names_.sort() + if sort: + for k, reverse in reversed(sort): + names_.sort( + key=lambda n: tuple( + (getattr(table_[n], k),) + if getattr(table_.get(n), k, None) is not None + else () + for k in ([k] if k else [ + k for k in Result._sort if k in fields])), + reverse=reverse ^ (not k or k in Result._fields)) for i, name in enumerate(names_): - if name not in line_table: - continue - line = line_table[name] + r = table_[name] is_last = (i == len(names_)-1) + line = table_entry(name, r) + line = [x if isinstance(x, tuple) else (x, []) + for x in line] print('%s%-*s %s' % ( prefixes[0+is_last], widths[0] - len(prefixes[0+is_last]), line[0][0], @@ -597,7 +618,7 @@ def table(Result, results, diff_results=None, *, # recurse? if depth_ > 1: recurse( - table[name].children, + r.children, depth_-1, seen | {name}, (prefixes[2+is_last] + "|-> ", @@ -605,7 +626,7 @@ def table(Result, results, diff_results=None, *, prefixes[2+is_last] + "| ", prefixes[2+is_last] + " ")) - # make the top layer a special case + # the top layer is a bit of a special case for name, line in zip(names, lines[1:-1]): print('%-*s %s' % ( widths[0], line[0][0],