From 616b4e1c9e97efa93ee1a0bb7a92a5863ab1565d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 3 Nov 2023 14:30:22 -0500 Subject: [PATCH] Tweaked scripts that consume .csv files to filter defines early With the quantity of data being output by bench.py now, filtering ASAP while parsing CSV files is a valuable optimization. And thanks to how CSV files are structured, we can even avoid ever loading the full contents into RAM. This does end up with use filtering for defines redundantly in a few places, but this is well worth the saved overhead from early filtering. Also tried to clean up the plot.py/plotmpl.py's data folding path, though that may have been wasted effort. --- scripts/code.py | 16 ++++-- scripts/cov.py | 16 ++++-- scripts/data.py | 16 ++++-- scripts/perf.py | 16 ++++-- scripts/perfbd.py | 16 ++++-- scripts/plot.py | 130 ++++++++++++++++++++++---------------------- scripts/plotmpl.py | 132 +++++++++++++++++++++++---------------------- scripts/stack.py | 16 ++++-- scripts/structs.py | 16 ++++-- scripts/summary.py | 112 +++++++++++++++++++------------------- 10 files changed, 274 insertions(+), 212 deletions(-) diff --git a/scripts/code.py b/scripts/code.py index 92f57533..e977a4c6 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -317,18 +317,18 @@ def collect(obj_paths, *, def fold(Result, results, *, by=None, - defines=None, + defines=[], **_): if by is None: by = Result._by - for k in it.chain(by or [], (k for k, _ in defines or [])): + for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k) sys.exit(-1) # filter by matching defines - if defines is not None: + if defines: results_ = [] for r in results: if all(getattr(r, k) in vs for k, vs in defines): @@ -524,7 +524,7 @@ def table(Result, results, diff_results=None, *, def main(obj_paths, *, by=None, fields=None, - defines=None, + defines=[], sort=None, **args): # find sizes @@ -535,6 +535,10 @@ def main(obj_paths, *, with openio(args['use']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('code_'+k in r and r['code_'+k].strip() for k in CodeResult._fields): continue @@ -582,6 +586,10 @@ def main(obj_paths, *, with openio(args['diff']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('code_'+k in r and r['code_'+k].strip() for k in CodeResult._fields): continue diff --git a/scripts/cov.py b/scripts/cov.py index ca235c5b..a29466d3 100755 --- a/scripts/cov.py +++ b/scripts/cov.py @@ -299,18 +299,18 @@ def collect(gcda_paths, *, def fold(Result, results, *, by=None, - defines=None, + defines=[], **_): if by is None: by = Result._by - for k in it.chain(by or [], (k for k, _ in defines or [])): + for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k) sys.exit(-1) # filter by matching defines - if defines is not None: + if defines: results_ = [] for r in results: if all(getattr(r, k) in vs for k, vs in defines): @@ -581,7 +581,7 @@ def annotate(Result, results, *, def main(gcda_paths, *, by=None, fields=None, - defines=None, + defines=[], sort=None, hits=False, **args): @@ -601,6 +601,10 @@ def main(gcda_paths, *, with openio(args['use']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('cov_'+k in r and r['cov_'+k].strip() for k in CovResult._fields): continue @@ -650,6 +654,10 @@ def main(gcda_paths, *, with openio(args['diff']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('cov_'+k in r and r['cov_'+k].strip() for k in CovResult._fields): continue diff --git a/scripts/data.py b/scripts/data.py index 554fd740..1ed3c273 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -317,18 +317,18 @@ def collect(obj_paths, *, def fold(Result, results, *, by=None, - defines=None, + defines=[], **_): if by is None: by = Result._by - for k in it.chain(by or [], (k for k, _ in defines or [])): + for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k) sys.exit(-1) # filter by matching defines - if defines is not None: + if defines: results_ = [] for r in results: if all(getattr(r, k) in vs for k, vs in defines): @@ -524,7 +524,7 @@ def table(Result, results, diff_results=None, *, def main(obj_paths, *, by=None, fields=None, - defines=None, + defines=[], sort=None, **args): # find sizes @@ -535,6 +535,10 @@ def main(obj_paths, *, with openio(args['use']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + try: results.append(DataResult( **{k: r[k] for k in DataResult._by @@ -579,6 +583,10 @@ def main(obj_paths, *, with openio(args['diff']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('data_'+k in r and r['data_'+k].strip() for k in DataResult._fields): continue diff --git a/scripts/perf.py b/scripts/perf.py index 583835b0..f5346dad 100755 --- a/scripts/perf.py +++ b/scripts/perf.py @@ -629,18 +629,18 @@ def collect(perf_paths, *, def fold(Result, results, *, by=None, - defines=None, + defines=[], **_): if by is None: by = Result._by - for k in it.chain(by or [], (k for k, _ in defines or [])): + for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k) sys.exit(-1) # filter by matching defines - if defines is not None: + if defines: results_ = [] for r in results: if all(getattr(r, k) in vs for k, vs in defines): @@ -1037,7 +1037,7 @@ def annotate(Result, results, *, def report(perf_paths, *, by=None, fields=None, - defines=None, + defines=[], sort=None, branches=False, caches=False, @@ -1062,6 +1062,10 @@ def report(perf_paths, *, with openio(args['use']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('perf_'+k in r and r['perf_'+k].strip() for k in PerfResult._fields): continue @@ -1109,6 +1113,10 @@ def report(perf_paths, *, with openio(args['diff']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('perf_'+k in r and r['perf_'+k].strip() for k in PerfResult._fields): continue diff --git a/scripts/perfbd.py b/scripts/perfbd.py index 01b4efe2..d146b564 100755 --- a/scripts/perfbd.py +++ b/scripts/perfbd.py @@ -595,18 +595,18 @@ def collect(obj_path, trace_paths, *, def fold(Result, results, *, by=None, - defines=None, + defines=[], **_): if by is None: by = Result._by - for k in it.chain(by or [], (k for k, _ in defines or [])): + for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k) sys.exit(-1) # filter by matching defines - if defines is not None: + if defines: results_ = [] for r in results: if all(getattr(r, k) in vs for k, vs in defines): @@ -1019,7 +1019,7 @@ def annotate(Result, results, *, def report(obj_path='', trace_paths=[], *, by=None, fields=None, - defines=None, + defines=[], sort=None, **args): # figure out what color should be @@ -1042,6 +1042,10 @@ def report(obj_path='', trace_paths=[], *, with openio(args['use']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('perfbd_'+k in r and r['perfbd_'+k].strip() for k in PerfBdResult._fields): continue @@ -1089,6 +1093,10 @@ def report(obj_path='', trace_paths=[], *, with openio(args['diff']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('perfbd_'+k in r and r['perfbd_'+k].strip() for k in PerfBdResult._fields): continue diff --git a/scripts/plot.py b/scripts/plot.py index 9c41cec5..61d20a26 100755 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -443,7 +443,7 @@ class Plot: return ''.join(row_) -def collect(csv_paths, renames=[]): +def collect(csv_paths, renames=[], defines=[]): # collect results from CSV files results = [] for path in csv_paths: @@ -451,64 +451,33 @@ def collect(csv_paths, renames=[]): with openio(path) as f: reader = csv.DictReader(f, restval='') for r in reader: + # apply any renames + if renames: + # make a copy so renames can overlap + r_ = {} + for new_k, old_k in renames: + if old_k in r: + r_[new_k] = r[old_k] + r.update(r_) + + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + results.append(r) except FileNotFoundError: pass - if renames: - for r in results: - # make a copy so renames can overlap - r_ = {} - for new_k, old_k in renames: - if old_k in r: - r_[new_k] = r[old_k] - r.update(r_) - return results -def dataset(results, x=None, y=None, define=[]): - # organize by 'by', x, and y - dataset = [] - i = 0 - for r in results: - # filter results by matching defines - if not all(k in r and r[k] in vs for k, vs in define): - continue - - # find xs - if x is not None: - if x not in r: - continue - try: - x_ = dat(r[x]) - except ValueError: - continue - else: - x_ = i - i += 1 - - # find ys - if y is not None: - if y not in r: - continue - try: - y_ = dat(r[y]) - except ValueError: - continue - else: - y_ = None - - dataset.append((x_, y_)) - - return dataset - -def datasets(results, by=None, x=None, y=None, define=[]): - # filter results by matching defines - results_ = [] - for r in results: - if all(k in r and r[k] in vs for k, vs in define): - results_.append(r) - results = results_ +def fold(results, by=None, x=None, y=None, defines=[]): + # filter by matching defines + if defines: + results_ = [] + for r in results: + if all(k in r and r[k] in vs for k, vs in defines): + results_.append(r) + results = results_ # if y not specified, try to guess from data if not y: @@ -535,16 +504,46 @@ def datasets(results, by=None, x=None, y=None, define=[]): for ks_ in (ks if by else [()]): for x_ in (x if x else [None]): for y_ in y: + # organize by 'by', x, and y + dataset = [] + i = 0 + for r in results: + # filter by 'by' + if by and not all( + k in r and r[k] == v + for k, v in zip(by, ks_)): + continue + + # find xs + if x_ is not None: + if x_ not in r: + continue + try: + x__ = dat(r[x_]) + except ValueError: + continue + else: + # fallback to enumeration + x__ = i + i += 1 + + # find ys + if y_ is not None: + if y_ not in r: + continue + try: + y__ = dat(r[y_]) + except ValueError: + continue + else: + y__ = None + + dataset.append((x__, y__)) + # hide x/y if there is only one field k_x = x_ if len(x or []) > 1 else '' k_y = y_ if len(y or []) > 1 or (not ks_ and not k_x) else '' - - datasets[ks_ + (k_x, k_y)] = dataset( - results, - x_, - y_, - [(by_, {k_}) for by_, k_ in zip(by, ks_)] - if by else []) + datasets[ks_ + (k_x, k_y)] = dataset return datasets @@ -898,6 +897,11 @@ def main(csv_paths, *, all_by = (by or []) + subplots_get('by', **subplot, subplots=subplots) all_x = (x or []) + subplots_get('x', **subplot, subplots=subplots) all_y = (y or []) + subplots_get('y', **subplot, subplots=subplots) + all_defines = co.defaultdict(lambda: set()) + for k, vs in it.chain(define or [], + subplots_get('define', **subplot, subplots=subplots)): + all_defines[k] |= vs + all_defines = sorted(all_defines.items()) # separate out renames renames = list(it.chain.from_iterable( @@ -990,10 +994,10 @@ def main(csv_paths, *, f.writeln = writeln # first collect results from CSV files - results = collect(csv_paths, renames) + results = collect(csv_paths, renames, all_defines) # then extract the requested datasets - datasets_ = datasets(results, all_by, all_x, all_y, define) + datasets_ = fold(results, all_by, all_x, all_y) # figure out colors/chars here so that subplot defines # don't change them later, that'd be bad @@ -1139,7 +1143,7 @@ def main(csv_paths, *, # data can be constrained by subplot-specific defines, # so re-extract for each plot - subdatasets = datasets(results, all_by, all_x, all_y, define_) + subdatasets = fold(results, all_by, all_x, all_y, define_) # filter by subplot x/y subdatasets = co.OrderedDict([(name, dataset) diff --git a/scripts/plotmpl.py b/scripts/plotmpl.py index 00c7c1e1..3d359825 100755 --- a/scripts/plotmpl.py +++ b/scripts/plotmpl.py @@ -189,7 +189,7 @@ def dat(x): # else give up raise ValueError("invalid dat %r" % x) -def collect(csv_paths, renames=[]): +def collect(csv_paths, renames=[], defines=[]): # collect results from CSV files results = [] for path in csv_paths: @@ -197,64 +197,33 @@ def collect(csv_paths, renames=[]): with openio(path) as f: reader = csv.DictReader(f, restval='') for r in reader: + # apply any renames + if renames: + # make a copy so renames can overlap + r_ = {} + for new_k, old_k in renames: + if old_k in r: + r_[new_k] = r[old_k] + r.update(r_) + + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + results.append(r) except FileNotFoundError: pass - if renames: - for r in results: - # make a copy so renames can overlap - r_ = {} - for new_k, old_k in renames: - if old_k in r: - r_[new_k] = r[old_k] - r.update(r_) - return results -def dataset(results, x=None, y=None, define=[]): - # organize by 'by', x, and y - dataset = [] - i = 0 - for r in results: - # filter results by matching defines - if not all(k in r and r[k] in vs for k, vs in define): - continue - - # find xs - if x is not None: - if x not in r: - continue - try: - x_ = dat(r[x]) - except ValueError: - continue - else: - x_ = i - i += 1 - - # find ys - if y is not None: - if y not in r: - continue - try: - y_ = dat(r[y]) - except ValueError: - continue - else: - y_ = None - - dataset.append((x_, y_)) - - return dataset - -def datasets(results, by=None, x=None, y=None, define=[]): - # filter results by matching defines - results_ = [] - for r in results: - if all(k in r and r[k] in vs for k, vs in define): - results_.append(r) - results = results_ +def fold(results, by=None, x=None, y=None, defines=[]): + # filter by matching defines + if defines: + results_ = [] + for r in results: + if all(k in r and r[k] in vs for k, vs in defines): + results_.append(r) + results = results_ # if y not specified, try to guess from data if not y: @@ -281,16 +250,46 @@ def datasets(results, by=None, x=None, y=None, define=[]): for ks_ in (ks if by else [()]): for x_ in (x if x else [None]): for y_ in y: + # organize by 'by', x, and y + dataset = [] + i = 0 + for r in results: + # filter by 'by' + if by and not all( + k in r and r[k] == v + for k, v in zip(by, ks_)): + continue + + # find xs + if x_ is not None: + if x_ not in r: + continue + try: + x__ = dat(r[x_]) + except ValueError: + continue + else: + # fallback to enumeration + x__ = i + i += 1 + + # find ys + if y_ is not None: + if y_ not in r: + continue + try: + y__ = dat(r[y_]) + except ValueError: + continue + else: + y__ = None + + dataset.append((x__, y__)) + # hide x/y if there is only one field k_x = x_ if len(x or []) > 1 else '' k_y = y_ if len(y or []) > 1 or (not ks_ and not k_x) else '' - - datasets[ks_ + (k_x, k_y)] = dataset( - results, - x_, - y_, - [(by_, {k_}) for by_, k_ in zip(by, ks_)] - if by else []) + datasets[ks_ + (k_x, k_y)] = dataset return datasets @@ -730,7 +729,7 @@ def main(csv_paths, output, *, # become a mess... subplots += subplot.pop('subplots', []) - # allow any subplots to contribute to by/x/y + # allow any subplots to contribute to by/x/y/defines def subplots_get(k, *, subplots=[], **args): v = args.get(k, []).copy() for _, subargs in subplots: @@ -740,6 +739,11 @@ def main(csv_paths, output, *, all_by = (by or []) + subplots_get('by', **subplot, subplots=subplots) all_x = (x or []) + subplots_get('x', **subplot, subplots=subplots) all_y = (y or []) + subplots_get('y', **subplot, subplots=subplots) + all_defines = co.defaultdict(lambda: set()) + for k, vs in it.chain(define or [], + subplots_get('define', **subplot, subplots=subplots)): + all_defines[k] |= vs + all_defines = sorted(all_defines.items()) # separate out renames renames = list(it.chain.from_iterable( @@ -750,10 +754,10 @@ def main(csv_paths, output, *, all_y = [k for k, _ in all_y] # first collect results from CSV files - results = collect(csv_paths, renames) + results = collect(csv_paths, renames, all_defines) # then extract the requested datasets - datasets_ = datasets(results, all_by, all_x, all_y, define) + datasets_ = fold(results, all_by, all_x, all_y) # figure out formats/colors/labels here so that subplot defines # don't change them later, that'd be bad @@ -830,7 +834,7 @@ def main(csv_paths, output, *, # data can be constrained by subplot-specific defines, # so re-extract for each plot - subdatasets = datasets(results, all_by, all_x, all_y, define_) + subdatasets = fold(results, all_by, all_x, all_y, define_) # filter by subplot x/y subdatasets = co.OrderedDict([(name, dataset) diff --git a/scripts/stack.py b/scripts/stack.py index eae9885e..c14f1c3f 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -275,18 +275,18 @@ def collect(ci_paths, *, def fold(Result, results, *, by=None, - defines=None, + defines=[], **_): if by is None: by = Result._by - for k in it.chain(by or [], (k for k, _ in defines or [])): + for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k) sys.exit(-1) # filter by matching defines - if defines is not None: + if defines: results_ = [] for r in results: if all(getattr(r, k) in vs for k, vs in defines): @@ -548,7 +548,7 @@ def table(Result, results, diff_results=None, *, def main(ci_paths, by=None, fields=None, - defines=None, + defines=[], sort=None, **args): if args.get('depth') is None: @@ -564,6 +564,10 @@ def main(ci_paths, with openio(args['use']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('stack_'+k in r and r['stack_'+k].strip() for k in StackResult._fields): continue @@ -611,6 +615,10 @@ def main(ci_paths, with openio(args['diff']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('stack_'+k in r and r['stack_'+k].strip() for k in StackResult._fields): continue diff --git a/scripts/structs.py b/scripts/structs.py index d62507d2..4784e8c3 100755 --- a/scripts/structs.py +++ b/scripts/structs.py @@ -266,18 +266,18 @@ def collect(obj_paths, *, def fold(Result, results, *, by=None, - defines=None, + defines=[], **_): if by is None: by = Result._by - for k in it.chain(by or [], (k for k, _ in defines or [])): + for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k) sys.exit(-1) # filter by matching defines - if defines is not None: + if defines: results_ = [] for r in results: if all(getattr(r, k) in vs for k, vs in defines): @@ -473,7 +473,7 @@ def table(Result, results, diff_results=None, *, def main(obj_paths, *, by=None, fields=None, - defines=None, + defines=[], sort=None, **args): # find sizes @@ -484,6 +484,10 @@ def main(obj_paths, *, with openio(args['use']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('struct_'+k in r and r['struct_'+k].strip() for k in StructResult._fields): continue @@ -533,6 +537,10 @@ def main(obj_paths, *, with openio(args['diff']) as f: reader = csv.DictReader(f, restval='') for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + if not any('struct_'+k in r and r['struct_'+k].strip() for k in StructResult._fields): continue diff --git a/scripts/summary.py b/scripts/summary.py index 1d5ff38e..b0dbedf4 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -239,6 +239,43 @@ TYPES = co.OrderedDict([ ]) +def openio(path, mode='r', buffering=-1): + # allow '-' for stdin/stdout + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) + else: + return open(path, mode, buffering) + +def collect(csv_paths, renames=[], defines=[]): + # collect results from CSV files + results = [] + for path in csv_paths: + try: + with openio(path) as f: + reader = csv.DictReader(f, restval='') + for r in reader: + # apply any renames + if renames: + # make a copy so renames can overlap + r_ = {} + for new_k, old_k in renames: + if old_k in r: + r_[new_k] = r[old_k] + r.update(r_) + + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + + results.append(r) + except FileNotFoundError: + pass + + return results + def infer(results, *, by=None, fields=None, @@ -346,18 +383,18 @@ def infer(results, *, def fold(Result, results, *, by=None, - defines=None, + defines=[], **_): if by is None: by = Result._by - for k in it.chain(by or [], (k for k, _ in defines or [])): + for k in it.chain(by or [], (k for k, _ in defines)): if k not in Result._by and k not in Result._fields: print("error: could not find field %r?" % k) sys.exit(-1) # filter by matching defines - if defines is not None: + if defines: results_ = [] for r in results: if all(getattr(r, k) in vs for k, vs in defines): @@ -550,20 +587,10 @@ def table(Result, results, diff_results=None, *, line[-1])) -def openio(path, mode='r', buffering=-1): - # allow '-' for stdin/stdout - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) - else: - return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) - else: - return open(path, mode, buffering) - def main(csv_paths, *, by=None, fields=None, - defines=None, + defines=[], sort=None, **args): # separate out renames @@ -608,24 +635,7 @@ def main(csv_paths, *, ops.update(ops_) # find CSV files - results = [] - for path in csv_paths: - try: - with openio(path) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # rename fields? - if renames: - # make a copy so renames can overlap - r_ = {} - for new_k, old_k in renames: - if old_k in r: - r_[new_k] = r[old_k] - r.update(r_) - - results.append(r) - except FileNotFoundError: - pass + results = collect(csv_paths, renames=renames, defines=defines) # homogenize Result = infer(results, @@ -672,31 +682,19 @@ def main(csv_paths, *, # find previous results? if args.get('diff'): - diff_results = [] - try: - with openio(args['diff']) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # rename fields? - if renames: - # make a copy so renames can overlap - r_ = {} - for new_k, old_k in renames: - if old_k in r: - r_[new_k] = r[old_k] - r.update(r_) - - if not any(k in r and r[k].strip() - for k in Result._fields): - continue - try: - diff_results.append(Result(**{ - k: r[k] for k in Result._by + Result._fields - if k in r and r[k].strip()})) - except TypeError: - pass - except FileNotFoundError: - pass + diff_results = collect([args['diff']], renames=renames, defines=defines) + diff_results_ = [] + for r in diff_results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + diff_results_.append(Result(**{ + k: r[k] for k in Result._by + Result._fields + if k in r and r[k].strip()})) + except TypeError: + pass + diff_results = diff_results_ # fold diff_results = fold(Result, diff_results, by=by, defines=defines)