From 48daeed509b6db9db2345f00acecb680a5482572 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 15 May 2025 16:08:04 -0500 Subject: [PATCH] scripts: Fixed rounding-towards-zero issue in si/si2 prefixes This should be floor (rounds towards -inf), not int (rounds towards zero), otherwise sub-integer results get funky: - floor si(0.00001) => 10u - int si(0.00001) => 0.01m - floor si(0.000001) => 1u - int si(0.000001) => m (???) --- scripts/plot.py | 4 ++-- scripts/plotmpl.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/plot.py b/scripts/plot.py index 079f04b1..b2d1e31a 100755 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -101,7 +101,7 @@ def si(x, w=4): # # note we adjust this so that 100K = .1M, which has more info # per character - p = 3*int(mt.log(abs(x)*10, 10**3)) + p = 3*mt.floor(mt.log(abs(x)*10, 10**3)) p = min(18, max(-18, p)) # format with enough digits s = '%.*f' % (w, abs(x) / (10.0**p)) @@ -120,7 +120,7 @@ def si2(x, w=5): # # note we adjust this so that 128Ki = .1Mi, which has more info # per character - p = 10*int(mt.log(abs(x)*10, 2**10)) + p = 10*mt.floor(mt.log(abs(x)*10, 2**10)) p = min(30, max(-30, p)) # format with enough digits s = '%.*f' % (w, abs(x) / (2.0**p)) diff --git a/scripts/plotmpl.py b/scripts/plotmpl.py index 0ac77394..34bb1f54 100755 --- a/scripts/plotmpl.py +++ b/scripts/plotmpl.py @@ -105,7 +105,7 @@ def si(x): if x == 0: return '0' # figure out prefix and scale - p = 3*int(mt.log(abs(x), 10**3)) + p = 3*mt.floor(mt.log(abs(x), 10**3)) p = min(18, max(-18, p)) # format with 3 digits of precision s = '%.3f' % (abs(x) / (10.0**p)) @@ -121,7 +121,7 @@ def si2(x): if x == 0: return '0' # figure out prefix and scale - p = 10*int(mt.log(abs(x), 2**10)) + p = 10*mt.floor(mt.log(abs(x), 2**10)) p = min(30, max(-30, p)) # format with 3 digits of precision s = '%.3f' % (abs(x) / (2.0**p))