forked from Imagelibrary/littlefs
15593ccc49
I was originally avoiding naming these orphans, as they're _technically_ not orphans. They do exist in the mtree. But the name orphan just describes this types purpose too well. This does lead to some confusing terms, such as the fact that orphan files can be non-orphaned if there are any in-device references. But I think this makes sense? - LFSR_TAG_SCRATCH -> LFSR_TAG_ORPHAN - LFSR_F_UNCREAT -> LFSR_F_ORPHAN - test_fscratch.toml -> test_forphan.toml
1839 lines
58 KiB
Python
Executable File
1839 lines
58 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import bisect
|
|
import collections as co
|
|
import functools as ft
|
|
import itertools as it
|
|
import math as m
|
|
import os
|
|
import shutil
|
|
import struct
|
|
|
|
|
|
TAG_NULL = 0x0000
|
|
TAG_CONFIG = 0x0000
|
|
TAG_MAGIC = 0x0003
|
|
TAG_VERSION = 0x0004
|
|
TAG_OCOMPATFLAGS = 0x0005
|
|
TAG_RCOMPATFLAGS = 0x0006
|
|
TAG_WCOMPATFLAGS = 0x0007
|
|
TAG_BLOCKSIZE = 0x0008
|
|
TAG_BLOCKCOUNT = 0x0009
|
|
TAG_NAMELIMIT = 0x000a
|
|
TAG_SIZELIMIT = 0x000b
|
|
TAG_GDELTA = 0x0100
|
|
TAG_GRMDELTA = 0x0100
|
|
TAG_NAME = 0x0200
|
|
TAG_REG = 0x0201
|
|
TAG_DIR = 0x0202
|
|
TAG_ORPHAN = 0x0203
|
|
TAG_BOOKMARK = 0x0204
|
|
TAG_STRUCT = 0x0300
|
|
TAG_DATA = 0x0300
|
|
TAG_BLOCK = 0x0304
|
|
TAG_BSHRUB = 0x0308
|
|
TAG_BTREE = 0x030c
|
|
TAG_DID = 0x0310
|
|
TAG_BECKSUM = 0x0314
|
|
TAG_BRANCH = 0x031c
|
|
TAG_MROOT = 0x0321
|
|
TAG_MDIR = 0x0325
|
|
TAG_MTREE = 0x032c
|
|
TAG_UATTR = 0x0400
|
|
TAG_SATTR = 0x0600
|
|
TAG_SHRUB = 0x1000
|
|
TAG_CKSUM = 0x3000
|
|
TAG_ECKSUM = 0x3100
|
|
TAG_ALT = 0x4000
|
|
TAG_GT = 0x2000
|
|
TAG_R = 0x1000
|
|
|
|
|
|
CHARS = 'mbd-'
|
|
COLORS = ['33', '34', '32', '90']
|
|
|
|
CHARS_DOTS = " .':"
|
|
CHARS_BRAILLE = (
|
|
'⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴'
|
|
'⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶'
|
|
'⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼'
|
|
'⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾'
|
|
'⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵'
|
|
'⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷'
|
|
'⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽'
|
|
'⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿')
|
|
|
|
|
|
# some ways of block geometry representations
|
|
# 512 -> 512
|
|
# 512x16 -> (512, 16)
|
|
# 0x200x10 -> (512, 16)
|
|
def bdgeom(s):
|
|
s = s.strip()
|
|
b = 10
|
|
if s.startswith('0x') or s.startswith('0X'):
|
|
s = s[2:]
|
|
b = 16
|
|
elif s.startswith('0o') or s.startswith('0O'):
|
|
s = s[2:]
|
|
b = 8
|
|
elif s.startswith('0b') or s.startswith('0B'):
|
|
s = s[2:]
|
|
b = 2
|
|
|
|
if 'x' in s:
|
|
s, s_ = s.split('x', 1)
|
|
return (int(s, b), int(s_, b))
|
|
else:
|
|
return int(s, b)
|
|
|
|
# parse some rbyd addr encodings
|
|
# 0xa -> [0xa]
|
|
# 0xa.c -> [(0xa, 0xc)]
|
|
# 0x{a,b} -> [0xa, 0xb]
|
|
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
|
|
def rbydaddr(s):
|
|
s = s.strip()
|
|
b = 10
|
|
if s.startswith('0x') or s.startswith('0X'):
|
|
s = s[2:]
|
|
b = 16
|
|
elif s.startswith('0o') or s.startswith('0O'):
|
|
s = s[2:]
|
|
b = 8
|
|
elif s.startswith('0b') or s.startswith('0B'):
|
|
s = s[2:]
|
|
b = 2
|
|
|
|
trunk = None
|
|
if '.' in s:
|
|
s, s_ = s.split('.', 1)
|
|
trunk = int(s_, b)
|
|
|
|
if s.startswith('{') and '}' in s:
|
|
ss = s[1:s.find('}')].split(',')
|
|
else:
|
|
ss = [s]
|
|
|
|
addr = []
|
|
for s in ss:
|
|
if trunk is not None:
|
|
addr.append((int(s, b), trunk))
|
|
else:
|
|
addr.append(int(s, b))
|
|
|
|
return addr
|
|
|
|
def crc32c(data, crc=0):
|
|
crc ^= 0xffffffff
|
|
for b in data:
|
|
crc ^= b
|
|
for j in range(8):
|
|
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
|
|
return 0xffffffff ^ crc
|
|
|
|
def popc(x):
|
|
return bin(x).count('1')
|
|
|
|
def fromle32(data):
|
|
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
|
|
|
|
def fromleb128(data):
|
|
word = 0
|
|
for i, b in enumerate(data):
|
|
word |= ((b & 0x7f) << 7*i)
|
|
word &= 0xffffffff
|
|
if not b & 0x80:
|
|
return word, i+1
|
|
return word, len(data)
|
|
|
|
def fromtag(data):
|
|
data = data.ljust(4, b'\0')
|
|
tag = (data[0] << 8) | data[1]
|
|
weight, d = fromleb128(data[2:])
|
|
size, d_ = fromleb128(data[2+d:])
|
|
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
|
|
|
|
def frommdir(data):
|
|
blocks = []
|
|
d = 0
|
|
while d < len(data):
|
|
block, d_ = fromleb128(data[d:])
|
|
blocks.append(block)
|
|
d += d_
|
|
return blocks
|
|
|
|
def frombranch(data):
|
|
d = 0
|
|
block, d_ = fromleb128(data[d:]); d += d_
|
|
trunk, d_ = fromleb128(data[d:]); d += d_
|
|
cksum = fromle32(data[d:]); d += 4
|
|
return block, trunk, cksum
|
|
|
|
def frombtree(data):
|
|
d = 0
|
|
w, d_ = fromleb128(data[d:]); d += d_
|
|
block, trunk, cksum = frombranch(data[d:])
|
|
return w, block, trunk, cksum
|
|
|
|
def frombptr(data):
|
|
d = 0
|
|
size, d_ = fromleb128(data[d:]); d += d_
|
|
block, d_ = fromleb128(data[d:]); d += d_
|
|
off, d_ = fromleb128(data[d:]); d += d_
|
|
return size, block, off
|
|
|
|
|
|
# space filling Hilbert-curve
|
|
#
|
|
# note we memoize the last curve since this is a bit expensive
|
|
#
|
|
@ft.lru_cache(1)
|
|
def hilbert_curve(width, height):
|
|
# based on generalized Hilbert curves:
|
|
# https://github.com/jakubcerveny/gilbert
|
|
#
|
|
def hilbert_(x, y, a_x, a_y, b_x, b_y):
|
|
w = abs(a_x+a_y)
|
|
h = abs(b_x+b_y)
|
|
a_dx = -1 if a_x < 0 else +1 if a_x > 0 else 0
|
|
a_dy = -1 if a_y < 0 else +1 if a_y > 0 else 0
|
|
b_dx = -1 if b_x < 0 else +1 if b_x > 0 else 0
|
|
b_dy = -1 if b_y < 0 else +1 if b_y > 0 else 0
|
|
|
|
# trivial row
|
|
if h == 1:
|
|
for _ in range(w):
|
|
yield (x,y)
|
|
x, y = x+a_dx, y+a_dy
|
|
return
|
|
|
|
# trivial column
|
|
if w == 1:
|
|
for _ in range(h):
|
|
yield (x,y)
|
|
x, y = x+b_dx, y+b_dy
|
|
return
|
|
|
|
a_x_, a_y_ = a_x//2, a_y//2
|
|
b_x_, b_y_ = b_x//2, b_y//2
|
|
w_ = abs(a_x_+a_y_)
|
|
h_ = abs(b_x_+b_y_)
|
|
|
|
if 2*w > 3*h:
|
|
# prefer even steps
|
|
if w_ % 2 != 0 and w > 2:
|
|
a_x_, a_y_ = a_x_+a_dx, a_y_+a_dy
|
|
|
|
# split in two
|
|
yield from hilbert_(x, y, a_x_, a_y_, b_x, b_y)
|
|
yield from hilbert_(x+a_x_, y+a_y_, a_x-a_x_, a_y-a_y_, b_x, b_y)
|
|
else:
|
|
# prefer even steps
|
|
if h_ % 2 != 0 and h > 2:
|
|
b_x_, b_y_ = b_x_+b_dx, b_y_+b_dy
|
|
|
|
# split in three
|
|
yield from hilbert_(x, y, b_x_, b_y_, a_x_, a_y_)
|
|
yield from hilbert_(x+b_x_, y+b_y_, a_x, a_y, b_x-b_x_, b_y-b_y_)
|
|
yield from hilbert_(
|
|
x+(a_x-a_dx)+(b_x_-b_dx), y+(a_y-a_dy)+(b_y_-b_dy),
|
|
-b_x_, -b_y_, -(a_x-a_x_), -(a_y-a_y_))
|
|
|
|
if width >= height:
|
|
curve = hilbert_(0, 0, +width, 0, 0, +height)
|
|
else:
|
|
curve = hilbert_(0, 0, 0, +height, +width, 0)
|
|
|
|
return list(curve)
|
|
|
|
# space filling Z-curve/Lebesgue-curve
|
|
#
|
|
# note we memoize the last curve since this is a bit expensive
|
|
#
|
|
@ft.lru_cache(1)
|
|
def lebesgue_curve(width, height):
|
|
# we create a truncated Z-curve by simply filtering out the points
|
|
# that are outside our region
|
|
curve = []
|
|
for i in range(2**(2*m.ceil(m.log2(max(width, height))))):
|
|
# we just operate on binary strings here because it's easier
|
|
b = '{:0{}b}'.format(i, 2*m.ceil(m.log2(i+1)/2))
|
|
x = int(b[1::2], 2) if b[1::2] else 0
|
|
y = int(b[0::2], 2) if b[0::2] else 0
|
|
if x < width and y < height:
|
|
curve.append((x, y))
|
|
|
|
return curve
|
|
|
|
|
|
# the rendering code is copied from tracebd.py, which is why it may look a
|
|
# little funny
|
|
#
|
|
# each block can be in one of 3 states: mdir, btree, or raw data, we keep track
|
|
# of these at the pixel-level via a bitmask
|
|
#
|
|
class Pixel(int):
|
|
__slots__ = ()
|
|
def __new__(cls, state=0, *,
|
|
mdir=False,
|
|
btree=False,
|
|
data=False):
|
|
return super().__new__(cls,
|
|
state
|
|
| (1 if mdir else 0)
|
|
| (2 if btree else 0)
|
|
| (4 if data else 0))
|
|
|
|
@property
|
|
def is_mdir(self):
|
|
return (self & 1) != 0
|
|
|
|
@property
|
|
def is_btree(self):
|
|
return (self & 2) != 0
|
|
|
|
@property
|
|
def is_data(self):
|
|
return (self & 4) != 0
|
|
|
|
def mdir(self):
|
|
return Pixel(int(self) | 1)
|
|
|
|
def btree(self):
|
|
return Pixel(int(self) | 2)
|
|
|
|
def data(self):
|
|
return Pixel(int(self) | 4)
|
|
|
|
def clear(self):
|
|
return Pixel(0)
|
|
|
|
def __or__(self, other):
|
|
return Pixel(int(self) | int(other))
|
|
|
|
def draw(self, char=None, *,
|
|
mdirs=True,
|
|
btrees=True,
|
|
datas=True,
|
|
color=True,
|
|
dots=False,
|
|
braille=False,
|
|
chars=None,
|
|
colors=None,
|
|
**_):
|
|
# fallback to default chars/colors
|
|
if chars is None:
|
|
chars = CHARS
|
|
if len(chars) < len(CHARS):
|
|
chars = chars + CHARS[len(chars):]
|
|
|
|
if colors is None:
|
|
colors = COLORS
|
|
if len(colors) < len(COLORS):
|
|
colors = colors + COLORS[len(colors):]
|
|
|
|
# compute char/color
|
|
c = chars[3]
|
|
f = [colors[3]]
|
|
|
|
if mdirs and self.is_mdir:
|
|
c = chars[0]
|
|
f.append(colors[0])
|
|
elif btrees and self.is_btree:
|
|
c = chars[1]
|
|
f.append(colors[1])
|
|
elif datas and self.is_data:
|
|
c = chars[2]
|
|
f.append(colors[2])
|
|
|
|
# override char?
|
|
if char:
|
|
c = char
|
|
|
|
# apply colors
|
|
if f and color:
|
|
c = '%s%s\x1b[m' % (
|
|
''.join('\x1b[%sm' % f_ for f_ in f),
|
|
c)
|
|
|
|
return c
|
|
|
|
|
|
class Bmap:
|
|
def __init__(self, *,
|
|
block_size=1,
|
|
block_count=1,
|
|
block_window=None,
|
|
off_window=None,
|
|
width=None,
|
|
height=1,
|
|
pixels=None):
|
|
# default width to block_window or block_size
|
|
if width is None:
|
|
if block_window is not None:
|
|
width = len(block_window)
|
|
else:
|
|
width = block_count
|
|
|
|
# allocate pixels if not provided
|
|
if pixels is None:
|
|
pixels = [Pixel() for _ in range(width*height)]
|
|
|
|
self.pixels = pixels
|
|
self.block_size = block_size
|
|
self.block_count = block_count
|
|
self.block_window = block_window
|
|
self.off_window = off_window
|
|
self.width = width
|
|
self.height = height
|
|
|
|
@property
|
|
def _block_window(self):
|
|
if self.block_window is None:
|
|
return range(0, self.block_count)
|
|
else:
|
|
return self.block_window
|
|
|
|
@property
|
|
def _off_window(self):
|
|
if self.off_window is None:
|
|
return range(0, self.block_size)
|
|
else:
|
|
return self.off_window
|
|
|
|
@property
|
|
def _window(self):
|
|
return len(self._off_window)*len(self._block_window)
|
|
|
|
def _op(self, f, block=None, off=None, size=None):
|
|
if block is None:
|
|
range_ = range(len(self.pixels))
|
|
else:
|
|
if off is None:
|
|
off, size = 0, self.block_size
|
|
elif size is None:
|
|
off, size = 0, off
|
|
|
|
# map into our window
|
|
if block not in self._block_window:
|
|
return
|
|
block -= self._block_window.start
|
|
|
|
size = (max(self._off_window.start,
|
|
min(self._off_window.stop, off+size))
|
|
- max(self._off_window.start,
|
|
min(self._off_window.stop, off)))
|
|
off = (max(self._off_window.start,
|
|
min(self._off_window.stop, off))
|
|
- self._off_window.start)
|
|
if size == 0:
|
|
return
|
|
|
|
# map to our block space
|
|
range_ = range(
|
|
block*len(self._off_window) + off,
|
|
block*len(self._off_window) + off+size)
|
|
range_ = range(
|
|
(range_.start*len(self.pixels)) // self._window,
|
|
(range_.stop*len(self.pixels)) // self._window)
|
|
range_ = range(
|
|
range_.start,
|
|
max(range_.stop, range_.start+1))
|
|
|
|
# apply the op
|
|
for i in range_:
|
|
self.pixels[i] = f(self.pixels[i])
|
|
|
|
def mdir(self, block=None, off=None, size=None):
|
|
self._op(Pixel.mdir, block, off, size)
|
|
|
|
def btree(self, block=None, off=None, size=None):
|
|
self._op(Pixel.btree, block, off, size)
|
|
|
|
def data(self, block=None, off=None, size=None):
|
|
self._op(Pixel.data, block, off, size)
|
|
|
|
def clear(self, block=None, off=None, size=None):
|
|
self._op(Pixel.clear, block, off, size)
|
|
|
|
def resize(self, *,
|
|
block_size=None,
|
|
block_count=None,
|
|
width=None,
|
|
height=None):
|
|
block_size = (block_size if block_size is not None
|
|
else self.block_size)
|
|
block_count = (block_count if block_count is not None
|
|
else self.block_count)
|
|
width = width if width is not None else self.width
|
|
height = height if height is not None else self.height
|
|
|
|
if (block_size == self.block_size
|
|
and block_count == self.block_count
|
|
and width == self.width
|
|
and height == self.height):
|
|
return
|
|
|
|
# transform our pixels
|
|
self.block_size = block_size
|
|
self.block_count = block_count
|
|
|
|
pixels = []
|
|
for x in range(width*height):
|
|
# map into our old bd space
|
|
range_ = range(
|
|
(x*self._window) // (width*height),
|
|
((x+1)*self._window) // (width*height))
|
|
range_ = range(
|
|
range_.start,
|
|
max(range_.stop, range_.start+1))
|
|
|
|
# aggregate state
|
|
pixels.append(ft.reduce(
|
|
Pixel.__or__,
|
|
self.pixels[range_.start:range_.stop],
|
|
Pixel()))
|
|
|
|
self.width = width
|
|
self.height = height
|
|
self.pixels = pixels
|
|
|
|
def draw(self, row, *,
|
|
mdirs=False,
|
|
btrees=False,
|
|
datas=False,
|
|
hilbert=False,
|
|
lebesgue=False,
|
|
dots=False,
|
|
braille=False,
|
|
**args):
|
|
# fold via a curve?
|
|
if hilbert:
|
|
grid = [None]*(self.width*self.height)
|
|
for (x,y), p in zip(
|
|
hilbert_curve(self.width, self.height),
|
|
self.pixels):
|
|
grid[x + y*self.width] = p
|
|
elif lebesgue:
|
|
grid = [None]*(self.width*self.height)
|
|
for (x,y), p in zip(
|
|
lebesgue_curve(self.width, self.height),
|
|
self.pixels):
|
|
grid[x + y*self.width] = p
|
|
else:
|
|
grid = self.pixels
|
|
|
|
line = []
|
|
if braille:
|
|
# encode into a byte
|
|
for x in range(0, self.width, 2):
|
|
byte_p = 0
|
|
best_p = Pixel()
|
|
for i in range(2*4):
|
|
p = grid[x+(2-1-(i%2)) + ((row*4)+(4-1-(i//2)))*self.width]
|
|
best_p |= p
|
|
if ((mdirs and p.is_mdir)
|
|
or (btrees and p.is_btree)
|
|
or (datas and p.is_data)):
|
|
byte_p |= 1 << i
|
|
|
|
line.append(best_p.draw(
|
|
CHARS_BRAILLE[byte_p],
|
|
braille=True,
|
|
mdirs=mdirs,
|
|
btrees=btrees,
|
|
datas=datas,
|
|
**args))
|
|
elif dots:
|
|
# encode into a byte
|
|
for x in range(self.width):
|
|
byte_p = 0
|
|
best_p = Pixel()
|
|
for i in range(2):
|
|
p = grid[x + ((row*2)+(2-1-i))*self.width]
|
|
best_p |= p
|
|
if ((mdirs and p.is_mdir)
|
|
or (btrees and p.is_btree)
|
|
or (datas and p.is_data)):
|
|
byte_p |= 1 << i
|
|
|
|
line.append(best_p.draw(
|
|
CHARS_DOTS[byte_p],
|
|
dots=True,
|
|
mdirs=mdirs,
|
|
btrees=btrees,
|
|
datas=datas,
|
|
**args))
|
|
else:
|
|
for x in range(self.width):
|
|
line.append(grid[x + row*self.width].draw(
|
|
mdirs=mdirs,
|
|
btrees=btrees,
|
|
datas=datas,
|
|
**args))
|
|
|
|
return ''.join(line)
|
|
|
|
|
|
# our core rbyd type
|
|
class Rbyd:
|
|
def __init__(self, block, data, rev, eoff, trunk, weight):
|
|
self.block = block
|
|
self.data = data
|
|
self.rev = rev
|
|
self.eoff = eoff
|
|
self.trunk = trunk
|
|
self.weight = weight
|
|
self.redund_blocks = []
|
|
|
|
@property
|
|
def blocks(self):
|
|
return (self.block, *self.redund_blocks)
|
|
|
|
def addr(self):
|
|
if not self.redund_blocks:
|
|
return '0x%x.%x' % (self.block, self.trunk)
|
|
else:
|
|
return '0x{%x,%s}.%x' % (
|
|
self.block,
|
|
','.join('%x' % block for block in self.redund_blocks),
|
|
self.trunk)
|
|
|
|
@classmethod
|
|
def fetch(cls, f, block_size, blocks, trunk=None):
|
|
if isinstance(blocks, int):
|
|
blocks = [blocks]
|
|
|
|
if len(blocks) > 1:
|
|
# fetch all blocks
|
|
rbyds = [cls.fetch(f, block_size, block, trunk) for block in blocks]
|
|
# determine most recent revision
|
|
i = 0
|
|
for i_, rbyd in enumerate(rbyds):
|
|
# compare with sequence arithmetic
|
|
if rbyd and (
|
|
not rbyds[i]
|
|
or not ((rbyd.rev - rbyds[i].rev) & 0x80000000)
|
|
or (rbyd.rev == rbyds[i].rev
|
|
and rbyd.trunk > rbyds[i].trunk)):
|
|
i = i_
|
|
# keep track of the other blocks
|
|
rbyd = rbyds[i]
|
|
rbyd.redund_blocks = [rbyds[(i+1+j) % len(rbyds)].block
|
|
for j in range(len(rbyds)-1)]
|
|
return rbyd
|
|
else:
|
|
# block may encode a trunk
|
|
block = blocks[0]
|
|
if isinstance(block, tuple):
|
|
if trunk is None:
|
|
trunk = block[1]
|
|
block = block[0]
|
|
|
|
# seek to the block
|
|
f.seek(block * block_size)
|
|
data = f.read(block_size)
|
|
|
|
# fetch the rbyd
|
|
rev = fromle32(data[0:4])
|
|
cksum = 0
|
|
cksum_ = crc32c(data[0:4])
|
|
eoff = 0
|
|
j_ = 4
|
|
trunk_ = 0
|
|
trunk__ = 0
|
|
trunk___ = 0
|
|
weight = 0
|
|
weight_ = 0
|
|
weight__ = 0
|
|
wastrunk = False
|
|
trunkeoff = None
|
|
while j_ < len(data) and (not trunk or eoff <= trunk):
|
|
v, tag, w, size, d = fromtag(data[j_:])
|
|
if v != (popc(cksum_) & 1):
|
|
break
|
|
cksum_ = crc32c(data[j_:j_+d], cksum_)
|
|
j_ += d
|
|
if not tag & TAG_ALT and j_ + size > len(data):
|
|
break
|
|
|
|
# take care of cksums
|
|
if not tag & TAG_ALT:
|
|
if (tag & 0xff00) != TAG_CKSUM:
|
|
cksum_ = crc32c(data[j_:j_+size], cksum_)
|
|
# found a cksum?
|
|
else:
|
|
cksum__ = fromle32(data[j_:j_+4])
|
|
if cksum_ != cksum__:
|
|
break
|
|
# commit what we have
|
|
eoff = trunkeoff if trunkeoff else j_ + size
|
|
cksum = cksum_
|
|
trunk_ = trunk__
|
|
weight = weight_
|
|
|
|
# evaluate trunks
|
|
if (tag & 0xf000) != TAG_CKSUM and (
|
|
not trunk or trunk >= j_-d or wastrunk):
|
|
# new trunk?
|
|
if not wastrunk:
|
|
wastrunk = True
|
|
trunk___ = j_-d
|
|
weight__ = 0
|
|
|
|
# keep track of weight
|
|
weight__ += w
|
|
|
|
# end of trunk?
|
|
if not tag & TAG_ALT:
|
|
wastrunk = False
|
|
# update trunk/weight unless we found a shrub or an
|
|
# explicit trunk (which may be a shrub) is requested
|
|
if not tag & TAG_SHRUB or trunk:
|
|
trunk__ = trunk___
|
|
weight_ = weight__
|
|
# keep track of eoff for best matching trunk
|
|
if trunk and j_ + size > trunk:
|
|
trunkeoff = j_ + size
|
|
cksum = cksum_
|
|
trunk_ = trunk__
|
|
weight = weight_
|
|
|
|
if not tag & TAG_ALT:
|
|
j_ += size
|
|
|
|
return cls(block, data, rev, eoff, trunk_, weight)
|
|
|
|
def lookup(self, rid, tag):
|
|
if not self:
|
|
return True, 0, -1, 0, 0, 0, b'', []
|
|
|
|
lower = 0
|
|
upper = self.weight
|
|
path = []
|
|
|
|
# descend down tree
|
|
j = self.trunk
|
|
while True:
|
|
_, alt, weight_, jump, d = fromtag(self.data[j:])
|
|
|
|
# found an alt?
|
|
if alt & TAG_ALT:
|
|
# follow?
|
|
if ((rid, tag & 0xfff) > (upper-weight_-1, alt & 0xfff)
|
|
if alt & TAG_GT
|
|
else ((rid, tag & 0xfff)
|
|
<= (lower+weight_-1, alt & 0xfff))):
|
|
lower += upper-lower-weight_ if alt & TAG_GT else 0
|
|
upper -= upper-lower-weight_ if not alt & TAG_GT else 0
|
|
j = j - jump
|
|
|
|
# figure out which color
|
|
if alt & TAG_R:
|
|
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:])
|
|
if nalt & TAG_R:
|
|
path.append((j+jump, j, True, 'y'))
|
|
else:
|
|
path.append((j+jump, j, True, 'r'))
|
|
else:
|
|
path.append((j+jump, j, True, 'b'))
|
|
|
|
# stay on path
|
|
else:
|
|
lower += weight_ if not alt & TAG_GT else 0
|
|
upper -= weight_ if alt & TAG_GT else 0
|
|
j = j + d
|
|
|
|
# figure out which color
|
|
if alt & TAG_R:
|
|
_, nalt, _, _, _ = fromtag(self.data[j:])
|
|
if nalt & TAG_R:
|
|
path.append((j-d, j, False, 'y'))
|
|
else:
|
|
path.append((j-d, j, False, 'r'))
|
|
else:
|
|
path.append((j-d, j, False, 'b'))
|
|
|
|
# found tag
|
|
else:
|
|
rid_ = upper-1
|
|
tag_ = alt
|
|
w_ = upper-lower
|
|
|
|
done = not tag_ or (rid_, tag_) < (rid, tag)
|
|
|
|
return done, rid_, tag_, w_, j, d, self.data[j+d:j+d+jump], path
|
|
|
|
def __bool__(self):
|
|
return bool(self.trunk)
|
|
|
|
def __eq__(self, other):
|
|
return self.block == other.block and self.trunk == other.trunk
|
|
|
|
def __ne__(self, other):
|
|
return not self.__eq__(other)
|
|
|
|
def __iter__(self):
|
|
tag = 0
|
|
rid = -1
|
|
|
|
while True:
|
|
done, rid, tag, w, j, d, data, _ = self.lookup(rid, tag+0x1)
|
|
if done:
|
|
break
|
|
|
|
yield rid, tag, w, j, d, data
|
|
|
|
# create tree representation for debugging
|
|
def tree(self):
|
|
trunks = co.defaultdict(lambda: (-1, 0))
|
|
alts = co.defaultdict(lambda: {})
|
|
|
|
rid, tag = -1, 0
|
|
while True:
|
|
done, rid, tag, w, j, d, data, path = self.lookup(rid, tag+0x1)
|
|
# found end of tree?
|
|
if done:
|
|
break
|
|
|
|
# keep track of trunks/alts
|
|
trunks[j] = (rid, tag)
|
|
|
|
for j_, j__, followed, c in path:
|
|
if followed:
|
|
alts[j_] |= {'f': j__, 'c': c}
|
|
else:
|
|
alts[j_] |= {'nf': j__, 'c': c}
|
|
|
|
# prune any alts with unreachable edges
|
|
pruned = {}
|
|
for j_, alt in alts.items():
|
|
if 'f' not in alt:
|
|
pruned[j_] = alt['nf']
|
|
elif 'nf' not in alt:
|
|
pruned[j_] = alt['f']
|
|
for j_ in pruned.keys():
|
|
del alts[j_]
|
|
|
|
for j_, alt in alts.items():
|
|
while alt['f'] in pruned:
|
|
alt['f'] = pruned[alt['f']]
|
|
while alt['nf'] in pruned:
|
|
alt['nf'] = pruned[alt['nf']]
|
|
|
|
# find the trunk and depth of each alt, assuming pruned alts
|
|
# didn't exist
|
|
def rec_trunk(j_):
|
|
if j_ not in alts:
|
|
return trunks[j_]
|
|
else:
|
|
if 'nft' not in alts[j_]:
|
|
alts[j_]['nft'] = rec_trunk(alts[j_]['nf'])
|
|
return alts[j_]['nft']
|
|
|
|
for j_ in alts.keys():
|
|
rec_trunk(j_)
|
|
for j_, alt in alts.items():
|
|
if alt['f'] in alts:
|
|
alt['ft'] = alts[alt['f']]['nft']
|
|
else:
|
|
alt['ft'] = trunks[alt['f']]
|
|
|
|
def rec_height(j_):
|
|
if j_ not in alts:
|
|
return 0
|
|
else:
|
|
if 'h' not in alts[j_]:
|
|
alts[j_]['h'] = max(
|
|
rec_height(alts[j_]['f']),
|
|
rec_height(alts[j_]['nf'])) + 1
|
|
return alts[j_]['h']
|
|
|
|
for j_ in alts.keys():
|
|
rec_height(j_)
|
|
|
|
t_depth = max((alt['h']+1 for alt in alts.values()), default=0)
|
|
|
|
# convert to more general tree representation
|
|
tree = set()
|
|
for j, alt in alts.items():
|
|
# note all non-trunk edges should be black
|
|
tree.add(TBranch(
|
|
a=alt['nft'],
|
|
b=alt['nft'],
|
|
d=t_depth-1 - alt['h'],
|
|
c=alt['c'],
|
|
))
|
|
tree.add(TBranch(
|
|
a=alt['nft'],
|
|
b=alt['ft'],
|
|
d=t_depth-1 - alt['h'],
|
|
c='b',
|
|
))
|
|
|
|
return tree, t_depth
|
|
|
|
# btree lookup with this rbyd as the root
|
|
def btree_lookup(self, f, block_size, bid, *,
|
|
depth=None):
|
|
rbyd = self
|
|
rid = bid
|
|
depth_ = 1
|
|
path = []
|
|
|
|
# corrupted? return a corrupted block once
|
|
if not rbyd:
|
|
return bid > 0, bid, 0, rbyd, -1, [], path
|
|
|
|
while True:
|
|
# collect all tags, normally you don't need to do this
|
|
# but we are debugging here
|
|
name = None
|
|
tags = []
|
|
branch = None
|
|
rid_ = rid
|
|
tag = 0
|
|
w = 0
|
|
for i in it.count():
|
|
done, rid__, tag, w_, j, d, data, _ = rbyd.lookup(
|
|
rid_, tag+0x1)
|
|
if done or (i != 0 and rid__ != rid_):
|
|
break
|
|
|
|
# first tag indicates the branch's weight
|
|
if i == 0:
|
|
rid_, w = rid__, w_
|
|
|
|
# catch any branches
|
|
if tag & 0xfff == TAG_BRANCH:
|
|
branch = (tag, j, d, data)
|
|
|
|
tags.append((tag, j, d, data))
|
|
|
|
# keep track of path
|
|
path.append((bid + (rid_-rid), w, rbyd, rid_, tags))
|
|
|
|
# descend down branch?
|
|
if branch is not None and (
|
|
not depth or depth_ < depth):
|
|
tag, j, d, data = branch
|
|
block, trunk, cksum = frombranch(data)
|
|
rbyd = Rbyd.fetch(f, block_size, block, trunk)
|
|
|
|
# corrupted? bail here so we can keep traversing the tree
|
|
if not rbyd:
|
|
return False, bid + (rid_-rid), w, rbyd, -1, [], path
|
|
|
|
rid -= (rid_-(w-1))
|
|
depth_ += 1
|
|
else:
|
|
return not tags, bid + (rid_-rid), w, rbyd, rid_, tags, path
|
|
|
|
# btree rbyd-tree generation for debugging
|
|
def btree_tree(self, f, block_size, *,
|
|
depth=None,
|
|
inner=False):
|
|
# find the max depth of each layer to nicely align trees
|
|
bdepths = {}
|
|
bid = -1
|
|
while True:
|
|
done, bid, w, rbyd, rid, tags, path = self.btree_lookup(
|
|
f, block_size, bid+1, depth=depth)
|
|
if done:
|
|
break
|
|
|
|
for d, (bid, w, rbyd, rid, tags) in enumerate(path):
|
|
_, rdepth = rbyd.tree()
|
|
bdepths[d] = max(bdepths.get(d, 0), rdepth)
|
|
|
|
# find all branches
|
|
tree = set()
|
|
root = None
|
|
branches = {}
|
|
bid = -1
|
|
while True:
|
|
done, bid, w, rbyd, rid, tags, path = self.btree_lookup(
|
|
f, block_size, bid+1, depth=depth)
|
|
if done:
|
|
break
|
|
|
|
d_ = 0
|
|
leaf = None
|
|
for d, (bid, w, rbyd, rid, tags) in enumerate(path):
|
|
if not tags:
|
|
continue
|
|
|
|
# map rbyd tree into B-tree space
|
|
rtree, rdepth = rbyd.tree()
|
|
|
|
# note we adjust our bid/rids to be left-leaning,
|
|
# this allows a global order and make tree rendering quite
|
|
# a bit easier
|
|
rtree_ = set()
|
|
for branch in rtree:
|
|
a_rid, a_tag = branch.a
|
|
b_rid, b_tag = branch.b
|
|
_, _, _, a_w, _, _, _, _ = rbyd.lookup(a_rid, 0)
|
|
_, _, _, b_w, _, _, _, _ = rbyd.lookup(b_rid, 0)
|
|
rtree_.add(TBranch(
|
|
a=(a_rid-(a_w-1), a_tag),
|
|
b=(b_rid-(b_w-1), b_tag),
|
|
d=branch.d,
|
|
c=branch.c,
|
|
))
|
|
rtree = rtree_
|
|
|
|
# connect our branch to the rbyd's root
|
|
if leaf is not None:
|
|
root = min(rtree,
|
|
key=lambda branch: branch.d,
|
|
default=None)
|
|
|
|
if root is not None:
|
|
r_rid, r_tag = root.a
|
|
else:
|
|
r_rid, r_tag = rid-(w-1), tags[0][0]
|
|
tree.add(TBranch(
|
|
a=leaf,
|
|
b=(bid-rid+r_rid, d, r_rid, r_tag),
|
|
d=d_-1,
|
|
c='b',
|
|
))
|
|
|
|
for branch in rtree:
|
|
# map rbyd branches into our btree space
|
|
a_rid, a_tag = branch.a
|
|
b_rid, b_tag = branch.b
|
|
tree.add(TBranch(
|
|
a=(bid-rid+a_rid, d, a_rid, a_tag),
|
|
b=(bid-rid+b_rid, d, b_rid, b_tag),
|
|
d=branch.d + d_ + bdepths.get(d, 0)-rdepth,
|
|
c=branch.c,
|
|
))
|
|
|
|
d_ += max(bdepths.get(d, 0), 1)
|
|
leaf = (bid-(w-1), d, rid-(w-1),
|
|
next((tag for tag, _, _, _ in tags
|
|
if tag & 0xfff == TAG_BRANCH),
|
|
TAG_BRANCH))
|
|
|
|
# remap branches to leaves if we aren't showing inner branches
|
|
if not inner:
|
|
# step through each layer backwards
|
|
b_depth = max((branch.a[1]+1 for branch in tree), default=0)
|
|
|
|
# keep track of the original bids, unfortunately because we
|
|
# store the bids in the branches we overwrite these
|
|
tree = {(branch.b[0] - branch.b[2], branch) for branch in tree}
|
|
|
|
for bd in reversed(range(b_depth-1)):
|
|
# find leaf-roots at this level
|
|
roots = {}
|
|
for bid, branch in tree:
|
|
# choose the highest node as the root
|
|
if (branch.b[1] == b_depth-1
|
|
and (bid not in roots
|
|
or branch.d < roots[bid].d)):
|
|
roots[bid] = branch
|
|
|
|
# remap branches to leaf-roots
|
|
tree_ = set()
|
|
for bid, branch in tree:
|
|
if branch.a[1] == bd and branch.a[0] in roots:
|
|
branch = TBranch(
|
|
a=roots[branch.a[0]].b,
|
|
b=branch.b,
|
|
d=branch.d,
|
|
c=branch.c,
|
|
)
|
|
if branch.b[1] == bd and branch.b[0] in roots:
|
|
branch = TBranch(
|
|
a=branch.a,
|
|
b=roots[branch.b[0]].b,
|
|
d=branch.d,
|
|
c=branch.c,
|
|
)
|
|
tree_.add((bid, branch))
|
|
tree = tree_
|
|
|
|
# strip out bids
|
|
tree = {branch for _, branch in tree}
|
|
|
|
return tree, max((branch.d+1 for branch in tree), default=0)
|
|
|
|
# btree B-tree generation for debugging
|
|
def btree_btree(self, f, block_size, *,
|
|
depth=None,
|
|
inner=False):
|
|
# find all branches
|
|
tree = set()
|
|
root = None
|
|
branches = {}
|
|
bid = -1
|
|
while True:
|
|
done, bid, w, rbyd, rid, tags, path = self.btree_lookup(
|
|
f, block_size, bid+1, depth=depth)
|
|
if done:
|
|
break
|
|
|
|
# if we're not showing inner nodes, prefer names higher in
|
|
# the tree since this avoids showing vestigial names
|
|
name = None
|
|
if not inner:
|
|
name = None
|
|
for bid_, w_, rbyd_, rid_, tags_ in reversed(path):
|
|
for tag_, j_, d_, data_ in tags_:
|
|
if tag_ & 0x7f00 == TAG_NAME:
|
|
name = (tag_, j_, d_, data_)
|
|
|
|
if rid_-(w_-1) != 0:
|
|
break
|
|
|
|
a = root
|
|
for d, (bid, w, rbyd, rid, tags) in enumerate(path):
|
|
if not tags:
|
|
continue
|
|
|
|
b = (bid-(w-1), d, rid-(w-1),
|
|
(name if name else tags[0])[0])
|
|
|
|
# remap branches to leaves if we aren't showing
|
|
# inner branches
|
|
if not inner:
|
|
if b not in branches:
|
|
bid, w, rbyd, rid, tags = path[-1]
|
|
if not tags:
|
|
continue
|
|
branches[b] = (
|
|
bid-(w-1), len(path)-1, rid-(w-1),
|
|
(name if name else tags[0])[0])
|
|
b = branches[b]
|
|
|
|
# found entry point?
|
|
if root is None:
|
|
root = b
|
|
a = root
|
|
|
|
tree.add(TBranch(
|
|
a=a,
|
|
b=b,
|
|
d=d,
|
|
c='b',
|
|
))
|
|
a = b
|
|
|
|
return tree, max((branch.d+1 for branch in tree), default=0)
|
|
|
|
# mtree lookup with this rbyd as the mroot
|
|
def mtree_lookup(self, f, block_size, mbid):
|
|
# have mtree?
|
|
done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE)
|
|
if not done and rid == -1 and tag == TAG_MTREE:
|
|
w, block, trunk, cksum = frombtree(data)
|
|
mtree = Rbyd.fetch(f, block_size, block, trunk)
|
|
# corrupted?
|
|
if not mtree:
|
|
return True, -1, 0, None
|
|
|
|
# lookup our mbid
|
|
done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup(
|
|
f, block_size, mbid)
|
|
if done:
|
|
return True, -1, 0, None
|
|
|
|
mdir = next(((tag, j, d, data)
|
|
for tag, j, d, data in tags
|
|
if tag == TAG_MDIR),
|
|
None)
|
|
if not mdir:
|
|
return True, -1, 0, None
|
|
|
|
# fetch the mdir
|
|
_, _, _, data = mdir
|
|
blocks = frommdir(data)
|
|
return False, mbid, mw, Rbyd.fetch(f, block_size, blocks)
|
|
|
|
else:
|
|
# have mdir?
|
|
done, rid, tag, w, j, _, data, _ = self.lookup(-1, TAG_MDIR)
|
|
if not done and rid == -1 and tag == TAG_MDIR:
|
|
blocks = frommdir(data)
|
|
return False, 0, 0, Rbyd.fetch(f, block_size, blocks)
|
|
|
|
else:
|
|
# I guess we're inlined?
|
|
if mbid == -1:
|
|
return False, -1, 0, self
|
|
else:
|
|
return True, -1, 0, None
|
|
|
|
# lookup by name
|
|
def namelookup(self, did, name):
|
|
# binary search
|
|
best = (False, -1, 0, 0)
|
|
lower = 0
|
|
upper = self.weight
|
|
while lower < upper:
|
|
done, rid, tag, w, j, d, data, _ = self.lookup(
|
|
lower + (upper-1-lower)//2, TAG_NAME)
|
|
if done:
|
|
break
|
|
|
|
# treat vestigial names as a catch-all
|
|
if ((tag == TAG_NAME and rid-(w-1) == 0)
|
|
or (tag & 0xff00) != TAG_NAME):
|
|
did_ = 0
|
|
name_ = b''
|
|
else:
|
|
did_, d = fromleb128(data)
|
|
name_ = data[d:]
|
|
|
|
# bisect search space
|
|
if (did_, name_) > (did, name):
|
|
upper = rid-(w-1)
|
|
elif (did_, name_) < (did, name):
|
|
lower = rid + 1
|
|
|
|
# keep track of best match
|
|
best = (False, rid, tag, w)
|
|
else:
|
|
# found a match
|
|
return True, rid, tag, w
|
|
|
|
return best
|
|
|
|
# lookup by name with this rbyd as the btree root
|
|
def btree_namelookup(self, f, block_size, did, name):
|
|
rbyd = self
|
|
bid = 0
|
|
|
|
while True:
|
|
found, rid, tag, w = rbyd.namelookup(did, name)
|
|
done, rid_, tag_, w_, j, d, data, _ = rbyd.lookup(rid, TAG_STRUCT)
|
|
|
|
# found another branch
|
|
if tag_ == TAG_BRANCH:
|
|
# update our bid
|
|
bid += rid - (w-1)
|
|
|
|
block, trunk, cksum = frombranch(data)
|
|
rbyd = Rbyd.fetch(f, block_size, block, trunk)
|
|
|
|
# found best match
|
|
else:
|
|
return bid + rid, tag_, w, data
|
|
|
|
# lookup by name with this rbyd as the mroot
|
|
def mtree_namelookup(self, f, block_size, did, name):
|
|
# have mtree?
|
|
done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE)
|
|
if not done and rid == -1 and tag == TAG_MTREE:
|
|
w, block, trunk, cksum = frombtree(data)
|
|
mtree = Rbyd.fetch(f, block_size, block, trunk)
|
|
# corrupted?
|
|
if not mtree:
|
|
return False, -1, 0, None, -1, 0, 0
|
|
|
|
# lookup our name in the mtree
|
|
mbid, tag_, mw, data = mtree.btree_namelookup(
|
|
f, block_size, did, name)
|
|
if tag_ != TAG_MDIR:
|
|
return False, -1, 0, None, -1, 0, 0
|
|
|
|
# fetch the mdir
|
|
blocks = frommdir(data)
|
|
mdir = Rbyd.fetch(f, block_size, blocks)
|
|
|
|
else:
|
|
# have mdir?
|
|
done, rid, tag, w, j, _, data, _ = self.lookup(-1, TAG_MDIR)
|
|
if not done and rid == -1 and tag == TAG_MDIR:
|
|
blocks = frommdir(data)
|
|
mbid = 0
|
|
mw = 0
|
|
mdir = Rbyd.fetch(f, block_size, blocks)
|
|
|
|
else:
|
|
# I guess we're inlined?
|
|
mbid = -1
|
|
mw = 0
|
|
mdir = self
|
|
|
|
# lookup name in our mdir
|
|
found, rid, tag, w = mdir.namelookup(did, name)
|
|
return found, mbid, mw, mdir, rid, tag, w
|
|
|
|
# iterate through a directory assuming this is the mtree root
|
|
def mtree_dir(self, f, block_size, did):
|
|
# lookup the bookmark
|
|
found, mbid, mw, mdir, rid, tag, w = self.mtree_namelookup(
|
|
f, block_size, did, b'')
|
|
# iterate through all files until the next bookmark
|
|
while found:
|
|
# lookup each rid
|
|
done, rid, tag, w, j, d, data, _ = mdir.lookup(rid, TAG_NAME)
|
|
if done:
|
|
break
|
|
|
|
# parse out each name
|
|
did_, d_ = fromleb128(data)
|
|
name_ = data[d_:]
|
|
|
|
# end if we see another did
|
|
if did_ != did:
|
|
break
|
|
|
|
# yield what we've found
|
|
yield name_, mbid, mw, mdir, rid, tag, w
|
|
|
|
rid += w
|
|
if rid >= mdir.weight:
|
|
rid -= mdir.weight
|
|
mbid += 1
|
|
|
|
done, mbid, mw, mdir = self.mtree_lookup(f, block_size, mbid)
|
|
if done:
|
|
break
|
|
|
|
|
|
def main(disk, mroots=None, *,
|
|
block_size=None,
|
|
block_count=None,
|
|
mleaf_weight=None,
|
|
block=None,
|
|
off=None,
|
|
size=None,
|
|
mdirs=False,
|
|
btrees=False,
|
|
datas=False,
|
|
no_header=False,
|
|
color='auto',
|
|
dots=False,
|
|
braille=False,
|
|
width=None,
|
|
height=None,
|
|
lines=None,
|
|
hilbert=False,
|
|
lebesgue=False,
|
|
**args):
|
|
# figure out what color should be
|
|
if color == 'auto':
|
|
color = sys.stdout.isatty()
|
|
elif color == 'always':
|
|
color = True
|
|
else:
|
|
color = False
|
|
|
|
# show all block types by default
|
|
if not mdirs and not btrees and not datas:
|
|
mdirs = True
|
|
btrees = True
|
|
datas = True
|
|
|
|
# assume a reasonable lines/height if not specified
|
|
#
|
|
# note that we let height = None if neither hilbert or lebesgue
|
|
# are specified, this is a bit special as the default may be less
|
|
# than one character in height.
|
|
if height is None and (hilbert or lebesgue):
|
|
if lines is not None:
|
|
height = lines
|
|
else:
|
|
height = 5
|
|
|
|
if lines is None:
|
|
if height is not None:
|
|
lines = height
|
|
else:
|
|
lines = 5
|
|
|
|
# is bd geometry specified?
|
|
if isinstance(block_size, tuple):
|
|
block_size, block_count_ = block_size
|
|
if block_count is None:
|
|
block_count = block_count_
|
|
|
|
# try to simplify the block/off/size arguments a bit
|
|
if not isinstance(block, tuple):
|
|
block = block,
|
|
if isinstance(off, tuple) and len(off) == 1:
|
|
off, = off
|
|
if isinstance(size, tuple) and len(size) == 1:
|
|
if off is None:
|
|
off, = size
|
|
size = None
|
|
|
|
if any(isinstance(b, list) and len(b) > 1 for b in block):
|
|
print("error: more than one block address?",
|
|
file=sys.stderr)
|
|
sys.exit(-1)
|
|
if isinstance(block[0], list):
|
|
block = (block[0][0], *block[1:])
|
|
if len(block) > 1 and isinstance(block[1], list):
|
|
block = (block[0], block[1][0])
|
|
if isinstance(block[0], tuple):
|
|
block, off_ = (block[0][0], *block[1:]), block[0][1]
|
|
if off is None:
|
|
off = off_
|
|
if len(block) > 1 and isinstance(block[1], tuple):
|
|
block = (block[0], block[1][0])
|
|
if len(block) == 1:
|
|
block, = block
|
|
|
|
if isinstance(off, tuple):
|
|
off, size_ = off[0], off[1] - off[0]
|
|
if size is None:
|
|
size = size_
|
|
if isinstance(size, tuple):
|
|
off_, size = off[0], off[1] - off[0]
|
|
if off is None:
|
|
off = off_
|
|
|
|
# is a block window specified?
|
|
block_window = None
|
|
if block is not None:
|
|
if isinstance(block, tuple):
|
|
block_window = range(*block)
|
|
else:
|
|
block_window = range(block, block+1)
|
|
|
|
off_window = None
|
|
if off is not None or size is not None:
|
|
off_ = off if off is not None else 0
|
|
size_ = size if size is not None else 1
|
|
off_window = range(off_, off_+size_)
|
|
|
|
# figure out best width/height
|
|
if width is None:
|
|
width_ = min(80, shutil.get_terminal_size((80, 5))[0])
|
|
elif width:
|
|
width_ = width
|
|
else:
|
|
width_ = shutil.get_terminal_size((80, 5))[0]
|
|
|
|
if height is None:
|
|
height_ = 0
|
|
elif height:
|
|
height_ = height
|
|
else:
|
|
height_ = shutil.get_terminal_size((80, 5))[1]
|
|
|
|
# create our block device representation
|
|
bmap = Bmap(
|
|
block_size=block_size,
|
|
block_count=block_count,
|
|
block_window=block_window,
|
|
off_window=off_window,
|
|
# scale if we're printing with dots or braille
|
|
width=2*width_ if braille else width_,
|
|
height=max(1,
|
|
4*height_ if braille
|
|
else 2*height_ if dots
|
|
else height_))
|
|
|
|
# keep track of how many blocks are in use
|
|
mdirs_ = 0
|
|
btrees_ = 0
|
|
datas_ = 0
|
|
|
|
# flatten mroots, default to 0x{0,1}
|
|
if not mroots:
|
|
mroots = [[0,1]]
|
|
mroots = [block for mroots_ in mroots for block in mroots_]
|
|
|
|
# we seek around a bunch, so just keep the disk open
|
|
with open(disk, 'rb') as f:
|
|
# if block_size is omitted, assume the block device is one big block
|
|
if block_size is None:
|
|
f.seek(0, os.SEEK_END)
|
|
block_size = f.tell()
|
|
block_count = 1
|
|
bmap.resize(
|
|
block_size=block_size,
|
|
block_count=block_count)
|
|
|
|
# if block_count is omitted, derive the block_count from our file size
|
|
if block_count is None:
|
|
f.seek(0, os.SEEK_END)
|
|
block_count = f.tell() // block_size
|
|
bmap.resize(
|
|
block_size=block_size,
|
|
block_count=block_count)
|
|
|
|
|
|
# determine the mleaf_weight from the block_size, this is just for
|
|
# printing purposes
|
|
if mleaf_weight is None:
|
|
mleaf_weight = 1 << m.ceil(m.log2(block_size // 16))
|
|
|
|
#### traverse the filesystem
|
|
|
|
# fetch the mroot chain
|
|
corrupted = False
|
|
btrees__ = []
|
|
mroot = Rbyd.fetch(f, block_size, mroots)
|
|
mdepth = 1
|
|
while True:
|
|
# corrupted?
|
|
if not mroot:
|
|
corrupted = True
|
|
break
|
|
|
|
# mark mroots in our bmap
|
|
for block in mroot.blocks:
|
|
bmap.mdir(block,
|
|
mroot.eoff if args.get('in_use') else block_size)
|
|
mdirs_ += 1;
|
|
|
|
# find any file btrees in our mroot
|
|
for rid, tag, w, j, d, data in mroot:
|
|
if tag == TAG_BLOCK or tag == TAG_BTREE:
|
|
btrees__.append((tag, data))
|
|
|
|
# stop here?
|
|
if args.get('depth') and mdepth >= args.get('depth'):
|
|
break
|
|
|
|
# fetch the next mroot
|
|
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MROOT)
|
|
if not (not done and rid == -1 and tag == TAG_MROOT):
|
|
break
|
|
|
|
blocks = frommdir(data)
|
|
mroot = Rbyd.fetch(f, block_size, blocks)
|
|
mdepth += 1
|
|
|
|
# fetch the mdir, if there is one
|
|
mdir = None
|
|
if not args.get('depth') or mdepth < args.get('depth'):
|
|
done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MDIR)
|
|
if not done and rid == -1 and tag == TAG_MDIR:
|
|
blocks = frommdir(data)
|
|
mdir = Rbyd.fetch(f, block_size, blocks)
|
|
|
|
# corrupted?
|
|
if not mdir:
|
|
corrupted = True
|
|
else:
|
|
# mark mdir in our bmap
|
|
for block in mdir.blocks:
|
|
bmap.mdir(block,
|
|
mdir.eoff if args.get('in_use') else block_size)
|
|
mdirs_ += 1
|
|
|
|
# find any file btrees in our mdir
|
|
for rid, tag, w, j, d, data in mdir:
|
|
if tag == TAG_BLOCK or tag == TAG_BTREE:
|
|
btrees__.append((tag, data))
|
|
|
|
# fetch the actual mtree, if there is one
|
|
mtree = None
|
|
if not args.get('depth') or mdepth < args.get('depth'):
|
|
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
|
|
if not done and rid == -1 and tag == TAG_MTREE:
|
|
w, block, trunk, cksum = frombtree(data)
|
|
mtree = Rbyd.fetch(f, block_size, block, trunk)
|
|
|
|
# traverse entries
|
|
mbid = -1
|
|
ppath = []
|
|
while True:
|
|
done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup(
|
|
f, block_size, mbid+1,
|
|
depth=args.get('depth', mdepth)-mdepth)
|
|
if done:
|
|
break
|
|
|
|
# traverse the inner btree nodes
|
|
changed = False
|
|
for (x, px) in it.zip_longest(
|
|
enumerate(path),
|
|
enumerate(ppath)):
|
|
if x is None:
|
|
break
|
|
if not (changed or px is None or x[0] != px[0]):
|
|
continue
|
|
changed = True
|
|
|
|
# mark btree inner nodes in our bmap
|
|
d, (mid_, w_, rbyd_, rid_, tags_) = x
|
|
for block in rbyd_.blocks:
|
|
bmap.btree(block,
|
|
rbyd_.eoff if args.get('in_use')
|
|
else block_size)
|
|
btrees_ += 1
|
|
ppath = path
|
|
|
|
# corrupted?
|
|
if not rbyd:
|
|
corrupted = True
|
|
continue
|
|
|
|
# found an mdir in the tags?
|
|
mdir__ = None
|
|
if (not args.get('depth')
|
|
or mdepth+len(path) < args.get('depth')):
|
|
mdir__ = next(((tag, j, d, data)
|
|
for tag, j, d, data in tags
|
|
if tag == TAG_MDIR),
|
|
None)
|
|
|
|
if mdir__:
|
|
# fetch the mdir
|
|
_, _, _, data = mdir__
|
|
blocks = frommdir(data)
|
|
mdir_ = Rbyd.fetch(f, block_size, blocks)
|
|
|
|
# corrupted?
|
|
if not mdir_:
|
|
corrupted = True
|
|
else:
|
|
# mark mdir in our bmap
|
|
for block in mdir_.blocks:
|
|
bmap.mdir(block, 0,
|
|
mdir_.eoff if args.get('in_use')
|
|
else block_size)
|
|
mdirs_ += 1
|
|
|
|
# find any file btrees in our mdir
|
|
for rid, tag, w, j, d, data in mdir_:
|
|
if tag == TAG_BLOCK or tag == TAG_BTREE:
|
|
btrees__.append((tag, data))
|
|
|
|
# fetch any file btrees we found
|
|
if not args.get('depth') or mdepth < args.get('depth'):
|
|
for tag, data in btrees__:
|
|
# direct block?
|
|
if tag == TAG_BLOCK:
|
|
size, block, off = frombptr(data)
|
|
# mark block in our bmap
|
|
bmap.data(block,
|
|
off if args.get('in_use') else 0,
|
|
size if args.get('in_use') else block_size)
|
|
datas_ += 1
|
|
|
|
# indirect btree?
|
|
elif tag == TAG_BTREE:
|
|
w, block, trunk, cksum = frombtree(data)
|
|
btree = Rbyd.fetch(f, block_size, block, trunk)
|
|
|
|
# traverse entries
|
|
bid = -1
|
|
ppath = []
|
|
while True:
|
|
(done, bid, w, rbyd, rid, tags, path
|
|
) = btree.btree_lookup(
|
|
f, block_size, bid+1,
|
|
depth=args.get('depth', mdepth)-mdepth)
|
|
if done:
|
|
break
|
|
|
|
# traverse the inner btree nodes
|
|
changed = False
|
|
for (x, px) in it.zip_longest(
|
|
enumerate(path),
|
|
enumerate(ppath)):
|
|
if x is None:
|
|
break
|
|
if not (changed or px is None or x[0] != px[0]):
|
|
continue
|
|
changed = True
|
|
|
|
# mark btree inner nodes in our bmap
|
|
d, (mid_, w_, rbyd_, rid_, tags_) = x
|
|
for block in rbyd_.blocks:
|
|
bmap.btree(block,
|
|
rbyd_.eoff if args.get('in_use')
|
|
else block_size)
|
|
btrees_ += 1
|
|
ppath = path
|
|
|
|
# corrupted?
|
|
if not rbyd:
|
|
corrupted = True
|
|
continue
|
|
|
|
# found a block in the tags?
|
|
bptr__ = None
|
|
if (not args.get('depth')
|
|
or mdepth+len(path) < args.get('depth')):
|
|
bptr__ = next(((tag, j, d, data)
|
|
for tag, j, d, data in tags
|
|
if tag == TAG_BLOCK),
|
|
None)
|
|
|
|
if bptr__:
|
|
# fetch the block
|
|
_, _, _, data = bptr__
|
|
size, block, off = frombptr(data)
|
|
|
|
# mark blocks in our bmap
|
|
bmap.data(block,
|
|
off if args.get('in_use') else 0,
|
|
size if args.get('in_use') else block_size)
|
|
datas_ += 1
|
|
|
|
#### actual rendering begins here
|
|
|
|
# print some information about the bmap
|
|
if not no_header:
|
|
print('bd %dx%d%s%s%s' % (
|
|
block_size, block_count,
|
|
', %6s mdir' % ('%.1f%%' % (100*mdirs_ / block_count))
|
|
if mdirs else '',
|
|
', %6s btree' % ('%.1f%%' % (100*btrees_ / block_count))
|
|
if btrees else '',
|
|
', %6s data' % ('%.1f%%' % (100*datas_ / block_count))
|
|
if datas else ''))
|
|
|
|
# and then print the bmap
|
|
for row in range(
|
|
m.ceil(bmap.height/4) if braille
|
|
else m.ceil(bmap.height/2) if dots
|
|
else bmap.height):
|
|
line = bmap.draw(row,
|
|
mdirs=mdirs,
|
|
btrees=btrees,
|
|
datas=datas,
|
|
color=color,
|
|
dots=dots,
|
|
braille=braille,
|
|
hilbert=hilbert,
|
|
lebesgue=lebesgue,
|
|
**args)
|
|
print(line)
|
|
|
|
if args.get('error_on_corrupt') and corrupted:
|
|
sys.exit(2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
import sys
|
|
parser = argparse.ArgumentParser(
|
|
description="Render currently used blocks in a littlefs image.",
|
|
allow_abbrev=False)
|
|
parser.add_argument(
|
|
'disk',
|
|
help="File containing the block device.")
|
|
parser.add_argument(
|
|
'mroots',
|
|
nargs='*',
|
|
type=rbydaddr,
|
|
help="Block address of the mroots. Defaults to 0x{0,1}.")
|
|
parser.add_argument(
|
|
'-B', '--block-size',
|
|
type=bdgeom,
|
|
help="Block size/geometry in bytes.")
|
|
parser.add_argument(
|
|
'--block-count',
|
|
type=lambda x: int(x, 0),
|
|
help="Block count in blocks.")
|
|
parser.add_argument(
|
|
'-M', '--mleaf-weight',
|
|
type=lambda x: int(x, 0),
|
|
help="Maximum weight of mdirs for mid decoding. Defaults to a "
|
|
"block_size derived value.")
|
|
parser.add_argument(
|
|
'-@', '--block',
|
|
nargs='?',
|
|
type=lambda x: tuple(
|
|
rbydaddr(x) if x.strip() else None
|
|
for x in x.split(',')),
|
|
help="Optional block to show, may be a range.")
|
|
parser.add_argument(
|
|
'--off',
|
|
type=lambda x: tuple(
|
|
int(x, 0) if x.strip() else None
|
|
for x in x.split(',')),
|
|
help="Show a specific offset, may be a range.")
|
|
parser.add_argument(
|
|
'--size',
|
|
type=lambda x: tuple(
|
|
int(x, 0) if x.strip() else None
|
|
for x in x.split(',')),
|
|
help="Show this many bytes, may be a range.")
|
|
parser.add_argument(
|
|
'-m', '--mdirs',
|
|
action='store_true',
|
|
help="Render mdir blocks.")
|
|
parser.add_argument(
|
|
'-b', '--btrees',
|
|
action='store_true',
|
|
help="Render btree blocks.")
|
|
parser.add_argument(
|
|
'-d', '--datas',
|
|
action='store_true',
|
|
help="Render data blocks.")
|
|
parser.add_argument(
|
|
'-N', '--no-header',
|
|
action='store_true',
|
|
help="Don't show the header.")
|
|
parser.add_argument(
|
|
'--color',
|
|
choices=['never', 'always', 'auto'],
|
|
default='auto',
|
|
help="When to use terminal colors. Defaults to 'auto'.")
|
|
parser.add_argument(
|
|
'-:', '--dots',
|
|
action='store_true',
|
|
help="Use 1x2 ascii dot characters.")
|
|
parser.add_argument(
|
|
'-⣿', '--braille',
|
|
action='store_true',
|
|
help="Use 2x4 unicode braille characters. Note that braille characters "
|
|
"sometimes suffer from inconsistent widths.")
|
|
parser.add_argument(
|
|
'--chars',
|
|
help="Characters to use for mdir, btree, data, unused blocks.")
|
|
parser.add_argument(
|
|
'--colors',
|
|
type=lambda x: [x.strip() for x in x.split(',')],
|
|
help="Colors to use for mdir, btree, data, unused blocks.")
|
|
parser.add_argument(
|
|
'-W', '--width',
|
|
nargs='?',
|
|
type=lambda x: int(x, 0),
|
|
const=0,
|
|
help="Width in columns. 0 uses the terminal width. Defaults to "
|
|
"min(terminal, 80).")
|
|
parser.add_argument(
|
|
'-H', '--height',
|
|
nargs='?',
|
|
type=lambda x: int(x, 0),
|
|
const=0,
|
|
help="Height in rows. 0 uses the terminal height. Defaults to 1.")
|
|
parser.add_argument(
|
|
'-n', '--lines',
|
|
nargs='?',
|
|
type=lambda x: int(x, 0),
|
|
const=0,
|
|
help="Show this many lines of history. 0 uses the terminal height. "
|
|
"Defaults to 5.")
|
|
parser.add_argument(
|
|
'-U', '--hilbert',
|
|
action='store_true',
|
|
help="Render as a space-filling Hilbert curve.")
|
|
parser.add_argument(
|
|
'-Z', '--lebesgue',
|
|
action='store_true',
|
|
help="Render as a space-filling Z-curve.")
|
|
parser.add_argument(
|
|
'-i', '--in-use',
|
|
action='store_true',
|
|
help="Show how much of each block is in use.")
|
|
parser.add_argument(
|
|
'--depth',
|
|
nargs='?',
|
|
type=lambda x: int(x, 0),
|
|
const=0,
|
|
help="Depth of the filesystem tree to parse.")
|
|
parser.add_argument(
|
|
'-e', '--error-on-corrupt',
|
|
action='store_true',
|
|
help="Error if the filesystem is corrupt.")
|
|
sys.exit(main(**{k: v
|
|
for k, v in vars(parser.parse_intermixed_args()).items()
|
|
if v is not None}))
|