forked from Imagelibrary/littlefs
a85f08cfe3
This merges LFS3_o_GRAFT into LFS3_o_UNCRYST, simplifying the file write
path and avoiding the mess that is ungrafted leaves.
---
This goes for a different lazy crystallization/grafting strategy that
was overlooked before. Instead of requiring all leaves to be both
crystallized and grafted, we allow leaves to be uncrystallied, but they
_must_ be grafted (in-tree) at all times.
This gets us most of the rewrite preformance of lazy-crystallization,
without needing to worry about out-of-date file leaves.
Out-of-date file leaves were a headache for both code cost and concerns
around confusing filesystem states and related bugs.
Note LFS3_o_UNCRYST gets some extra behavior here:
- LFS3_o_UNCRYST indicates when crystallization is _necessary_, and no
longer when crystallization is _possible_.
We already keep track of when crystallization is _possible_ via bptr's
erased-state, and this lets us control recrystallization in
lfs3_file_flush_ without erased-state-clearing hacks (which probably
wouldn't work with the future ddtree).
- We opportunistically clear the UNCRYST flag if it's not possible for
future lfs3_file_crystallize_ calls to make progress:
- When we crystallize a full block
- When we hit the end of the file
- When we hit a hole
- When we hit an unaligned block
---
Note this does impact performance!
Unlike true lazy grafting, eagerly grafting means we're always
committing to the bshrub/btree more than is strictly necessary, and this
translates to more frequent btree node erases/compactions.
Current simulated benchmarks show a ~3x increase (~20us -> ~60us) in
write times for linear file writes on NOR flash.
However:
- The moment you need unaligned progs, this performance optimization
goes out the window, as we need to graft bptrs before any padding
fragments.
- This only kicks in once we start crystallizing. So any writes <
crystal_thresh (both in new files and in between blocks) are forced
to commit to the bshrub/btree every flush.
This risks a difficult to predict performance characteristic.
- If you sync frequently (logging), we're forced to crystallize/graft
anyways.
- The performance hit can be alleviated with either larger writes or
larger caches, though I realize this goes against littlefs's
"RAM-not-required" mantra.
Worst case, we can always bring back "lazy grafting" as a
high-performance option in the future.
Though note the above concerns around in-between/pre crystallization
performance. This may only make sense when cache_size >= both prog_size
and crystal_thresh.
And of course, there's a significant code tradeoff!
code stack ctx
before: 38020 2456 656
after: 37588 (-1.1%) 2472 (+0.7%) 656 (+0.0%)
Uh, ignore that stack cost. The simplified logic leads to more functions
being inlined, which makes a mess of our stack measurements because we
don't take shrinkwrapping into account.
363 lines
16 KiB
Python
Executable File
363 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# prevent local imports
|
|
if __name__ == "__main__":
|
|
__import__('sys').path.pop(0)
|
|
|
|
import collections as co
|
|
|
|
|
|
ALIASES = [
|
|
{'O', 'OPEN'},
|
|
{'A', 'ATTR'},
|
|
{'F', 'FORMAT'},
|
|
{'M', 'MOUNT'},
|
|
{'GC'},
|
|
{'I', 'INFO'},
|
|
{'T', 'TRAVERSAL'},
|
|
{'RC', 'RCOMPAT'},
|
|
{'WC', 'WCOMPAT'},
|
|
{'OC', 'OCOMPAT'}
|
|
]
|
|
|
|
FLAGS = [
|
|
# File open flags
|
|
('O', 'MODE', 3, "The file's access mode" ),
|
|
('^', 'RDONLY', 0, "Open a file as read only" ),
|
|
('^', 'WRONLY', 1, "Open a file as write only" ),
|
|
('^', 'RDWR', 2, "Open a file as read and write" ),
|
|
('O', 'CREAT', 0x00000004, "Create a file if it does not exist" ),
|
|
('O', 'EXCL', 0x00000008, "Fail if a file already exists" ),
|
|
('O', 'TRUNC', 0x00000010, "Truncate the existing file to zero size" ),
|
|
('O', 'APPEND', 0x00000020, "Move to end of file on every write" ),
|
|
('O', 'FLUSH', 0x00000040, "Flush data on every write" ),
|
|
('O', 'SYNC', 0x00000080, "Sync metadata on every write" ),
|
|
('O', 'DESYNC', 0x04000000, "Do not sync or recieve file updates" ),
|
|
('O', 'CKMETA', 0x00001000, "Check metadata checksums" ),
|
|
('O', 'CKDATA', 0x00002000, "Check metadata + data checksums" ),
|
|
|
|
('o', 'WRSET', 3, "Open a file as an atomic write" ),
|
|
('o', 'TYPE', 0xf0000000, "The file's type" ),
|
|
('^', 'REG', 0x10000000, "Type = regular-file" ),
|
|
('^', 'DIR', 0x20000000, "Type = directory" ),
|
|
('^', 'STICKYNOTE',0x30000000, "Type = stickynote" ),
|
|
('^', 'BOOKMARK', 0x40000000, "Type = bookmark" ),
|
|
('^', 'ORPHAN', 0x50000000, "Type = orphan" ),
|
|
('^', 'TRAVERSAL', 0x60000000, "Type = traversal" ),
|
|
('^', 'UNKNOWN', 0x70000000, "Type = unknown" ),
|
|
('o', 'ZOMBIE', 0x08000000, "File has been removed" ),
|
|
('o', 'UNCREAT', 0x02000000, "File does not exist yet" ),
|
|
('o', 'UNSYNC', 0x01000000, "File's metadata does not match disk" ),
|
|
('o', 'UNCRYST', 0x00800000, "File's leaf not fully crystallized" ),
|
|
('o', 'UNFLUSH', 0x00400000, "File's cache does not match disk" ),
|
|
|
|
# Custom attribute flags
|
|
('A', 'MODE', 3, "The attr's access mode" ),
|
|
('^', 'RDONLY', 0, "Open an attr as read only" ),
|
|
('^', 'WRONLY', 1, "Open an attr as write only" ),
|
|
('^', 'RDWR', 2, "Open an attr as read and write" ),
|
|
('A', 'LAZY', 0x04, "Only write attr if file changed" ),
|
|
|
|
# Filesystem format flags
|
|
('F', 'MODE', 1, "Format's access mode" ),
|
|
('^', 'RDWR', 0, "Format the filesystem as read and write" ),
|
|
('F', 'REVDBG', 0x00000010, "Add debug info to revision counts" ),
|
|
('F', 'REVNOISE', 0x00000020, "Add noise to revision counts" ),
|
|
('F', 'CKPROGS', 0x00080000, "Check progs by reading back progged data" ),
|
|
('F', 'CKFETCHES', 0x00100000, "Check block checksums before first use" ),
|
|
('F', 'CKMETAPARITY',
|
|
0x00200000, "Check metadata tag parity bits" ),
|
|
('F', 'CKDATACKSUMREADS',
|
|
0x00800000, "Check data checksums on reads" ),
|
|
|
|
('F', 'CKMETA', 0x00001000, "Check metadata checksums" ),
|
|
('F', 'CKDATA', 0x00002000, "Check metadata + data checksums" ),
|
|
|
|
# Filesystem mount flags
|
|
('M', 'MODE', 1, "Mount's access mode" ),
|
|
('^', 'RDWR', 0, "Mount the filesystem as read and write" ),
|
|
('^', 'RDONLY', 1, "Mount the filesystem as read only" ),
|
|
('M', 'FLUSH', 0x00000040, "Open all files with LFS3_O_FLUSH" ),
|
|
('M', 'SYNC', 0x00000080, "Open all files with LFS3_O_SYNC" ),
|
|
('M', 'REVDBG', 0x00000010, "Add debug info to revision counts" ),
|
|
('M', 'REVNOISE', 0x00000020, "Add noise to revision counts" ),
|
|
('M', 'CKPROGS', 0x00080000, "Check progs by reading back progged data" ),
|
|
('M', 'CKFETCHES', 0x00100000, "Check block checksums before first use" ),
|
|
('M', 'CKMETAPARITY',
|
|
0x00200000, "Check metadata tag parity bits" ),
|
|
('M', 'CKDATACKSUMREADS',
|
|
0x00800000, "Check data checksums on reads" ),
|
|
|
|
('M', 'MKCONSISTENT',
|
|
0x00000100, "Make the filesystem consistent" ),
|
|
('M', 'LOOKAHEAD', 0x00000200, "Populate lookahead buffer" ),
|
|
('M', 'COMPACT', 0x00000800, "Compact metadata logs" ),
|
|
('M', 'CKMETA', 0x00001000, "Check metadata checksums" ),
|
|
('M', 'CKDATA', 0x00002000, "Check metadata + data checksums" ),
|
|
|
|
# GC flags
|
|
('GC', 'MKCONSISTENT',
|
|
0x00000100, "Make the filesystem consistent" ),
|
|
('GC', 'LOOKAHEAD',0x00000200, "Populate lookahead buffer" ),
|
|
('GC', 'COMPACT', 0x00000800, "Compact metadata logs" ),
|
|
('GC', 'CKMETA', 0x00001000, "Check metadata checksums" ),
|
|
('GC', 'CKDATA', 0x00002000, "Check metadata + data checksums" ),
|
|
|
|
# Filesystem info flags
|
|
('I', 'RDONLY', 0x00000001, "Mounted read only" ),
|
|
('I', 'FLUSH', 0x00000040, "Mounted with LFS3_M_FLUSH" ),
|
|
('I', 'SYNC', 0x00000080, "Mounted with LFS3_M_SYNC" ),
|
|
('I', 'REVDBG', 0x00000010, "Mounted with LFS3_M_REVDBG" ),
|
|
('I', 'REVNOISE', 0x00000020, "Mounted with LFS3_M_REVNOISE" ),
|
|
('I', 'CKPROGS', 0x00080000, "Mounted with LFS3_M_CKPROGS" ),
|
|
('I', 'CKFETCHES', 0x00100000, "Mounted with LFS3_M_CKFETCHES" ),
|
|
('I', 'CKMETAPARITY',
|
|
0x00200000, "Mounted with LFS3_M_CKMETAPARITY" ),
|
|
('I', 'CKDATACKSUMREADS',
|
|
0x00800000, "Mounted with LFS3_M_CKDATACKSUMREADS" ),
|
|
|
|
('I', 'MKCONSISTENT',
|
|
0x00000100, "Filesystem needs mkconsistent to write" ),
|
|
('I', 'LOOKAHEAD', 0x00000200, "Lookahead buffer is not full" ),
|
|
('I', 'COMPACT', 0x00000800, "Filesystem may have uncompacted metadata" ),
|
|
('I', 'CKMETA', 0x00001000, "Metadata checksums not checked recently" ),
|
|
('I', 'CKDATA', 0x00002000, "Data checksums not checked recently" ),
|
|
|
|
('i', 'INMTREE', 0x08000000, "Committing to mtree" ),
|
|
|
|
# Traversal flags
|
|
('T', 'MODE', 1, "The traversal's access mode" ),
|
|
('^', 'RDWR', 0, "Open traversal as read and write" ),
|
|
('^', 'RDONLY', 1, "Open traversal as read only" ),
|
|
('T', 'MTREEONLY', 0x00000002, "Only traverse the mtree" ),
|
|
('T', 'MKCONSISTENT',
|
|
0x00000100, "Make the filesystem consistent" ),
|
|
('T', 'LOOKAHEAD', 0x00000200, "Populate lookahead buffer" ),
|
|
('T', 'COMPACT', 0x00000800, "Compact metadata logs" ),
|
|
('T', 'CKMETA', 0x00001000, "Check metadata checksums" ),
|
|
('T', 'CKDATA', 0x00002000, "Check metadata + data checksums" ),
|
|
|
|
('t', 'TYPE', 0xf0000000, "The traversal's type" ),
|
|
('^', 'REG', 0x10000000, "Type = regular-file" ),
|
|
('^', 'DIR', 0x20000000, "Type = directory" ),
|
|
('^', 'STICKYNOTE',0x30000000, "Type = stickynote" ),
|
|
('^', 'BOOKMARK', 0x40000000, "Type = bookmark" ),
|
|
('^', 'ORPHAN', 0x50000000, "Type = orphan" ),
|
|
('^', 'TRAVERSAL', 0x60000000, "Type = traversal" ),
|
|
('^', 'UNKNOWN', 0x70000000, "Type = unknown" ),
|
|
('t', 'TSTATE', 0x000f0000, "The current traversal state" ),
|
|
('^', 'MROOTANCHOR',
|
|
0x00000000, "Tstate = mroot-anchor" ),
|
|
('^', 'MROOTCHAIN',0x00010000, "Tstate = mroot-chain" ),
|
|
('^', 'MTREE', 0x00020000, "Tstate = mtree" ),
|
|
('^', 'MDIRS', 0x00030000, "Tstate = mtree-mdirs" ),
|
|
('^', 'MDIR', 0x00040000, "Tstate = mdir" ),
|
|
('^', 'BTREE', 0x00050000, "Tstate = btree" ),
|
|
('^', 'OMDIRS', 0x00060000, "Tstate = open-mdirs" ),
|
|
('^', 'OBTREE', 0x00070000, "Tstate = open-btree" ),
|
|
('^', 'DONE', 0x00080000, "Tstate = done" ),
|
|
('t', 'BTYPE', 0x00f00000, "The current block type" ),
|
|
('^', 'MDIR', 0x00100000, "Btype = mdir" ),
|
|
('^', 'BTREE', 0x00200000, "Btype = btree" ),
|
|
('^', 'DATA', 0x00300000, "Btype = data" ),
|
|
('t', 'ZOMBIE', 0x08000000, "File has been removed" ),
|
|
('t', 'DIRTY', 0x02000000, "Filesystem modified during traversal" ),
|
|
('t', 'MUTATED', 0x01000000, "Filesystem modified by traversal" ),
|
|
|
|
# Read-compat flags
|
|
('RCOMPAT', 'NONSTANDARD',
|
|
0x00000001, "Non-standard filesystem format" ),
|
|
('RCOMPAT', 'WRONLY',
|
|
0x00000002, "Reading is disallowed" ),
|
|
('RCOMPAT', 'BMOSS',
|
|
0x00000010, "Files may use inlined data" ),
|
|
('RCOMPAT', 'BSPROUT',
|
|
0x00000020, "Files may use block pointers" ),
|
|
('RCOMPAT', 'BSHRUB',
|
|
0x00000040, "Files may use inlined btrees" ),
|
|
('RCOMPAT', 'BTREE',
|
|
0x00000080, "Files may use btrees" ),
|
|
('RCOMPAT', 'MMOSS',
|
|
0x00000100, "May use an inlined mdir" ),
|
|
('RCOMPAT', 'MSPROUT',
|
|
0x00000200, "May use an mdir pointer" ),
|
|
('RCOMPAT', 'MSHRUB',
|
|
0x00000400, "May use an inlined mtree" ),
|
|
('RCOMPAT', 'MTREE',
|
|
0x00000800, "May use an mdir btree" ),
|
|
('RCOMPAT', 'GRM',
|
|
0x00001000, "Global-remove in use" ),
|
|
('rcompat', 'OVERFLOW',
|
|
0x80000000, "Can't represent all flags" ),
|
|
|
|
# Write-compat flags
|
|
('WCOMPAT', 'NONSTANDARD',
|
|
0x00000001, "Non-standard filesystem format" ),
|
|
('WCOMPAT', 'RDONLY',
|
|
0x00000002, "Writing is disallowed" ),
|
|
('WCOMPAT', 'DIR',
|
|
0x00000010, "Directory file types in use" ),
|
|
('WCOMPAT', 'GCKSUM',
|
|
0x00001000, "Global-checksum in use" ),
|
|
('wcompat', 'OVERFLOW',
|
|
0x80000000, "Can't represent all flags" ),
|
|
|
|
# Optional-compat flags
|
|
('OCOMPAT', 'NONSTANDARD',
|
|
0x00000001, "Non-standard filesystem format" ),
|
|
('ocompat', 'OVERFLOW',
|
|
0x80000000, "Can't represent all flags" ),
|
|
]
|
|
|
|
|
|
def main(flags, *,
|
|
list=False,
|
|
all=False):
|
|
import builtins
|
|
list_, list = list, builtins.list
|
|
all_, all = all, builtins.all
|
|
|
|
# first compile prefixes
|
|
prefixes = {}
|
|
for a in ALIASES:
|
|
for k in a:
|
|
prefixes[k] = a
|
|
for p, n, f, h in FLAGS:
|
|
if p not in prefixes:
|
|
prefixes[p] = {p}
|
|
|
|
# only look at specific prefix?
|
|
flags_ = []
|
|
prefix = set()
|
|
for f in flags:
|
|
# accept prefix prefix
|
|
if f.upper() in prefixes:
|
|
prefix.update(prefixes[f.upper()])
|
|
# accept LFS3_+prefix prefix
|
|
elif (f.upper().startswith('LFS3_')
|
|
and f.upper()[len('LFS3_'):] in prefixes):
|
|
prefix.update(prefixes[f.upper()[len('LFS3_'):]])
|
|
else:
|
|
flags_.append(f)
|
|
|
|
# filter by prefix
|
|
flags__ = []
|
|
types__ = co.defaultdict(lambda: set())
|
|
for p, n, f, h in FLAGS:
|
|
if p == '^':
|
|
p = last_p
|
|
t = last_t
|
|
types__[p].add(t)
|
|
else:
|
|
t = None
|
|
last_p = p
|
|
last_t = f
|
|
if not prefix or p.upper() in prefix:
|
|
flags__.append((p, t, n, f, h))
|
|
|
|
lines = []
|
|
# list all known flags
|
|
if list_:
|
|
for p, t, n, f, h in flags__:
|
|
if not all_ and (t is not None or p[0].islower()):
|
|
continue
|
|
lines.append(('LFS3_%s_%s' % (p, n), '0x%08x' % f, h))
|
|
|
|
# find flags by name or value
|
|
else:
|
|
for f_ in flags_:
|
|
found = False
|
|
# find by LFS3_+prefix+_+name
|
|
for p, t, n, f, h in flags__:
|
|
if 'LFS3_%s_%s' % (p, n) == f_.upper():
|
|
lines.append(('LFS3_%s_%s' % (p, n), '0x%08x' % f, h))
|
|
found = True
|
|
if found:
|
|
continue
|
|
# find by prefix+_+name
|
|
for p, t, n, f, h in flags__:
|
|
if '%s_%s' % (p, n) == f_.upper():
|
|
lines.append(('LFS3_%s_%s' % (p, n), '0x%08x' % f, h))
|
|
found = True
|
|
if found:
|
|
continue
|
|
# find by name
|
|
for p, t, n, f, h in flags__:
|
|
if n == f_.upper():
|
|
lines.append(('LFS3_%s_%s' % (p, n), '0x%08x' % f, h))
|
|
found = True
|
|
if found:
|
|
continue
|
|
# find by value
|
|
try:
|
|
f__ = int(f_, 0)
|
|
f___ = f__
|
|
for p, t, n, f, h in flags__:
|
|
# ignore type masks here
|
|
if t is None and f in types__[p]:
|
|
continue
|
|
# matches flag?
|
|
if t is None and (f__ & f) == f:
|
|
lines.append(('LFS3_%s_%s' % (p, n), '0x%08x' % f, h))
|
|
f___ &= ~f
|
|
# matches type?
|
|
elif t is not None and (f__ & t) == f:
|
|
lines.append(('LFS3_%s_%s' % (p, n), '0x%08x' % f, h))
|
|
f___ &= ~t
|
|
if f___:
|
|
lines.append(('?', '0x%08x' % f___, 'Unknown flags'))
|
|
except ValueError:
|
|
lines.append(('?', f_, 'Unknown flag'))
|
|
|
|
# first find widths
|
|
w = [0, 0]
|
|
for l in lines:
|
|
w[0] = max(w[0], len(l[0]))
|
|
w[1] = max(w[1], len(l[1]))
|
|
|
|
# then print results
|
|
for l in lines:
|
|
print('%-*s %-*s %s' % (
|
|
w[0], l[0],
|
|
w[1], l[1],
|
|
l[2]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
import sys
|
|
parser = argparse.ArgumentParser(
|
|
description="Decode littlefs flags.",
|
|
allow_abbrev=False)
|
|
class AppendFlags(argparse.Action):
|
|
def __call__(self, parser, namespace, value, option):
|
|
if getattr(namespace, 'flags', None) is None:
|
|
namespace.flags = []
|
|
if value is None:
|
|
pass
|
|
elif isinstance(value, str):
|
|
namespace.flags.append(value)
|
|
else:
|
|
namespace.flags.extend(value)
|
|
parser.add_argument(
|
|
'prefix',
|
|
nargs='?',
|
|
action=AppendFlags,
|
|
help="Flag prefix to consider, defaults to all flags.")
|
|
parser.add_argument(
|
|
'flags',
|
|
nargs='*',
|
|
action=AppendFlags,
|
|
help="Flags or names of flags to decode.")
|
|
parser.add_argument(
|
|
'-l', '--list',
|
|
action='store_true',
|
|
help="List all known flags.")
|
|
parser.add_argument(
|
|
'-a', '--all',
|
|
action='store_true',
|
|
help="Also show internal flags and types.")
|
|
sys.exit(main(**{k: v
|
|
for k, v in vars(parser.parse_intermixed_args()).items()
|
|
if v is not None}))
|