MagnitudeOS/littlefs
1
0
Fork 0
forked from Imagelibrary/littlefs
Code Pull Requests Activity

Cleaned up tag encoding, now with clear chunk field

Browse Source 
Before, the tag format's type field was limited to 9-bits. This sounds
like a lot, but this field needed to encode up to 256 user-specified
types. This limited the flexibility of the encoded types. As time went
on, more bits in the type field were repurposed for various things,
leaving a rather fragile type field.

Here we make the jump to full 11-bit type fields. This comes at the cost
of a smaller length field, however the use of the length field was
always going to come with a RAM limitation. Rather than putting pressure
on RAM for inline files, the new type field lets us encode a chunk
number, splitting up inline files into multiple updatable units. This
actually pushes the theoretical inline max from 8KiB to 256KiB! (Note
that we only allow a single 1KiB chunk for now, chunky inline files
is just a theoretical future improvement).

Here is the new 32-bit tag format, note that there are multiple levels
of types which break down into more info:

[----            32             ----]
[1|--  11   --|--  10  --|--  10  --]
 ^.     ^     .     ^          ^- entry length
 |.     |     .     \------------ file id chunk info
 |.     \-----.------------------ type info (type3)
 \.-----------.------------------ valid bit
  [-3-|-- 8 --]
    ^     ^- chunk info
    \------- type info (type1)

Additionally, I've split the CREATE tag into separate SPLICE and NAME
tags. This simplified the new compact logic a bit. For now, littlefs
still follows the rule that a NAME tag precedes any other tags related
to a file, but this can change in the future.
This commit is contained in:
Christopher Haster
2018-12-29 07:53:12 -06:00
parent a548ce68c1
commit b989b4a89f
6 changed files with 426 additions and 460 deletions
Download Patch File Download Diff File Expand all files Collapse all files

663
lfs.c
View File

File diff suppressed because it is too large Load Diff

56
lfs.h
View File

@@ -45,25 +45,25 @@ typedef int32_t lfs_soff_t;
typedef uint32_t lfs_block_t; typedef uint32_t lfs_block_t;
// Maximum name size in bytes, may be redefined to reduce the size of the // Maximum name size in bytes, may be redefined to reduce the size of the
// info struct. Limited to <= 8190. Stored in superblock and must be // info struct. Limited to <= 1022. Stored in superblock and must be
// respected by other littlefs drivers. // respected by other littlefs drivers.
#ifndef LFS_NAME_MAX #ifndef LFS_NAME_MAX
#define LFS_NAME_MAX 0xff #define LFS_NAME_MAX 255
#endif #endif
// Maximum inline file size in bytes, may be redefined to limit RAM usage, // Maximum inline file size in bytes, may be redefined to limit RAM usage,
// but littlefs will automatically limit the LFS_INLINE_MAX to the // but littlefs will automatically limit the LFS_INLINE_MAX to the
// configured cache_size. Limited to <= 8190. Stored in superblock and must // configured cache_size. Limited to <= 1022. Stored in superblock and must
// be respected by other littlefs drivers. // be respected by other littlefs drivers.
#ifndef LFS_INLINE_MAX #ifndef LFS_INLINE_MAX
#define LFS_INLINE_MAX 0x1ffe #define LFS_INLINE_MAX 1022
#endif #endif
// Maximum size of custom attributes in bytes, may be redefined, but there is // Maximum size of custom attributes in bytes, may be redefined, but there is
// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 8190. Stored // no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 1022. Stored
// in superblock and must be respected by other littlefs drivers. // in superblock and must be respected by other littlefs drivers.
#ifndef LFS_ATTR_MAX #ifndef LFS_ATTR_MAX
#define LFS_ATTR_MAX 0x1ffe #define LFS_ATTR_MAX 1022
#endif #endif
// Maximum size of a file in bytes, may be redefined to limit to support other // Maximum size of a file in bytes, may be redefined to limit to support other
@@ -98,31 +98,33 @@ enum lfs_error {
// File types // File types
enum lfs_type { enum lfs_type {
// file types // file types
LFS_TYPE_REG = 0x011, LFS_TYPE_REG = 0x001,
LFS_TYPE_DIR = 0x010, LFS_TYPE_DIR = 0x002,
// internally used types // internally used types
LFS_TYPE_USERATTR = 0x100, LFS_TYPE_SPLICE = 0x400,
LFS_TYPE_CREATE = 0x000, LFS_TYPE_NAME = 0x000,
LFS_TYPE_DELETE = 0x020, LFS_TYPE_STRUCT = 0x200,
LFS_TYPE_STRUCT = 0x040, LFS_TYPE_USERATTR = 0x300,
LFS_TYPE_TAIL = 0x080, LFS_TYPE_FROM = 0x100,
LFS_TYPE_SOFTTAIL = 0x080, LFS_TYPE_TAIL = 0x600,
LFS_TYPE_HARDTAIL = 0x081, LFS_TYPE_GLOBALS = 0x700,
LFS_TYPE_CRC = 0x0a0, LFS_TYPE_CRC = 0x500,
LFS_TYPE_SUPERBLOCK = 0x001,
LFS_TYPE_GLOBALS = 0x0e0,
LFS_TYPE_DIRSTRUCT = 0x040, // internally used type specializations
LFS_TYPE_INLINESTRUCT = 0x041, LFS_TYPE_CREATE = 0x401,
LFS_TYPE_CTZSTRUCT = 0x042, LFS_TYPE_DELETE = 0x4ff,
LFS_TYPE_SUPERBLOCK = 0x0ff,
LFS_TYPE_DIRSTRUCT = 0x200,
LFS_TYPE_CTZSTRUCT = 0x202,
LFS_TYPE_INLINESTRUCT = 0x201,
LFS_TYPE_SOFTTAIL = 0x600,
LFS_TYPE_HARDTAIL = 0x601,
LFS_TYPE_MOVESTATE = 0x7ff,
// internal chip sources // internal chip sources
LFS_TYPE_FROM = 0x060, LFS_FROM_MOVE = 0x101,
LFS_FROM_MEM = 0x000, LFS_FROM_USERATTRS = 0x102,
LFS_FROM_DISK = 0x200,
LFS_FROM_MOVE = 0x061,
LFS_FROM_USERATTRS = 0x062,
}; };
// File open flags // File open flags
@@ -314,8 +316,8 @@ typedef struct lfs_cache {
typedef struct lfs_mdir { typedef struct lfs_mdir {
lfs_block_t pair[2]; lfs_block_t pair[2];
uint32_t rev; uint32_t rev;
uint32_t etag;
lfs_off_t off; lfs_off_t off;
uint32_t etag;
uint16_t count; uint16_t count;
bool erased; bool erased;
bool split; bool split;

2
tests/corrupt.py
View File

@@ -19,7 +19,7 @@ def corrupt(block):
break break
tag ^= ntag tag ^= ntag
size = (tag & 0x1fff) if (tag & 0x1fff) != 0x1fff else 0 size = (tag & 0x3ff) if (tag & 0x3ff) != 0x3ff else 0
file.seek(size, os.SEEK_CUR) file.seek(size, os.SEEK_CUR)
# lob off last 3 bytes # lob off last 3 bytes

54
tests/debug.py
View File

@@ -4,23 +4,29 @@ import struct
import binascii import binascii
TYPES = { TYPES = {
(0x1ff, 0x011): 'create reg', (0x700, 0x400): 'splice',
(0x1ff, 0x010): 'create dir', (0x7ff, 0x401): 'create',
(0x1ff, 0x001): 'superblock', (0x7ff, 0x4ff): 'delete',
(0x1ff, 0x020): 'delete', (0x700, 0x000): 'name',
(0x1f0, 0x0e0): 'globals', (0x7ff, 0x001): 'name reg',
(0x1ff, 0x080): 'tail soft', (0x7ff, 0x002): 'name dir',
(0x1ff, 0x081): 'tail hard', (0x7ff, 0x0ff): 'name superblock',
(0x1f0, 0x0a0): 'crc', (0x700, 0x200): 'struct',
(0x1ff, 0x040): 'struct dir', (0x7ff, 0x200): 'struct dir',
(0x1ff, 0x041): 'struct inline', (0x7ff, 0x202): 'struct ctz',
(0x1ff, 0x042): 'struct ctz', (0x7ff, 0x201): 'struct inline',
(0x100, 0x100): 'attr', (0x700, 0x300): 'userattr',
(0x700, 0x600): 'tail',
(0x7ff, 0x600): 'tail soft',
(0x7ff, 0x601): 'tail hard',
(0x700, 0x700): 'gstate',
(0x7ff, 0x7ff): 'gstate move',
(0x700, 0x500): 'crc',
} }
def typeof(type): def typeof(type):
for prefix in range(9): for prefix in range(12):
mask = 0x1ff & ~((1 << prefix)-1) mask = 0x7ff & ~((1 << prefix)-1)
if (mask, type & mask) in TYPES: if (mask, type & mask) in TYPES:
return TYPES[mask, type & mask] + ( return TYPES[mask, type & mask] + (
' %0*x' % (prefix/4, type & ((1 << prefix)-1)) ' %0*x' % (prefix/4, type & ((1 << prefix)-1))
@@ -59,7 +65,7 @@ def main(*blocks):
print "--- %s ---" % ', '.join(v for _,v in sorted(versions, reverse=True)) print "--- %s ---" % ', '.join(v for _,v in sorted(versions, reverse=True))
# go through each tag, print useful information # go through each tag, print useful information
print "%-4s %-8s %-14s %3s %3s %s" % ( print "%-4s %-8s %-14s %3s %4s %s" % (
'off', 'tag', 'type', 'id', 'len', 'dump') 'off', 'tag', 'type', 'id', 'len', 'dump')
tag = 0xffffffff tag = 0xffffffff
@@ -75,26 +81,26 @@ def main(*blocks):
tag ^= ntag tag ^= ntag
off += 4 off += 4
type = (tag & 0x7fc00000) >> 22 type = (tag & 0x7ff00000) >> 20
id = (tag & 0x003fe000) >> 13 id = (tag & 0x000ffc00) >> 10
size = (tag & 0x00001fff) >> 0 size = (tag & 0x000003ff) >> 0
iscrc = (type & 0x1f0) == 0x0f0 iscrc = (type & 0x700) == 0x500
data = file.read(size if size != 0x1fff else 0) data = file.read(size if size != 0x3ff else 0)
if iscrc: if iscrc:
crc = binascii.crc32(data[:4], crc) crc = binascii.crc32(data[:4], crc)
else: else:
crc = binascii.crc32(data, crc) crc = binascii.crc32(data, crc)
print '%04x: %08x %-14s %3s %3s %-23s %-8s' % ( print '%04x: %08x %-15s %3s %4s %-23s %-8s' % (
off, tag, off, tag,
typeof(type) + (' bad!' if iscrc and ~crc else ''), typeof(type) + (' bad!' if iscrc and ~crc else ''),
id if id != 0x1ff else '.', id if id != 0x3ff else '.',
size if size != 0x1fff else 'x', size if size != 0x3ff else 'x',
' '.join('%02x' % ord(c) for c in data[:8]), ' '.join('%02x' % ord(c) for c in data[:8]),
''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8])) ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8]))
off += size if size != 0x1fff else 0 off += size if size != 0x3ff else 0
if iscrc: if iscrc:
crc = 0 crc = 0
tag ^= (type & 1) << 31 tag ^= (type & 1) << 31

6
tests/test_alloc.sh
View File

@@ -253,7 +253,7 @@ tests/test.py << TEST
// find out max file size // find out max file size
lfs_mkdir(&lfs, "exhaustiondir") => 0; lfs_mkdir(&lfs, "exhaustiondir") => 0;
for (int i = 0; i < 9; i++) { for (int i = 0; i < 10; i++) {
sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i);
lfs_mkdir(&lfs, (char*)buffer) => 0; lfs_mkdir(&lfs, (char*)buffer) => 0;
} }
@@ -275,7 +275,7 @@ tests/test.py << TEST
lfs_remove(&lfs, "exhaustion") => 0; lfs_remove(&lfs, "exhaustion") => 0;
lfs_remove(&lfs, "exhaustiondir") => 0; lfs_remove(&lfs, "exhaustiondir") => 0;
for (int i = 0; i < 9; i++) { for (int i = 0; i < 10; i++) {
sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i);
lfs_remove(&lfs, (char*)buffer) => 0; lfs_remove(&lfs, (char*)buffer) => 0;
} }
@@ -287,7 +287,7 @@ tests/test.py << TEST
} }
lfs_file_sync(&lfs, &file[0]) => 0; lfs_file_sync(&lfs, &file[0]) => 0;
for (int i = 0; i < 9; i++) { for (int i = 0; i < 10; i++) {
sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i);
lfs_mkdir(&lfs, (char*)buffer) => 0; lfs_mkdir(&lfs, (char*)buffer) => 0;
} }

3
tests/test_format.sh
View File

@@ -12,8 +12,7 @@ TEST
echo "--- Basic mounting ---" echo "--- Basic mounting ---"
tests/test.py << TEST tests/test.py << TEST
lfs_format(&lfs, &cfg) => 0; lfs_format(&lfs, &cfg) => 0;
TEST
tests/test.py << TEST
lfs_mount(&lfs, &cfg) => 0; lfs_mount(&lfs, &cfg) => 0;
lfs_unmount(&lfs) => 0; lfs_unmount(&lfs) => 0;
TEST TEST