Merge branch 'feature/dynamic-control-size' into 'idf'

tlsf: allow each heap to have different control block

See merge request espressif/tlsf!4
This commit is contained in:
Guillaume Souchere
2022-10-25 13:19:48 +08:00
3 changed files with 182 additions and 108 deletions

190
tlsf.c
View File

@@ -215,12 +215,6 @@ tlsf_static_assert(sizeof(int) * CHAR_BIT == 32);
tlsf_static_assert(sizeof(size_t) * CHAR_BIT >= 32);
tlsf_static_assert(sizeof(size_t) * CHAR_BIT <= 64);
/* SL_INDEX_COUNT must be <= number of bits in sl_bitmap's storage type. */
tlsf_static_assert(sizeof(unsigned int) * CHAR_BIT >= SL_INDEX_COUNT);
/* Ensure we've properly tuned our sizes. */
tlsf_static_assert(ALIGN_SIZE == SMALL_BLOCK_SIZE / SL_INDEX_COUNT);
static inline __attribute__((always_inline)) size_t align_up(size_t x, size_t align)
{
tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
@@ -245,7 +239,7 @@ static inline __attribute__((always_inline)) void* align_ptr(const void* ptr, si
** Adjust an allocation size to be aligned to word size, and no smaller
** than internal minimum.
*/
static inline __attribute__((always_inline)) size_t adjust_request_size(size_t size, size_t align)
static inline __attribute__((always_inline)) size_t adjust_request_size(tlsf_t tlsf, size_t size, size_t align)
{
size_t adjust = 0;
if (size)
@@ -253,7 +247,7 @@ static inline __attribute__((always_inline)) size_t adjust_request_size(size_t s
const size_t aligned = align_up(size, align);
/* aligned sized must not exceed block_size_max or we'll go out of bounds on sl_bitmap */
if (aligned < block_size_max)
if (aligned < tlsf_block_size_max(tlsf))
{
adjust = tlsf_max(aligned, block_size_min);
}
@@ -266,34 +260,34 @@ static inline __attribute__((always_inline)) size_t adjust_request_size(size_t s
** the documentation found in the white paper.
*/
static inline __attribute__((always_inline)) void mapping_insert(size_t size, int* fli, int* sli)
static inline __attribute__((always_inline)) void mapping_insert(control_t* control, size_t size, int* fli, int* sli)
{
int fl, sl;
if (size < SMALL_BLOCK_SIZE)
if (size < control->small_block_size)
{
/* Store small blocks in first list. */
fl = 0;
sl = tlsf_cast(int, size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT);
sl = tlsf_cast(int, size) / (control->small_block_size / control->sl_index_count);
}
else
{
fl = tlsf_fls_sizet(size);
sl = tlsf_cast(int, size >> (fl - SL_INDEX_COUNT_LOG2)) ^ (1 << SL_INDEX_COUNT_LOG2);
fl -= (FL_INDEX_SHIFT - 1);
sl = tlsf_cast(int, size >> (fl - control->sl_index_count_log2)) ^ (1 << control->sl_index_count_log2);
fl -= (control->fl_index_shift - 1);
}
*fli = fl;
*sli = sl;
}
/* This version rounds up to the next block size (for allocations) */
static inline __attribute__((always_inline)) void mapping_search(size_t size, int* fli, int* sli)
static inline __attribute__((always_inline)) void mapping_search(control_t* control, size_t size, int* fli, int* sli)
{
if (size >= SMALL_BLOCK_SIZE)
if (size >= control->small_block_size)
{
const size_t round = (1 << (tlsf_fls_sizet(size) - SL_INDEX_COUNT_LOG2)) - 1;
const size_t round = (1 << (tlsf_fls_sizet(size) - control->sl_index_count_log2)) - 1;
size += round;
}
mapping_insert(size, fli, sli);
mapping_insert(control, size, fli, sli);
}
static inline __attribute__((always_inline)) block_header_t* search_suitable_block(control_t* control, int* fli, int* sli)
@@ -325,7 +319,7 @@ static inline __attribute__((always_inline)) block_header_t* search_suitable_blo
*sli = sl;
/* Return the first block in the free list. */
return control->blocks[fl][sl];
return control->blocks[fl * control->sl_index_count + sl];
}
/* Remove a free block from the free list.*/
@@ -339,9 +333,9 @@ static inline __attribute__((always_inline)) void remove_free_block(control_t* c
prev->next_free = next;
/* If this block is the head of the free list, set new head. */
if (control->blocks[fl][sl] == block)
if (control->blocks[fl * control->sl_index_count + sl] == block)
{
control->blocks[fl][sl] = next;
control->blocks[fl * control->sl_index_count + sl] = next;
/* If the new head is null, clear the bitmap. */
if (next == &control->block_null)
@@ -360,7 +354,7 @@ static inline __attribute__((always_inline)) void remove_free_block(control_t* c
/* Insert a free block into the free block list. */
static inline __attribute__((always_inline)) void insert_free_block(control_t* control, block_header_t* block, int fl, int sl)
{
block_header_t* current = control->blocks[fl][sl];
block_header_t* current = control->blocks[fl * control->sl_index_count + sl];
tlsf_assert(current && "free list cannot have a null entry");
tlsf_assert(block && "cannot insert a null entry into the free list");
block->next_free = current;
@@ -373,7 +367,7 @@ static inline __attribute__((always_inline)) void insert_free_block(control_t* c
** Insert the new block at the head of the list, and mark the first-
** and second-level bitmaps appropriately.
*/
control->blocks[fl][sl] = block;
control->blocks[fl * control->sl_index_count + sl] = block;
control->fl_bitmap |= (1U << fl);
control->sl_bitmap[fl] |= (1U << sl);
}
@@ -382,7 +376,7 @@ static inline __attribute__((always_inline)) void insert_free_block(control_t* c
static inline __attribute__((always_inline)) void block_remove(control_t* control, block_header_t* block)
{
int fl, sl;
mapping_insert(block_size(block), &fl, &sl);
mapping_insert(control, block_size(block), &fl, &sl);
remove_free_block(control, block, fl, sl);
}
@@ -390,7 +384,7 @@ static inline __attribute__((always_inline)) void block_remove(control_t* contro
static inline __attribute__((always_inline)) void block_insert(control_t* control, block_header_t* block)
{
int fl, sl;
mapping_insert(block_size(block), &fl, &sl);
mapping_insert(control, block_size(block), &fl, &sl);
insert_free_block(control, block, fl, sl);
}
@@ -553,7 +547,7 @@ static inline __attribute__((always_inline)) block_header_t* block_locate_free(c
if (size)
{
mapping_search(size, &fl, &sl);
mapping_search(control, size, &fl, &sl);
/*
** mapping_search can futz with the size, so for excessively large sizes it can sometimes wind up
@@ -561,7 +555,7 @@ static inline __attribute__((always_inline)) block_header_t* block_locate_free(c
** So, we protect against that here, since this is the only callsite of mapping_search.
** Note that we don't need to check sl, since it comes from a modulo operation that guarantees it's always in range.
*/
if (fl < FL_INDEX_COUNT)
if (fl < control->fl_index_count)
{
block = search_suitable_block(control, &fl, &sl);
}
@@ -590,22 +584,67 @@ static inline __attribute__((always_inline)) void* block_prepare_used(control_t*
}
/* Clear structure and point all empty lists at the null block. */
static void control_construct(control_t* control)
static control_t* control_construct(control_t* control, size_t bytes)
{
int i, j;
// check that the requested size can at least hold the control_t. This will allow us
// to fill in the field of control_t necessary to determine the final size of
// the metadata overhead and check that the requested size can hold
// this data and at least a block of minimum size
if (bytes < sizeof(control_t))
{
return NULL;
}
/* Find the closest power of two for first layer */
control->fl_index_max = 32 - __builtin_clz(bytes);
/* Adapt second layer to the pool */
if (bytes <= 16 * 1024) control->sl_index_count_log2 = 3;
else if (bytes <= 256 * 1024) control->sl_index_count_log2 = 4;
else control->sl_index_count_log2 = 5;
control->fl_index_shift = (control->sl_index_count_log2 + ALIGN_SIZE_LOG2);
control->sl_index_count = 1 << control->sl_index_count_log2;
control->fl_index_count = control->fl_index_max - control->fl_index_shift + 1;
control->small_block_size = 1 << control->fl_index_shift;
// the total size fo the metadata overhead is the size of the control_t
// added to the size of the sl_bitmaps and the size of blocks
control->size = sizeof(control_t) + (sizeof(*control->sl_bitmap) * control->fl_index_count) +
(sizeof(*control->blocks) * (control->fl_index_count * control->sl_index_count));
// check that the requested size can hold the whole control structure and
// a small block at least
if (bytes < control->size + block_size_min)
{
return NULL;
}
control->block_null.next_free = &control->block_null;
control->block_null.prev_free = &control->block_null;
control->fl_bitmap = 0;
for (i = 0; i < FL_INDEX_COUNT; ++i)
control->sl_bitmap = align_ptr(control + 1, sizeof(*control->sl_bitmap));
control->blocks = align_ptr(control->sl_bitmap + control->fl_index_count, sizeof(*control->blocks));
/* SL_INDEX_COUNT must be <= number of bits in sl_bitmap's storage type. */
tlsf_assert(sizeof(unsigned int) * CHAR_BIT >= control->sl_index_count
&& "CHAR_BIT less than sl_index_count");
/* Ensure we've properly tuned our sizes. */
tlsf_assert(ALIGN_SIZE == control->small_block_size / control->sl_index_count); //ALIGN_SIZE does not match");
for (int i = 0; i < control->fl_index_count; ++i)
{
control->sl_bitmap[i] = 0;
for (j = 0; j < SL_INDEX_COUNT; ++j)
for (int j = 0; j < control->sl_index_count; ++j)
{
control->blocks[i][j] = &control->block_null;
control->blocks[i * control->sl_index_count + j] = &control->block_null;
}
}
return control;
}
/*
@@ -645,14 +684,14 @@ int tlsf_check(tlsf_t tlsf)
int status = 0;
/* Check that the free lists and bitmaps are accurate. */
for (i = 0; i < FL_INDEX_COUNT; ++i)
for (i = 0; i < control->fl_index_count; ++i)
{
for (j = 0; j < SL_INDEX_COUNT; ++j)
for (j = 0; j < control->sl_index_count; ++j)
{
const int fl_map = control->fl_bitmap & (1U << i);
const int sl_list = control->sl_bitmap[i];
const int sl_map = sl_list & (1U << j);
const block_header_t* block = control->blocks[i][j];
const block_header_t* block = control->blocks[i * control->sl_index_count + j];
/* Check that first- and second-level lists agree. */
if (!fl_map)
@@ -680,7 +719,7 @@ int tlsf_check(tlsf_t tlsf)
tlsf_insist(block_is_prev_free(block_next(block)) && "block should be free");
tlsf_insist(block_size(block) >= block_size_min && "block not minimum size");
mapping_insert(block_size(block), &fli, &sli);
mapping_insert(control, block_size(block), &fli, &sli);
tlsf_insist(fli == i && sli == j && "block size indexed in wrong list");
if (tlsf_check_hook != NULL)
@@ -753,13 +792,32 @@ int tlsf_check_pool(pool_t pool)
return integ.status;
}
size_t tlsf_fit_size(tlsf_t tlsf, size_t size)
{
/* because it's GoodFit, allocable size is one range lower */
if (size && tlsf != NULL)
{
size_t sl_interval;
control_t* control = tlsf_cast(control_t*, tlsf);
sl_interval = (1 << (32 - __builtin_clz(size) - 1)) / control->sl_index_count;
return size & ~(sl_interval - 1);
}
return 0;
}
/*
** Size of the TLSF structures in a given memory block passed to
** tlsf_create, equal to the size of a control_t
*/
size_t tlsf_size(void)
size_t tlsf_size(tlsf_t tlsf)
{
return sizeof(control_t);
if (tlsf == NULL)
{
return 0;
}
control_t* control = tlsf_cast(control_t*, tlsf);
return control->size;
}
size_t tlsf_align_size(void)
@@ -772,9 +830,14 @@ size_t tlsf_block_size_min(void)
return block_size_min;
}
size_t tlsf_block_size_max(void)
size_t tlsf_block_size_max(tlsf_t tlsf)
{
return block_size_max;
if (tlsf == NULL)
{
return 0;
}
control_t* control = tlsf_cast(control_t*, tlsf);
return tlsf_cast(size_t, 1) << control->fl_index_max;
}
/*
@@ -807,16 +870,16 @@ pool_t tlsf_add_pool(tlsf_t tlsf, void* mem, size_t bytes)
return 0;
}
if (pool_bytes < block_size_min || pool_bytes > block_size_max)
if (pool_bytes < block_size_min || pool_bytes > tlsf_block_size_max(tlsf))
{
#if defined (TLSF_64BIT)
printf("tlsf_add_pool: Memory size must be between 0x%x and 0x%x00 bytes.\n",
(unsigned int)(pool_overhead + block_size_min),
(unsigned int)((pool_overhead + block_size_max) / 256));
(unsigned int)((pool_overhead + tlsf_block_size_max(tlsf)) / 256));
#else
printf("tlsf_add_pool: Memory size must be between %u and %u bytes.\n",
(unsigned int)(pool_overhead + block_size_min),
(unsigned int)(pool_overhead + block_size_max));
(unsigned int)(pool_overhead + tlsf_block_size_max(tlsf)));
#endif
return 0;
}
@@ -852,7 +915,7 @@ void tlsf_remove_pool(tlsf_t tlsf, pool_t pool)
tlsf_assert(!block_is_free(block_next(block)) && "next block should not be free");
tlsf_assert(block_size(block_next(block)) == 0 && "next block size should be zero");
mapping_insert(block_size(block), &fl, &sl);
mapping_insert(control, block_size(block), &fl, &sl);
remove_free_block(control, block, fl, sl);
}
@@ -888,31 +951,38 @@ int test_ffs_fls()
}
#endif
tlsf_t tlsf_create(void* mem)
tlsf_t tlsf_create(void* mem, size_t max_bytes)
{
#if _DEBUG
if (test_ffs_fls())
{
return 0;
return NULL;
}
#endif
if (mem == NULL)
{
return NULL;
}
if (((tlsfptr_t)mem % ALIGN_SIZE) != 0)
{
printf("tlsf_create: Memory must be aligned to %u bytes.\n",
(unsigned int)ALIGN_SIZE);
return 0;
return NULL;
}
control_construct(tlsf_cast(control_t*, mem));
return tlsf_cast(tlsf_t, mem);
control_t* control_ptr = control_construct(tlsf_cast(control_t*, mem), max_bytes);
return tlsf_cast(tlsf_t, control_ptr);
}
tlsf_t tlsf_create_with_pool(void* mem, size_t bytes)
tlsf_t tlsf_create_with_pool(void* mem, size_t pool_bytes, size_t max_bytes)
{
tlsf_t tlsf = tlsf_create(mem);
tlsf_add_pool(tlsf, (char*)mem + tlsf_size(), bytes - tlsf_size());
tlsf_t tlsf = tlsf_create(mem, max_bytes ? max_bytes : pool_bytes);
if (tlsf != NULL)
{
tlsf_add_pool(tlsf, (char*)mem + tlsf_size(tlsf), pool_bytes - tlsf_size(tlsf));
}
return tlsf;
}
@@ -924,13 +994,13 @@ void tlsf_destroy(tlsf_t tlsf)
pool_t tlsf_get_pool(tlsf_t tlsf)
{
return tlsf_cast(pool_t, (char*)tlsf + tlsf_size());
return tlsf_cast(pool_t, (char*)tlsf + tlsf_size(tlsf));
}
void* tlsf_malloc(tlsf_t tlsf, size_t size)
{
control_t* control = tlsf_cast(control_t*, tlsf);
const size_t adjust = adjust_request_size(size, ALIGN_SIZE);
const size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE);
block_header_t* block = block_locate_free(control, adjust);
return block_prepare_used(control, block, adjust);
}
@@ -961,7 +1031,7 @@ void* tlsf_malloc(tlsf_t tlsf, size_t size)
void* tlsf_memalign_offs(tlsf_t tlsf, size_t align, size_t size, size_t data_offset)
{
control_t* control = tlsf_cast(control_t*, tlsf);
const size_t adjust = adjust_request_size(size, ALIGN_SIZE);
const size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE);
const size_t off_adjust = align_up(data_offset, ALIGN_SIZE);
/*
@@ -976,7 +1046,7 @@ void* tlsf_memalign_offs(tlsf_t tlsf, size_t align, size_t size, size_t data_off
/* The offset is included in both `adjust` and `gap_minimum`, so we
** need to subtract it once.
*/
const size_t size_with_gap = adjust_request_size(adjust + align + gap_minimum - off_adjust, align);
const size_t size_with_gap = adjust_request_size(tlsf, adjust + align + gap_minimum - off_adjust, align);
/*
** If alignment is less than or equal to base alignment, we're done, because
@@ -1089,7 +1159,13 @@ void* tlsf_realloc(tlsf_t tlsf, void* ptr, size_t size)
const size_t cursize = block_size(block);
const size_t combined = cursize + block_size(next) + block_header_overhead;
const size_t adjust = adjust_request_size(size, ALIGN_SIZE);
const size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE);
// if adjust if equal to 0, the size is too big
if (adjust == 0)
{
return p;
}
tlsf_assert(!block_is_free(block) && "block already marked as free");

18
tlsf.h
View File

@@ -20,8 +20,8 @@ typedef void* tlsf_t;
typedef void* pool_t;
/* Create/destroy a memory pool. */
tlsf_t tlsf_create(void* mem);
tlsf_t tlsf_create_with_pool(void* mem, size_t bytes);
tlsf_t tlsf_create(void* mem, size_t max_bytes);
tlsf_t tlsf_create_with_pool(void* mem, size_t pool_bytes, size_t max_bytes);
void tlsf_destroy(tlsf_t tlsf);
pool_t tlsf_get_pool(tlsf_t tlsf);
@@ -40,13 +40,23 @@ void tlsf_free(tlsf_t tlsf, void* ptr);
size_t tlsf_block_size(void* ptr);
/* Overheads/limits of internal structures. */
size_t tlsf_size(void);
size_t tlsf_size(tlsf_t tlsf);
size_t tlsf_align_size(void);
size_t tlsf_block_size_min(void);
size_t tlsf_block_size_max(void);
size_t tlsf_block_size_max(tlsf_t tlsf);
size_t tlsf_pool_overhead(void);
size_t tlsf_alloc_overhead(void);
/**
* @brief Return the allocable size based on the size passed
* as parameter
*
* @param tlsf Pointer to the tlsf structure
* @param size The allocation size
* @return size_t The updated allocation size
*/
size_t tlsf_fit_size(tlsf_t tlsf, size_t size);
/* Debugging. */
typedef void (*tlsf_walker)(void* ptr, size_t size, int used, void* user);
void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void* user);

View File

@@ -35,41 +35,9 @@ extern "C" {
enum tlsf_config
{
/* log2 of number of linear subdivisions of block sizes. Larger
** values require more memory in the control structure. Values of
** 4 or 5 are typical.
*/
SL_INDEX_COUNT_LOG2 = 5,
/* All allocation sizes and addresses are aligned to 4 bytes. */
ALIGN_SIZE_LOG2 = 2,
ALIGN_SIZE = (1 << ALIGN_SIZE_LOG2),
/*
** We support allocations of sizes up to (1 << FL_INDEX_MAX) bits.
** However, because we linearly subdivide the second-level lists, and
** our minimum size granularity is 4 bytes, it doesn't make sense to
** create first-level lists for sizes smaller than SL_INDEX_COUNT * 4,
** or (1 << (SL_INDEX_COUNT_LOG2 + 2)) bytes, as there we will be
** trying to split size ranges into more slots than we have available.
** Instead, we calculate the minimum threshold size, and place all
** blocks below that size into the 0th first-level list.
*/
/* Tunning the first level, we can reduce TLSF pool overhead
* in exchange of manage a pool smaller than 4GB
*/
#ifdef FL_INDEX_MAX_PLATFORM
FL_INDEX_MAX = FL_INDEX_MAX_PLATFORM,
#else
FL_INDEX_MAX = 30,
#endif
SL_INDEX_COUNT = (1 << SL_INDEX_COUNT_LOG2),
FL_INDEX_SHIFT = (SL_INDEX_COUNT_LOG2 + ALIGN_SIZE_LOG2),
FL_INDEX_COUNT = (FL_INDEX_MAX - FL_INDEX_SHIFT + 1),
SMALL_BLOCK_SIZE = (1 << FL_INDEX_SHIFT),
};
/*
@@ -92,20 +60,6 @@ typedef struct block_header_t
struct block_header_t* prev_free;
} block_header_t;
/* The TLSF control structure. */
typedef struct control_t
{
/* Empty lists point at this block to indicate they are free. */
block_header_t block_null;
/* Bitmaps for free lists. */
unsigned int fl_bitmap;
unsigned int sl_bitmap[FL_INDEX_COUNT];
/* Head of free lists. */
block_header_t* blocks[FL_INDEX_COUNT][SL_INDEX_COUNT];
} control_t;
/*
** Since block sizes are always at least a multiple of 4, the two least
** significant bits of the size field are used to store the block status:
@@ -132,7 +86,41 @@ static const size_t block_start_offset =
*/
static const size_t block_size_min =
sizeof(block_header_t) - sizeof(block_header_t*);
static const size_t block_size_max = tlsf_cast(size_t, 1) << FL_INDEX_MAX;
/* The TLSF control structure. */
typedef struct control_t
{
/* Empty lists point at this block to indicate they are free. */
block_header_t block_null;
/* Local parameter for the pool. Given the maximum
* value of each field, all the following parameters
* can fit on 4 bytes when using bitfields
*/
unsigned int fl_index_count : 5; // 5 cumulated bits
unsigned int fl_index_shift : 3; // 8 cumulated bits
unsigned int fl_index_max : 6; // 14 cumulated bits
unsigned int sl_index_count : 6; // 20 cumulated bits
/* log2 of number of linear subdivisions of block sizes. Larger
** values require more memory in the control structure. Values of
** 4 or 5 are typical.
*/
unsigned int sl_index_count_log2 : 3; // 23 cumulated bits
unsigned int small_block_size : 8; // 31 cumulated bits
/* size of the metadata ( size of control block,
* sl_bitmap and blocks )
*/
size_t size;
/* Bitmaps for free lists. */
unsigned int fl_bitmap;
unsigned int *sl_bitmap;
/* Head of free lists. */
block_header_t** blocks;
} control_t;
#if defined(__cplusplus)
};