Imagelibrary/tlsf
1
0
Fork 0
mirror of https://github.com/espressif/tlsf.git synced 2025-11-16 12:34:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/move-inline-in-header' into 'idf'

Browse Source 
feat(tlsf): Create header for remaining static inline functions

See merge request espressif/tlsf!13
This commit is contained in:
Guillaume Souchere
2024-08-29 16:38:30 +08:00
parent ea82911f4c e45fc9bfbb
commit 64170ec611
5 changed files with 610 additions and 596 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
tlsf.h → include/tlsf.h
View File

@@ -9,16 +9,12 @@
#include <stddef.h>
#include <stdbool.h>
#include "tlsf_common.h"
#if defined(__cplusplus)
extern "C" {
#endif
/* tlsf_t: a TLSF structure. Can contain 1 to N pools. */
/* pool_t: a block of memory that TLSF can manage. */
typedef void* tlsf_t;
typedef void* pool_t;
/* Create/destroy a memory pool. */
tlsf_t tlsf_create(void* mem, size_t max_bytes);
tlsf_t tlsf_create_with_pool(void* mem, size_t pool_bytes, size_t max_bytes);
@@ -42,9 +38,6 @@ size_t tlsf_block_size(void* ptr);
/* Overheads/limits of internal structures. */
size_t tlsf_size(tlsf_t tlsf);
size_t tlsf_align_size(void);
size_t tlsf_block_size_min(void);
size_t tlsf_block_size_max(tlsf_t tlsf);
size_t tlsf_pool_overhead(void);
size_t tlsf_alloc_overhead(void);
@@ -65,16 +58,6 @@ void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void* user);
int tlsf_check(tlsf_t tlsf);
int tlsf_check_pool(pool_t pool);
/*!
* @brief Weak function filling the given memory with a given fill pattern.
*
* @param start: pointer to the start of the memory region to fill
* @param size: size of the memory region to fill
* @param is_free: Indicate if the pattern to use the fill the region should be
* an after free or after allocation pattern.
*/
__attribute__((weak)) void block_absorb_post_hook(void *start, size_t size, bool is_free);
/**
* @brief Weak function called on every free block of memory allowing the user to implement
* application specific checks on the memory.

0
tlsf_block_functions.h → include/tlsf_block_functions.h
View File

6
tlsf_common.h → include/tlsf_common.h
View File

@@ -7,6 +7,7 @@
#pragma once
#include <stddef.h>
#include <assert.h>
#include <stdbool.h>
#if defined(__cplusplus)
extern "C" {
@@ -44,6 +45,11 @@ enum tlsf_config
** Data structures and associated constants.
*/
/* tlsf_t: a TLSF structure. Can contain 1 to N pools. */
/* pool_t: a block of memory that TLSF can manage. */
typedef void* tlsf_t;
typedef void* pool_t;
/* A type used for casting when doing pointer arithmetic. */
typedef ptrdiff_t tlsfptr_t;

602
include/tlsf_control_functions.h Normal file
View File

@@ -0,0 +1,602 @@
/*
* SPDX-FileCopyrightText: 2024 Matthew Conte
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#pragma once
#include "tlsf_common.h"
#include "tlsf_block_functions.h"
#if defined(__cplusplus)
extern "C" {
#define tlsf_decl static inline
#else
#define tlsf_decl static inline __attribute__((always_inline))
#endif
/*
** Architecture-specific bit manipulation routines.
**
** TLSF achieves O(1) cost for malloc and free operations by limiting
** the search for a free block to a free list of guaranteed size
** adequate to fulfill the request, combined with efficient free list
** queries using bitmasks and architecture-specific bit-manipulation
** routines.
**
** Most modern processors provide instructions to count leading zeroes
** in a word, find the lowest and highest set bit, etc. These
** specific implementations will be used when available, falling back
** to a reasonably efficient generic implementation.
**
** NOTE: TLSF spec relies on ffs/fls returning value 0..31.
** ffs/fls return 1-32 by default, returning 0 for error.
*/
/*
** Detect whether or not we are building for a 32- or 64-bit (LP/LLP)
** architecture. There is no reliable portable method at compile-time.
*/
#if defined (__alpha__) || defined (__ia64__) || defined (__x86_64__) \
|| defined (_WIN64) || defined (__LP64__) || defined (__LLP64__)
#define TLSF_64BIT
#endif
/*
** gcc 3.4 and above have builtin support, specialized for architecture.
** Some compilers masquerade as gcc; patchlevel test filters them out.
*/
#if defined (__GNUC__) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) \
&& defined (__GNUC_PATCHLEVEL__)
#if defined (__SNC__)
/* SNC for Playstation 3. */
tlsf_decl int tlsf_ffs(unsigned int word)
{
const unsigned int reverse = word & (~word + 1);
const int bit = 32 - __builtin_clz(reverse);
return bit - 1;
}
#else
tlsf_decl int tlsf_ffs(unsigned int word)
{
return __builtin_ffs(word) - 1;
}
#endif
tlsf_decl int tlsf_fls(unsigned int word)
{
const int bit = word ? 32 - __builtin_clz(word) : 0;
return bit - 1;
}
#elif defined (_MSC_VER) && (_MSC_VER >= 1400) && (defined (_M_IX86) || defined (_M_X64))
/* Microsoft Visual C++ support on x86/X64 architectures. */
#include <intrin.h>
#pragma intrinsic(_BitScanReverse)
#pragma intrinsic(_BitScanForward)
tlsf_decl int tlsf_fls(unsigned int word)
{
unsigned long index;
return _BitScanReverse(&index, word) ? index : -1;
}
tlsf_decl int tlsf_ffs(unsigned int word)
{
unsigned long index;
return _BitScanForward(&index, word) ? index : -1;
}
#elif defined (_MSC_VER) && defined (_M_PPC)
/* Microsoft Visual C++ support on PowerPC architectures. */
#include <ppcintrinsics.h>
tlsf_decl int tlsf_fls(unsigned int word)
{
const int bit = 32 - _CountLeadingZeros(word);
return bit - 1;
}
tlsf_decl int tlsf_ffs(unsigned int word)
{
const unsigned int reverse = word & (~word + 1);
const int bit = 32 - _CountLeadingZeros(reverse);
return bit - 1;
}
#elif defined (__ARMCC_VERSION)
/* RealView Compilation Tools for ARM */
tlsf_decl int tlsf_ffs(unsigned int word)
{
const unsigned int reverse = word & (~word + 1);
const int bit = 32 - __clz(reverse);
return bit - 1;
}
tlsf_decl int tlsf_fls(unsigned int word)
{
const int bit = word ? 32 - __clz(word) : 0;
return bit - 1;
}
#elif defined (__ghs__)
/* Green Hills support for PowerPC */
#include <ppc_ghs.h>
tlsf_decl int tlsf_ffs(unsigned int word)
{
const unsigned int reverse = word & (~word + 1);
const int bit = 32 - __CLZ32(reverse);
return bit - 1;
}
tlsf_decl int tlsf_fls(unsigned int word)
{
const int bit = word ? 32 - __CLZ32(word) : 0;
return bit - 1;
}
#else
/* Fall back to generic implementation. */
tlsf_decl int tlsf_fls_generic(unsigned int word)
{
int bit = 32;
if (!word) bit -= 1;
if (!(word & 0xffff0000)) { word <<= 16; bit -= 16; }
if (!(word & 0xff000000)) { word <<= 8; bit -= 8; }
if (!(word & 0xf0000000)) { word <<= 4; bit -= 4; }
if (!(word & 0xc0000000)) { word <<= 2; bit -= 2; }
if (!(word & 0x80000000)) { word <<= 1; bit -= 1; }
return bit;
}
/* Implement ffs in terms of fls. */
tlsf_decl int tlsf_ffs(unsigned int word)
{
return tlsf_fls_generic(word & (~word + 1)) - 1;
}
tlsf_decl int tlsf_fls(unsigned int word)
{
return tlsf_fls_generic(word) - 1;
}
#endif
/* Possibly 64-bit version of tlsf_fls. */
#if defined (TLSF_64BIT)
tlsf_decl int tlsf_fls_sizet(size_t size)
{
int high = (int)(size >> 32);
int bits = 0;
if (high)
{
bits = 32 + tlsf_fls(high);
}
else
{
bits = tlsf_fls((int)size & 0xffffffff);
}
return bits;
}
#else
#define tlsf_fls_sizet tlsf_fls
#endif
tlsf_decl size_t align_up(size_t x, size_t align)
{
tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
return (x + (align - 1)) & ~(align - 1);
}
tlsf_decl size_t align_down(size_t x, size_t align)
{
tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
return x - (x & (align - 1));
}
tlsf_decl void* align_ptr(const void* ptr, size_t align)
{
const tlsfptr_t aligned =
(tlsf_cast(tlsfptr_t, ptr) + (align - 1)) & ~(align - 1);
tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
return tlsf_cast(void*, aligned);
}
tlsf_decl size_t tlsf_align_size(void)
{
return ALIGN_SIZE;
}
tlsf_decl size_t tlsf_block_size_min(void)
{
return block_size_min;
}
tlsf_decl size_t tlsf_block_size_max(tlsf_t tlsf)
{
if (tlsf == NULL)
{
return 0;
}
control_t* control = tlsf_cast(control_t*, tlsf);
return tlsf_cast(size_t, 1) << control->fl_index_max;
}
/*
** Adjust an allocation size to be aligned to word size, and no smaller
** than internal minimum.
*/
tlsf_decl size_t adjust_request_size(tlsf_t tlsf, size_t size, size_t align)
{
size_t adjust = 0;
if (size)
{
const size_t aligned = align_up(size, align);
/* aligned sized must not exceed block_size_max or we'll go out of bounds on sl_bitmap */
if (aligned < tlsf_block_size_max(tlsf))
{
adjust = tlsf_max(aligned, block_size_min);
}
}
return adjust;
}
/*
** TLSF utility functions. In most cases, these are direct translations of
** the documentation found in the white paper.
*/
tlsf_decl void mapping_insert(control_t* control, size_t size, int* fli, int* sli)
{
int fl, sl;
if (size < control->small_block_size)
{
/* Store small blocks in first list. */
fl = 0;
sl = tlsf_cast(int, size) / (control->small_block_size / control->sl_index_count);
}
else
{
fl = tlsf_fls_sizet(size);
sl = tlsf_cast(int, size >> (fl - control->sl_index_count_log2)) ^ (1 << control->sl_index_count_log2);
fl -= (control->fl_index_shift - 1);
}
*fli = fl;
*sli = sl;
}
/* This version rounds up to the next block size (for allocations) */
tlsf_decl void mapping_search(control_t* control, size_t* size, int* fli, int* sli)
{
if (*size >= control->small_block_size)
{
const size_t round = (1 << (tlsf_fls_sizet(*size) - control->sl_index_count_log2));
*size = align_up(*size, round);
}
mapping_insert(control, *size, fli, sli);
}
tlsf_decl block_header_t* search_suitable_block(control_t* control, int* fli, int* sli)
{
int fl = *fli;
int sl = *sli;
/*
** First, search for a block in the list associated with the given
** fl/sl index.
*/
unsigned int sl_map = control->sl_bitmap[fl] & (~0U << sl);
if (!sl_map)
{
/* No block exists. Search in the next largest first-level list. */
const unsigned int fl_map = control->fl_bitmap & (~0U << (fl + 1));
if (!fl_map)
{
/* No free blocks available, memory has been exhausted. */
return 0;
}
fl = tlsf_ffs(fl_map);
*fli = fl;
sl_map = control->sl_bitmap[fl];
}
tlsf_assert(sl_map && "internal error - second level bitmap is null");
sl = tlsf_ffs(sl_map);
*sli = sl;
/* Return the first block in the free list. */
return control->blocks[fl * control->sl_index_count + sl];
}
/* Remove a free block from the free list.*/
tlsf_decl void remove_free_block(control_t* control, block_header_t* block, int fl, int sl)
{
block_header_t* prev = block->prev_free;
block_header_t* next = block->next_free;
tlsf_assert(prev && "prev_free field can not be null");
tlsf_assert(next && "next_free field can not be null");
next->prev_free = prev;
prev->next_free = next;
/* If this block is the head of the free list, set new head. */
if (control->blocks[fl * control->sl_index_count + sl] == block)
{
control->blocks[fl * control->sl_index_count + sl] = next;
/* If the new head is null, clear the bitmap. */
if (next == &control->block_null)
{
control->sl_bitmap[fl] &= ~(1U << sl);
/* If the second bitmap is now empty, clear the fl bitmap. */
if (!control->sl_bitmap[fl])
{
control->fl_bitmap &= ~(1U << fl);
}
}
}
}
/* Insert a free block into the free block list. */
tlsf_decl void insert_free_block(control_t* control, block_header_t* block, int fl, int sl)
{
block_header_t* current = control->blocks[fl * control->sl_index_count + sl];
tlsf_assert(current && "free list cannot have a null entry");
tlsf_assert(block && "cannot insert a null entry into the free list");
block->next_free = current;
block->prev_free = &control->block_null;
current->prev_free = block;
tlsf_assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE)
&& "block not aligned properly");
/*
** Insert the new block at the head of the list, and mark the first-
** and second-level bitmaps appropriately.
*/
control->blocks[fl * control->sl_index_count + sl] = block;
control->fl_bitmap |= (1U << fl);
control->sl_bitmap[fl] |= (1U << sl);
}
/* Remove a given block from the free list. */
tlsf_decl void block_remove(control_t* control, block_header_t* block)
{
int fl, sl;
mapping_insert(control, block_size(block), &fl, &sl);
remove_free_block(control, block, fl, sl);
}
/* Insert a given block into the free list. */
tlsf_decl void block_insert(control_t* control, block_header_t* block)
{
int fl, sl;
mapping_insert(control, block_size(block), &fl, &sl);
insert_free_block(control, block, fl, sl);
}
tlsf_decl int block_can_split(block_header_t* block, size_t size)
{
return block_size(block) >= sizeof(block_header_t) + size;
}
/* Split a block into two, the second of which is free. */
tlsf_decl block_header_t* block_split(block_header_t* block, size_t size)
{
/* Calculate the amount of space left in the remaining block.
* REMINDER: remaining pointer's first field is `prev_phys_block` but this field is part of the
* previous physical block. */
block_header_t* remaining =
offset_to_block(block_to_ptr(block), size - block_header_overhead);
/* `size` passed as an argument is the first block's new size, thus, the remaining block's size
* is `block_size(block) - size`. However, the block's data must be precedeed by the data size.
* This field is NOT part of the size, so it has to be substracted from the calculation. */
const size_t remain_size = block_size(block) - (size + block_header_overhead);
tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE)
&& "remaining block not aligned properly");
tlsf_assert(block_size(block) == remain_size + size + block_header_overhead);
block_set_size(remaining, remain_size);
tlsf_assert(block_size(remaining) >= block_size_min && "block split with invalid size");
block_set_size(block, size);
block_mark_as_free(remaining);
/**
* Here is the final outcome of this function:
*
* block remaining (block_ptr + size - BHO)
* + +
* | |
* v v
* +----------------------------------------------------------------------+
* |0000| |xxxxxxxxxxxxxxxxxxxxxx|xxxx| |###########################|
* |0000| |xxxxxxxxxxxxxxxxxxxxxx|xxxx| |###########################|
* |0000| |xxxxxxxxxxxxxxxxxxxxxx|xxxx| |###########################|
* |0000| |xxxxxxxxxxxxxxxxxxxxxx|xxxx| |###########################|
* +----------------------------------------------------------------------+
* | | | |
* + +<------------------------->+ +<------------------------->
* BHO `size` (argument) bytes BHO `remain_size` bytes
*
* Where BHO = block_header_overhead,
* 0: part of the memory owned by a `block`'s previous neighbour,
* x: part of the memory owned by `block`.
* #: part of the memory owned by `remaining`.
*/
return remaining;
}
/*!
* @brief Weak function filling the given memory with a given fill pattern.
*
* @param start: pointer to the start of the memory region to fill
* @param size: size of the memory region to fill
* @param is_free: Indicate if the pattern to use the fill the region should be
* an after free or after allocation pattern.
*/
__attribute__((weak)) void block_absorb_post_hook(void *start, size_t size, bool is_free);
/* Absorb a free block's storage into an adjacent previous free block. */
tlsf_decl block_header_t* block_absorb(block_header_t* prev, block_header_t* block)
{
tlsf_assert(!block_is_last(prev) && "previous block can't be last");
/* Note: Leaves flags untouched. */
prev->size += block_size(block) + block_header_overhead;
block_link_next(prev);
if (block_absorb_post_hook != NULL)
{
block_absorb_post_hook(block, sizeof(block_header_t), POISONING_AFTER_FREE);
}
return prev;
}
/* Merge a just-freed block with an adjacent previous free block. */
tlsf_decl block_header_t* block_merge_prev(control_t* control, block_header_t* block)
{
if (block_is_prev_free(block))
{
block_header_t* prev = block_prev(block);
tlsf_assert(prev && "prev physical block can't be null");
tlsf_assert(block_is_free(prev) && "prev block is not free though marked as such");
block_remove(control, prev);
block = block_absorb(prev, block);
}
return block;
}
/* Merge a just-freed block with an adjacent free block. */
tlsf_decl block_header_t* block_merge_next(control_t* control, block_header_t* block)
{
block_header_t* next = block_next(block);
tlsf_assert(next && "next physical block can't be null");
if (block_is_free(next))
{
tlsf_assert(!block_is_last(block) && "previous block can't be last");
block_remove(control, next);
block = block_absorb(block, next);
}
return block;
}
/* Trim any trailing block space off the end of a block, return to pool. */
tlsf_decl void block_trim_free(control_t* control, block_header_t* block, size_t size)
{
tlsf_assert(block_is_free(block) && "block must be free");
if (block_can_split(block, size))
{
block_header_t* remaining_block = block_split(block, size);
block_link_next(block);
block_set_prev_free(remaining_block);
block_insert(control, remaining_block);
}
}
/* Trim any trailing block space off the end of a used block, return to pool. */
tlsf_decl void block_trim_used(control_t* control, block_header_t* block, size_t size)
{
tlsf_assert(!block_is_free(block) && "block must be used");
if (block_can_split(block, size))
{
/* If the next block is free, we must coalesce. */
block_header_t* remaining_block = block_split(block, size);
block_set_prev_used(remaining_block);
remaining_block = block_merge_next(control, remaining_block);
block_insert(control, remaining_block);
}
}
tlsf_decl block_header_t* block_trim_free_leading(control_t* control, block_header_t* block, size_t size)
{
block_header_t* remaining_block = block;
if (block_can_split(block, size))
{
/* We want to split `block` in two: the first block will be freed and the
* second block will be returned. */
remaining_block = block_split(block, size - block_header_overhead);
/* `remaining_block` is the second block, mark its predecessor (first
* block) as free. */
block_set_prev_free(remaining_block);
block_link_next(block);
/* Put back the first block into the free memory list. */
block_insert(control, block);
}
return remaining_block;
}
tlsf_decl block_header_t* block_locate_free(control_t* control, size_t* size)
{
int fl = 0, sl = 0;
block_header_t* block = 0;
if (*size)
{
mapping_search(control, size, &fl, &sl);
/*
** mapping_search can futz with the size, so for excessively large sizes it can sometimes wind up
** with indices that are off the end of the block array.
** So, we protect against that here, since this is the only callsite of mapping_search.
** Note that we don't need to check sl, since it comes from a modulo operation that guarantees it's always in range.
*/
if (fl < control->fl_index_count)
{
block = search_suitable_block(control, &fl, &sl);
}
}
if (block)
{
tlsf_assert(block_size(block) >= *size);
remove_free_block(control, block, fl, sl);
}
return block;
}
tlsf_decl void* block_prepare_used(control_t* control, block_header_t* block, size_t size)
{
void* p = 0;
if (block)
{
tlsf_assert(size && "size must be non-zero");
block_trim_free(control, block, size);
block_mark_as_used(block);
p = block_to_ptr(block);
}
return p;
}
#undef tlsf_decl
#if defined(__cplusplus)
};
#endif

579
tlsf.c
View File

@@ -10,196 +10,7 @@
#include "tlsf.h"
#include "tlsf_common.h"
#include "tlsf_block_functions.h"
#if defined(__cplusplus)
#define tlsf_decl inline
#else
#define tlsf_decl static inline __attribute__((always_inline))
#endif
/*
** Architecture-specific bit manipulation routines.
**
** TLSF achieves O(1) cost for malloc and free operations by limiting
** the search for a free block to a free list of guaranteed size
** adequate to fulfill the request, combined with efficient free list
** queries using bitmasks and architecture-specific bit-manipulation
** routines.
**
** Most modern processors provide instructions to count leading zeroes
** in a word, find the lowest and highest set bit, etc. These
** specific implementations will be used when available, falling back
** to a reasonably efficient generic implementation.
**
** NOTE: TLSF spec relies on ffs/fls returning value 0..31.
** ffs/fls return 1-32 by default, returning 0 for error.
*/
/*
** Detect whether or not we are building for a 32- or 64-bit (LP/LLP)
** architecture. There is no reliable portable method at compile-time.
*/
#if defined (__alpha__) || defined (__ia64__) || defined (__x86_64__) \
|| defined (_WIN64) || defined (__LP64__) || defined (__LLP64__)
#define TLSF_64BIT
#endif
/*
** gcc 3.4 and above have builtin support, specialized for architecture.
** Some compilers masquerade as gcc; patchlevel test filters them out.
*/
#if defined (__GNUC__) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) \
&& defined (__GNUC_PATCHLEVEL__)
#if defined (__SNC__)
/* SNC for Playstation 3. */
tlsf_decl int tlsf_ffs(unsigned int word)
{
const unsigned int reverse = word & (~word + 1);
const int bit = 32 - __builtin_clz(reverse);
return bit - 1;
}
#else
tlsf_decl int tlsf_ffs(unsigned int word)
{
return __builtin_ffs(word) - 1;
}
#endif
tlsf_decl int tlsf_fls(unsigned int word)
{
const int bit = word ? 32 - __builtin_clz(word) : 0;
return bit - 1;
}
#elif defined (_MSC_VER) && (_MSC_VER >= 1400) && (defined (_M_IX86) || defined (_M_X64))
/* Microsoft Visual C++ support on x86/X64 architectures. */
#include <intrin.h>
#pragma intrinsic(_BitScanReverse)
#pragma intrinsic(_BitScanForward)
tlsf_decl int tlsf_fls(unsigned int word)
{
unsigned long index;
return _BitScanReverse(&index, word) ? index : -1;
}
tlsf_decl int tlsf_ffs(unsigned int word)
{
unsigned long index;
return _BitScanForward(&index, word) ? index : -1;
}
#elif defined (_MSC_VER) && defined (_M_PPC)
/* Microsoft Visual C++ support on PowerPC architectures. */
#include <ppcintrinsics.h>
tlsf_decl int tlsf_fls(unsigned int word)
{
const int bit = 32 - _CountLeadingZeros(word);
return bit - 1;
}
tlsf_decl int tlsf_ffs(unsigned int word)
{
const unsigned int reverse = word & (~word + 1);
const int bit = 32 - _CountLeadingZeros(reverse);
return bit - 1;
}
#elif defined (__ARMCC_VERSION)
/* RealView Compilation Tools for ARM */
tlsf_decl int tlsf_ffs(unsigned int word)
{
const unsigned int reverse = word & (~word + 1);
const int bit = 32 - __clz(reverse);
return bit - 1;
}
tlsf_decl int tlsf_fls(unsigned int word)
{
const int bit = word ? 32 - __clz(word) : 0;
return bit - 1;
}
#elif defined (__ghs__)
/* Green Hills support for PowerPC */
#include <ppc_ghs.h>
tlsf_decl int tlsf_ffs(unsigned int word)
{
const unsigned int reverse = word & (~word + 1);
const int bit = 32 - __CLZ32(reverse);
return bit - 1;
}
tlsf_decl int tlsf_fls(unsigned int word)
{
const int bit = word ? 32 - __CLZ32(word) : 0;
return bit - 1;
}
#else
/* Fall back to generic implementation. */
tlsf_decl int tlsf_fls_generic(unsigned int word)
{
int bit = 32;
if (!word) bit -= 1;
if (!(word & 0xffff0000)) { word <<= 16; bit -= 16; }
if (!(word & 0xff000000)) { word <<= 8; bit -= 8; }
if (!(word & 0xf0000000)) { word <<= 4; bit -= 4; }
if (!(word & 0xc0000000)) { word <<= 2; bit -= 2; }
if (!(word & 0x80000000)) { word <<= 1; bit -= 1; }
return bit;
}
/* Implement ffs in terms of fls. */
tlsf_decl int tlsf_ffs(unsigned int word)
{
return tlsf_fls_generic(word & (~word + 1)) - 1;
}
tlsf_decl int tlsf_fls(unsigned int word)
{
return tlsf_fls_generic(word) - 1;
}
#endif
/* Possibly 64-bit version of tlsf_fls. */
#if defined (TLSF_64BIT)
tlsf_decl int tlsf_fls_sizet(size_t size)
{
int high = (int)(size >> 32);
int bits = 0;
if (high)
{
bits = 32 + tlsf_fls(high);
}
else
{
bits = tlsf_fls((int)size & 0xffffffff);
}
return bits;
}
#else
#define tlsf_fls_sizet tlsf_fls
#endif
#undef tlsf_decl
#include "tlsf_control_functions.h"
/*
** Static assertion mechanism.
@@ -215,374 +26,6 @@ tlsf_static_assert(sizeof(int) * CHAR_BIT == 32);
tlsf_static_assert(sizeof(size_t) * CHAR_BIT >= 32);
tlsf_static_assert(sizeof(size_t) * CHAR_BIT <= 64);
static inline __attribute__((always_inline)) size_t align_up(size_t x, size_t align)
{
tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
return (x + (align - 1)) & ~(align - 1);
}
static inline __attribute__((always_inline)) size_t align_down(size_t x, size_t align)
{
tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
return x - (x & (align - 1));
}
static inline __attribute__((always_inline)) void* align_ptr(const void* ptr, size_t align)
{
const tlsfptr_t aligned =
(tlsf_cast(tlsfptr_t, ptr) + (align - 1)) & ~(align - 1);
tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
return tlsf_cast(void*, aligned);
}
/*
** Adjust an allocation size to be aligned to word size, and no smaller
** than internal minimum.
*/
static inline __attribute__((always_inline)) size_t adjust_request_size(tlsf_t tlsf, size_t size, size_t align)
{
size_t adjust = 0;
if (size)
{
const size_t aligned = align_up(size, align);
/* aligned sized must not exceed block_size_max or we'll go out of bounds on sl_bitmap */
if (aligned < tlsf_block_size_max(tlsf))
{
adjust = tlsf_max(aligned, block_size_min);
}
}
return adjust;
}
/*
** TLSF utility functions. In most cases, these are direct translations of
** the documentation found in the white paper.
*/
static inline __attribute__((always_inline)) void mapping_insert(control_t* control, size_t size, int* fli, int* sli)
{
int fl, sl;
if (size < control->small_block_size)
{
/* Store small blocks in first list. */
fl = 0;
sl = tlsf_cast(int, size) / (control->small_block_size / control->sl_index_count);
}
else
{
fl = tlsf_fls_sizet(size);
sl = tlsf_cast(int, size >> (fl - control->sl_index_count_log2)) ^ (1 << control->sl_index_count_log2);
fl -= (control->fl_index_shift - 1);
}
*fli = fl;
*sli = sl;
}
/* This version rounds up to the next block size (for allocations) */
static inline __attribute__((always_inline)) void mapping_search(control_t* control, size_t* size, int* fli, int* sli)
{
if (*size >= control->small_block_size)
{
const size_t round = (1 << (tlsf_fls_sizet(*size) - control->sl_index_count_log2));
*size = align_up(*size, round);
}
mapping_insert(control, *size, fli, sli);
}
static inline __attribute__((always_inline)) block_header_t* search_suitable_block(control_t* control, int* fli, int* sli)
{
int fl = *fli;
int sl = *sli;
/*
** First, search for a block in the list associated with the given
** fl/sl index.
*/
unsigned int sl_map = control->sl_bitmap[fl] & (~0U << sl);
if (!sl_map)
{
/* No block exists. Search in the next largest first-level list. */
const unsigned int fl_map = control->fl_bitmap & (~0U << (fl + 1));
if (!fl_map)
{
/* No free blocks available, memory has been exhausted. */
return 0;
}
fl = tlsf_ffs(fl_map);
*fli = fl;
sl_map = control->sl_bitmap[fl];
}
tlsf_assert(sl_map && "internal error - second level bitmap is null");
sl = tlsf_ffs(sl_map);
*sli = sl;
/* Return the first block in the free list. */
return control->blocks[fl * control->sl_index_count + sl];
}
/* Remove a free block from the free list.*/
static inline __attribute__((always_inline)) void remove_free_block(control_t* control, block_header_t* block, int fl, int sl)
{
block_header_t* prev = block->prev_free;
block_header_t* next = block->next_free;
tlsf_assert(prev && "prev_free field can not be null");
tlsf_assert(next && "next_free field can not be null");
next->prev_free = prev;
prev->next_free = next;
/* If this block is the head of the free list, set new head. */
if (control->blocks[fl * control->sl_index_count + sl] == block)
{
control->blocks[fl * control->sl_index_count + sl] = next;
/* If the new head is null, clear the bitmap. */
if (next == &control->block_null)
{
control->sl_bitmap[fl] &= ~(1U << sl);
/* If the second bitmap is now empty, clear the fl bitmap. */
if (!control->sl_bitmap[fl])
{
control->fl_bitmap &= ~(1U << fl);
}
}
}
}
/* Insert a free block into the free block list. */
static inline __attribute__((always_inline)) void insert_free_block(control_t* control, block_header_t* block, int fl, int sl)
{
block_header_t* current = control->blocks[fl * control->sl_index_count + sl];
tlsf_assert(current && "free list cannot have a null entry");
tlsf_assert(block && "cannot insert a null entry into the free list");
block->next_free = current;
block->prev_free = &control->block_null;
current->prev_free = block;
tlsf_assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE)
&& "block not aligned properly");
/*
** Insert the new block at the head of the list, and mark the first-
** and second-level bitmaps appropriately.
*/
control->blocks[fl * control->sl_index_count + sl] = block;
control->fl_bitmap |= (1U << fl);
control->sl_bitmap[fl] |= (1U << sl);
}
/* Remove a given block from the free list. */
static inline __attribute__((always_inline)) void block_remove(control_t* control, block_header_t* block)
{
int fl, sl;
mapping_insert(control, block_size(block), &fl, &sl);
remove_free_block(control, block, fl, sl);
}
/* Insert a given block into the free list. */
static inline __attribute__((always_inline)) void block_insert(control_t* control, block_header_t* block)
{
int fl, sl;
mapping_insert(control, block_size(block), &fl, &sl);
insert_free_block(control, block, fl, sl);
}
static inline __attribute__((always_inline)) int block_can_split(block_header_t* block, size_t size)
{
return block_size(block) >= sizeof(block_header_t) + size;
}
/* Split a block into two, the second of which is free. */
static inline __attribute__((always_inline)) block_header_t* block_split(block_header_t* block, size_t size)
{
/* Calculate the amount of space left in the remaining block.
* REMINDER: remaining pointer's first field is `prev_phys_block` but this field is part of the
* previous physical block. */
block_header_t* remaining =
offset_to_block(block_to_ptr(block), size - block_header_overhead);
/* `size` passed as an argument is the first block's new size, thus, the remaining block's size
* is `block_size(block) - size`. However, the block's data must be precedeed by the data size.
* This field is NOT part of the size, so it has to be substracted from the calculation. */
const size_t remain_size = block_size(block) - (size + block_header_overhead);
tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE)
&& "remaining block not aligned properly");
tlsf_assert(block_size(block) == remain_size + size + block_header_overhead);
block_set_size(remaining, remain_size);
tlsf_assert(block_size(remaining) >= block_size_min && "block split with invalid size");
block_set_size(block, size);
block_mark_as_free(remaining);
/**
* Here is the final outcome of this function:
*
* block remaining (block_ptr + size - BHO)
* + +
* | |
* v v
* +----------------------------------------------------------------------+
* |0000| |xxxxxxxxxxxxxxxxxxxxxx|xxxx| |###########################|
* |0000| |xxxxxxxxxxxxxxxxxxxxxx|xxxx| |###########################|
* |0000| |xxxxxxxxxxxxxxxxxxxxxx|xxxx| |###########################|
* |0000| |xxxxxxxxxxxxxxxxxxxxxx|xxxx| |###########################|
* +----------------------------------------------------------------------+
* | | | |
* + +<------------------------->+ +<------------------------->
* BHO `size` (argument) bytes BHO `remain_size` bytes
*
* Where BHO = block_header_overhead,
* 0: part of the memory owned by a `block`'s previous neighbour,
* x: part of the memory owned by `block`.
* #: part of the memory owned by `remaining`.
*/
return remaining;
}
/* Absorb a free block's storage into an adjacent previous free block. */
static inline __attribute__((always_inline)) block_header_t* block_absorb(block_header_t* prev, block_header_t* block)
{
tlsf_assert(!block_is_last(prev) && "previous block can't be last");
/* Note: Leaves flags untouched. */
prev->size += block_size(block) + block_header_overhead;
block_link_next(prev);
if (block_absorb_post_hook != NULL)
{
block_absorb_post_hook(block, sizeof(block_header_t), POISONING_AFTER_FREE);
}
return prev;
}
/* Merge a just-freed block with an adjacent previous free block. */
static inline __attribute__((always_inline)) block_header_t* block_merge_prev(control_t* control, block_header_t* block)
{
if (block_is_prev_free(block))
{
block_header_t* prev = block_prev(block);
tlsf_assert(prev && "prev physical block can't be null");
tlsf_assert(block_is_free(prev) && "prev block is not free though marked as such");
block_remove(control, prev);
block = block_absorb(prev, block);
}
return block;
}
/* Merge a just-freed block with an adjacent free block. */
static inline __attribute__((always_inline)) block_header_t* block_merge_next(control_t* control, block_header_t* block)
{
block_header_t* next = block_next(block);
tlsf_assert(next && "next physical block can't be null");
if (block_is_free(next))
{
tlsf_assert(!block_is_last(block) && "previous block can't be last");
block_remove(control, next);
block = block_absorb(block, next);
}
return block;
}
/* Trim any trailing block space off the end of a block, return to pool. */
static inline __attribute__((always_inline)) void block_trim_free(control_t* control, block_header_t* block, size_t size)
{
tlsf_assert(block_is_free(block) && "block must be free");
if (block_can_split(block, size))
{
block_header_t* remaining_block = block_split(block, size);
block_link_next(block);
block_set_prev_free(remaining_block);
block_insert(control, remaining_block);
}
}
/* Trim any trailing block space off the end of a used block, return to pool. */
static inline __attribute__((always_inline)) void block_trim_used(control_t* control, block_header_t* block, size_t size)
{
tlsf_assert(!block_is_free(block) && "block must be used");
if (block_can_split(block, size))
{
/* If the next block is free, we must coalesce. */
block_header_t* remaining_block = block_split(block, size);
block_set_prev_used(remaining_block);
remaining_block = block_merge_next(control, remaining_block);
block_insert(control, remaining_block);
}
}
static inline __attribute__((always_inline)) block_header_t* block_trim_free_leading(control_t* control, block_header_t* block, size_t size)
{
block_header_t* remaining_block = block;
if (block_can_split(block, size))
{
/* We want to split `block` in two: the first block will be freed and the
* second block will be returned. */
remaining_block = block_split(block, size - block_header_overhead);
/* `remaining_block` is the second block, mark its predecessor (first
* block) as free. */
block_set_prev_free(remaining_block);
block_link_next(block);
/* Put back the first block into the free memory list. */
block_insert(control, block);
}
return remaining_block;
}
static inline __attribute__((always_inline)) block_header_t* block_locate_free(control_t* control, size_t* size)
{
int fl = 0, sl = 0;
block_header_t* block = 0;
if (*size)
{
mapping_search(control, size, &fl, &sl);
/*
** mapping_search can futz with the size, so for excessively large sizes it can sometimes wind up
** with indices that are off the end of the block array.
** So, we protect against that here, since this is the only callsite of mapping_search.
** Note that we don't need to check sl, since it comes from a modulo operation that guarantees it's always in range.
*/
if (fl < control->fl_index_count)
{
block = search_suitable_block(control, &fl, &sl);
}
}
if (block)
{
tlsf_assert(block_size(block) >= *size);
remove_free_block(control, block, fl, sl);
}
return block;
}
static inline __attribute__((always_inline)) void* block_prepare_used(control_t* control, block_header_t* block, size_t size)
{
void* p = 0;
if (block)
{
tlsf_assert(size && "size must be non-zero");
block_trim_free(control, block, size);
block_mark_as_used(block);
p = block_to_ptr(block);
}
return p;
}
/* Clear structure and point all empty lists at the null block. */
static control_t* control_construct(control_t* control, size_t bytes)
{
@@ -831,26 +274,6 @@ size_t tlsf_size(tlsf_t tlsf)
return control->size;
}
size_t tlsf_align_size(void)
{
return ALIGN_SIZE;
}
size_t tlsf_block_size_min(void)
{
return block_size_min;
}
size_t tlsf_block_size_max(tlsf_t tlsf)
{
if (tlsf == NULL)
{
return 0;
}
control_t* control = tlsf_cast(control_t*, tlsf);
return tlsf_cast(size_t, 1) << control->fl_index_max;
}
/*
** Overhead of the TLSF structures in a given memory block passed to
** tlsf_add_pool, equal to the overhead of a free block and the