527 lines
16 KiB
C
527 lines
16 KiB
C
#include "libmalloc/bitmap_alloc.h"
|
|
#include "libmalloc/common.h"
|
|
#include "util.h"
|
|
|
|
static const int BIT_AVAIL = 0;
|
|
static const int BIT_USED = 1;
|
|
|
|
/*
|
|
* Sets all elements in the cache's underlying array to 0.
|
|
*/
|
|
static inline void clear_cache(bitmap_heap_descriptor_t *heap)
|
|
{
|
|
if(heap->cache == (unsigned long*)0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
for(int i = 0; i < heap->cache_capacity; i++)
|
|
{
|
|
heap->cache[i] = 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Clears all bits in the heap's bitmap.
|
|
*/
|
|
static inline void clear_bitmap(bitmap_heap_descriptor_t *heap)
|
|
{
|
|
for(int i = 0; i < heap->bitmap_size / sizeof(*heap->bitmap); i++)
|
|
{
|
|
heap->bitmap[i] = 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Sets bit `index` in the heap's bitmap, marking the underlying block as
|
|
* available.
|
|
*/
|
|
static inline void set_bit(bitmap_heap_descriptor_t *heap, int index, int bit)
|
|
{
|
|
if(bit < heap->block_bits)
|
|
{
|
|
int bitmap_index = index / heap->blocks_in_word;
|
|
int bitmap_offset = index % heap->blocks_in_word;
|
|
unsigned long mask = (unsigned long)1 << (heap->block_bits * (bitmap_offset + 1) - 1 - bit);
|
|
heap->bitmap[bitmap_index] |= mask;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Clears bit `index` in the heap's bitmap, marking the underlying block as
|
|
* reserved.
|
|
*/
|
|
static inline void clear_bit(bitmap_heap_descriptor_t *heap, int index, int bit)
|
|
{
|
|
if(bit < heap->block_bits)
|
|
{
|
|
int bitmap_index = index / heap->blocks_in_word;
|
|
int bitmap_offset = index % heap->blocks_in_word;
|
|
unsigned long mask = ~((unsigned long)1
|
|
<< (heap->block_bits * (bitmap_offset + 1) - 1 - bit));
|
|
heap->bitmap[bitmap_index] &= mask;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Tests whether the block at bit `index` is available. If so, returns nonzero,
|
|
* else returns 0.
|
|
*/
|
|
static inline int test_bit(bitmap_heap_descriptor_t *heap, int index, int bit)
|
|
{
|
|
if(bit > (heap->block_bits - 1))
|
|
{
|
|
return 1;
|
|
}
|
|
unsigned long mask = ((unsigned long)1
|
|
<< (heap->block_bits * ((index % heap->blocks_in_word) + 1)
|
|
- 1
|
|
- bit));
|
|
return (heap->bitmap[index / heap->blocks_in_word] & mask) != 0;
|
|
}
|
|
|
|
/*
|
|
* Sets bit `index` and its buddy in the heap's bitmap, marking the underlying
|
|
* blocks as available. Operation is used while spltting a block to reserve one
|
|
* of its child blocks.
|
|
*/
|
|
static inline void set_pair(bitmap_heap_descriptor_t *heap, int index, int bit)
|
|
{
|
|
if(bit < heap->block_bits)
|
|
{
|
|
int bitmap_index = index / heap->blocks_in_word;
|
|
int bitmap_offset = index % heap->blocks_in_word;
|
|
|
|
unsigned long mask_a = (unsigned long)1 << (heap->block_bits * (bitmap_offset + 1) - 1 - bit);
|
|
unsigned long mask_b = (unsigned long)1 << (heap->block_bits * ((bitmap_offset ^ 1) + 1) - 1 - bit);
|
|
|
|
heap->bitmap[bitmap_index] |= mask_a;
|
|
heap->bitmap[bitmap_index] |= mask_b;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Clears bit `index` and its buddy in the heap's bitmap, marking the underlying
|
|
* blocks as reserved. Used when merging two child blocks into a single parent
|
|
* block.
|
|
*/
|
|
static inline void clear_pair(bitmap_heap_descriptor_t *heap, int index, int bit)
|
|
{
|
|
if(bit < heap->block_bits)
|
|
{
|
|
int bitmap_index = index / heap->blocks_in_word;
|
|
int bitmap_offset = index % heap->blocks_in_word;
|
|
|
|
unsigned long mask_a = ~((unsigned long)1 << (heap->block_bits * (bitmap_offset + 1) - 1 - bit));
|
|
unsigned long mask_b = ~((unsigned long)1 << (heap->block_bits * ((bitmap_offset ^ 1) + 1) - 1 - bit));
|
|
|
|
heap->bitmap[bitmap_index] &= mask_a;
|
|
heap->bitmap[bitmap_index] &= mask_b;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Computes the location in the cache that `index` would be stored at, if it
|
|
* were cached.
|
|
*/
|
|
static inline int cache_location_from_index(bitmap_heap_descriptor_t *heap, int index)
|
|
{
|
|
return llog2(index + 1) - llog2(heap->blocks_in_word) - 1;
|
|
}
|
|
|
|
/*
|
|
* Computes the location in the cache that blocks at the indicated height are
|
|
* cached at.
|
|
*/
|
|
static inline int cache_location_from_height(bitmap_heap_descriptor_t *heap, int height)
|
|
{
|
|
return heap->height - height - llog2(heap->blocks_in_word);
|
|
}
|
|
|
|
/*
|
|
* Returns a cached value from the desired location, removing that value from
|
|
* the cache.
|
|
*/
|
|
static inline int check_cache(bitmap_heap_descriptor_t *heap, int height)
|
|
{
|
|
if(heap->cache != (unsigned long*)0)
|
|
{
|
|
int loc = cache_location_from_height(heap, height);
|
|
unsigned long n = heap->cache[loc];
|
|
heap->cache[loc] = 0;
|
|
return n;
|
|
}
|
|
else
|
|
{
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* If space in the cache exists, stores the provided index at the appropriate
|
|
* location.
|
|
*/
|
|
static inline void store_cache(bitmap_heap_descriptor_t *heap, int index)
|
|
{
|
|
if(heap->cache != (unsigned long*)0)
|
|
{
|
|
int level = cache_location_from_index(heap, index);
|
|
if(level >= 0 && heap->cache[level] == 0)
|
|
{
|
|
heap->cache[level] = index;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* If `index` is stored in the cache, remove it. Otherwise, leave the cache
|
|
* as is.
|
|
*/
|
|
static inline void uncache(bitmap_heap_descriptor_t *heap, int index)
|
|
{
|
|
if(heap->cache != (unsigned long*)0)
|
|
{
|
|
int level = cache_location_from_index(heap, index);
|
|
if(level >= 0 && heap->cache[level] == index)
|
|
{
|
|
heap->cache[level] = 0;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Computes the bitmap index of the block at `location` and `height`.
|
|
*/
|
|
static inline int block_index(bitmap_heap_descriptor_t *heap,
|
|
unsigned long location, unsigned long height)
|
|
{
|
|
return (location / (heap->block_size * ((unsigned long)1 << height))) + (1 << (heap->height - height));
|
|
}
|
|
|
|
/*
|
|
* Marks the indicated block as unavailable, and marks its children as both
|
|
* available. Stores the right-hand child in the cache, and returns the index
|
|
* of the left-hand child. The caller is expected to either split or allocate
|
|
* the left-hand child.
|
|
*/
|
|
static inline int split_block(bitmap_heap_descriptor_t *heap, int index)
|
|
{
|
|
if(index)
|
|
{
|
|
clear_bit(heap, index, BIT_AVAIL);
|
|
index *= 2;
|
|
set_pair(heap, index, BIT_AVAIL);
|
|
store_cache(heap, index + 1);
|
|
}
|
|
return index;
|
|
}
|
|
|
|
/*
|
|
* If the buddy of the indicated block is marked as available, marks the
|
|
* indicated block and its buddy as unavailable, and marks their parent
|
|
* as available. If the buddy of the indicated block is cached, it kill be
|
|
* removed from the cache.
|
|
*
|
|
* If the buddy of the indicated block is marked as unavailable, this function
|
|
* does nothing. The block indicated by `index` is assumed to be available.
|
|
*/
|
|
static int merge_block(bitmap_heap_descriptor_t *heap, int index)
|
|
{
|
|
while(index > 1 && test_bit(heap, index ^ 1, BIT_AVAIL))
|
|
{
|
|
uncache(heap, index ^ 1);
|
|
clear_pair(heap, index, BIT_AVAIL);
|
|
index /= 2;
|
|
set_bit(heap, index, BIT_AVAIL);
|
|
}
|
|
return index;
|
|
}
|
|
|
|
/*
|
|
* Finds the index of the first available block at `height`. If no such block
|
|
* is available, recursively searches higher blocks and splits them until an
|
|
* appropriate block is available. Returns 0 if no available blocks of sufficient
|
|
* size exist.
|
|
*/
|
|
static int find_free_region(bitmap_heap_descriptor_t *heap, int height)
|
|
{
|
|
if (height > heap->height || height < 0)
|
|
{
|
|
return 0;
|
|
}
|
|
else if (height <= heap->height - ilog2(heap->blocks_in_word))
|
|
{
|
|
unsigned long cached_index = check_cache(heap, height);
|
|
if(cached_index)
|
|
{
|
|
return cached_index;
|
|
}
|
|
unsigned long start = (1 << (heap->height - height)) / heap->blocks_in_word;
|
|
unsigned long end = ((1 << (heap->height - height + 1)) / heap->blocks_in_word);
|
|
for (int index = start; index < end; index++)
|
|
{
|
|
unsigned long avail_mask = heap->bitmap[index] & heap->mask;
|
|
if (avail_mask != 0)
|
|
{
|
|
return heap->blocks_in_word * index + (__builtin_ctzl(avail_mask) / heap->block_bits);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
#if __SIZEOF_LONG__ == 8
|
|
static const unsigned long bitmasks[] = {0x00000002, 0x0000000C, 0x000000F0, 0x0000FF00, 0xFFFF0000, 0xFFFFFFFF00000000};
|
|
#else
|
|
static const unsigned long bitmasks[] = {0x00000002, 0x0000000C, 0x000000F0, 0x0000FF00, 0xFFFF0000};
|
|
#endif
|
|
int bitmask_index = heap->height - height + llog2(heap->block_bits);
|
|
if (heap->bitmap[0] & bitmasks[bitmask_index] & heap->mask)
|
|
{
|
|
return __builtin_ctzl(heap->bitmap[0] & bitmasks[bitmask_index] & heap->mask) / heap->block_bits;
|
|
}
|
|
}
|
|
return split_block(heap, find_free_region(heap, height + 1));
|
|
}
|
|
|
|
static unsigned long compute_memory_size(const memory_map_t *map)
|
|
{
|
|
// Find the last available region in the memory map.
|
|
int map_index = map->size - 1;
|
|
while(map->array[map_index].type != M_AVAILABLE)
|
|
{
|
|
map_index--;
|
|
}
|
|
|
|
return map->array[map_index].location + map->array[map_index].size;
|
|
}
|
|
|
|
static unsigned long generate_mask(unsigned long block_bits)
|
|
{
|
|
unsigned long blocks_in_word = 8 * sizeof(unsigned long) / block_bits;
|
|
unsigned long mask = 0;
|
|
for(unsigned long i = 1; i <= blocks_in_word; i++)
|
|
{
|
|
mask |= 1UL << (i * block_bits - 1UL);
|
|
}
|
|
return mask;
|
|
}
|
|
|
|
static int construct_heap_desc(bitmap_heap_descriptor_t *heap, const memory_map_t *map)
|
|
{
|
|
if(heap->block_bits == 0 || heap->block_bits > (8 * sizeof(*heap->bitmap)))
|
|
{
|
|
return -1;
|
|
}
|
|
else if(heap->block_size == 0)
|
|
{
|
|
return -1;
|
|
}
|
|
else if((1 << llog2(heap->block_bits)) != heap->block_bits)
|
|
{
|
|
return -1;
|
|
}
|
|
|
|
unsigned long memory_size = compute_memory_size(map);
|
|
heap->blocks_in_word = 8 * sizeof(*heap->bitmap) / heap->block_bits;
|
|
heap->bitmap_size = heap->block_bits * (memory_size / heap->block_size) / 4;
|
|
heap->bitmap_size = 1 << llog2(heap->bitmap_size);
|
|
heap->height = llog2(memory_size / heap->block_size);
|
|
heap->free_block_count = 0;
|
|
heap->mask = generate_mask(heap->block_bits);
|
|
|
|
if(heap->bitmap_size <= sizeof(*heap->bitmap))
|
|
{
|
|
return -1;
|
|
}
|
|
else if(heap->bitmap_size >= memory_size && heap->bitmap == (unsigned long*)0)
|
|
{
|
|
return -1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static void initialize_bitmap(bitmap_heap_descriptor_t *heap, const memory_map_t *map)
|
|
{
|
|
clear_bitmap(heap);
|
|
for(int i = 0; i < map->size; i++)
|
|
{
|
|
if(map->array[i].type != M_AVAILABLE)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
unsigned long location = (map->array[i].location + heap->block_size - 1);// & ~(heap->block_size - 1);
|
|
location -= location % heap->block_size;
|
|
unsigned long region_end = map->array[i].location + map->array[i].size;
|
|
|
|
while(location + heap->block_size <= region_end)
|
|
{
|
|
int bit_offset = (location / heap->block_size) % heap->blocks_in_word;
|
|
int bitmap_index = ((1UL << (heap->height - 0)) / heap->blocks_in_word) + (location / heap->block_size) / heap->blocks_in_word;
|
|
unsigned long chunk_size = (heap->blocks_in_word - bit_offset) * heap->block_size;
|
|
if(bit_offset == 0 && (region_end - location) >= chunk_size)
|
|
{
|
|
// Set all bits in the word
|
|
heap->bitmap[bitmap_index] = heap->mask & ~0;
|
|
heap->free_block_count += heap->blocks_in_word;
|
|
}
|
|
else if(bit_offset == 0)
|
|
{
|
|
// Set the first 'count' bits
|
|
int count = (region_end - location) / heap->block_size;
|
|
heap->bitmap[bitmap_index] |= heap->mask & ((1UL << (heap->block_bits * count)) - 1);
|
|
heap->free_block_count += count;
|
|
}
|
|
else if((region_end - location) >= chunk_size)
|
|
{
|
|
// Set all bits starting at 'bit_offset'
|
|
heap->bitmap[bitmap_index] |= heap->mask & ~((1UL << (heap->block_bits * bit_offset)) - 1);
|
|
heap->free_block_count += heap->blocks_in_word - bit_offset;
|
|
}
|
|
else
|
|
{
|
|
// Set all bits starting at 'bit_offset' up to 'count'
|
|
int count = (region_end - location) / heap->block_size;
|
|
heap->bitmap[bitmap_index] |= heap->mask & ((1UL << (heap->block_bits * count)) - 1) & ~((1UL << (heap->block_bits * bit_offset)) - 1);
|
|
heap->free_block_count += count - bit_offset;
|
|
}
|
|
|
|
// Merge 'buddies' when both available
|
|
unsigned long mask = ((1UL << (2 * heap->block_bits)) - 1) & heap->mask;
|
|
for(int j = 0; j < heap->blocks_in_word / 2; j++)
|
|
{
|
|
if((heap->bitmap[bitmap_index] & mask) == mask)
|
|
{
|
|
merge_block(heap, bitmap_index * heap->blocks_in_word + j * 2);
|
|
}
|
|
mask <<= 2 * heap->block_bits;
|
|
}
|
|
|
|
location += chunk_size;
|
|
}
|
|
}
|
|
}
|
|
|
|
unsigned long read_bit(bitmap_heap_descriptor_t *heap, unsigned long location,
|
|
unsigned long bit)
|
|
{
|
|
int index = block_index(heap, location, 0);
|
|
while(index && !test_bit(heap, index, BIT_USED))
|
|
{
|
|
index /= 2;
|
|
}
|
|
|
|
if(index)
|
|
{
|
|
return test_bit(heap, index, bit);
|
|
}
|
|
else
|
|
{
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
int write_bit(bitmap_heap_descriptor_t *heap, unsigned long location,
|
|
unsigned long bit, int value)
|
|
{
|
|
int index = block_index(heap, location, 0);
|
|
while(index && !test_bit(heap, index, BIT_USED))
|
|
{
|
|
index /= 2;
|
|
}
|
|
|
|
if(index && value)
|
|
{
|
|
set_bit(heap, index, bit);
|
|
return value;
|
|
}
|
|
else if(index)
|
|
{
|
|
clear_bit(heap, index, bit);
|
|
return value;
|
|
}
|
|
else
|
|
{
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
unsigned long reserve_region(bitmap_heap_descriptor_t *heap, unsigned long size)
|
|
{
|
|
int height = llog2((size - 1) / heap->block_size + 1);
|
|
int index = find_free_region(heap, height);
|
|
if(index)
|
|
{
|
|
clear_bit(heap, index, BIT_AVAIL);
|
|
set_bit(heap, index, BIT_USED);
|
|
heap->free_block_count -= 1 << height;
|
|
return heap->offset + (heap->block_size << height) * (index - ((unsigned long)1 << (heap->height - height)));
|
|
}
|
|
else
|
|
{
|
|
return NOMEM;
|
|
}
|
|
}
|
|
|
|
void free_region(bitmap_heap_descriptor_t *heap, unsigned long location, unsigned long size)
|
|
{
|
|
location -= heap->offset;
|
|
int height = llog2(size / heap->block_size);
|
|
int index = block_index(heap, location, height);
|
|
while(!test_bit(heap, index, BIT_USED))
|
|
{
|
|
height++;
|
|
index /= 2;
|
|
}
|
|
set_bit(heap, index, BIT_AVAIL);
|
|
clear_bit(heap, index, BIT_USED);
|
|
index = merge_block(heap, index);
|
|
store_cache(heap, index);
|
|
heap->free_block_count += 1 << height;
|
|
}
|
|
|
|
unsigned long bitmap_size(const memory_map_t *map, unsigned long block_size, unsigned long block_bits)
|
|
{
|
|
return 1UL << llog2((block_bits * compute_memory_size(map) / block_size) / 4);
|
|
}
|
|
|
|
int initialize_heap(bitmap_heap_descriptor_t *heap, memory_map_t *map, int (*mmap)(void *location, unsigned long size))
|
|
{
|
|
if(construct_heap_desc(heap, map))
|
|
{
|
|
return -1;
|
|
}
|
|
|
|
if(heap->bitmap == (unsigned long*)0)
|
|
{
|
|
int map_index = 0;
|
|
while(map->array[map_index].size < heap->bitmap_size)
|
|
{
|
|
map_index++;
|
|
if(map_index >= map->size)
|
|
{
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
heap->bitmap = (unsigned long*)(heap->offset + map->array[map_index].location);
|
|
memmap_insert_region(map, map->array[map_index].location, heap->bitmap_size, M_UNAVAILABLE);
|
|
if(mmap && mmap(heap->bitmap, heap->bitmap_size))
|
|
{
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
initialize_bitmap(heap, map);
|
|
clear_cache(heap);
|
|
return 0;
|
|
}
|