/*--------------------------------------------------------------------------- * Slab-Alloc Memory Manager * * Written and put into the public domain by Lars Duening. * Based on the 'SMalloc' allocator written and put into the public domain by * Sean T. "Satoria" Barrett. *--------------------------------------------------------------------------- * Slab-Alloc is intended to be optimized for lpmud. This memory manager * distinguishes between two sizes of blocks: small and large. It manages * them separately in the hopes of avoiding fragmentation between them. * It expects small blocks to mostly be temporaries. * * slaballoc IS NOT THREADSAFE. And the wrong way of fixing this would be * to wrap all mem_alloc()/mem_free() calls into a mutex. The right way of doing * this would be to allocate a set of management structures for each thread * so that only the calls to system malloc()/sbrk() need to be guarded * by mutexes. * * Allocations are measured in 'word_t's which are the same size as void*. * * For MALLOC_CHECK the allocated blocks are tagged with magic words. This * costs a bit of time and memory, but is a good defense against the most * basic memory misuses. * * * -- Small Blocks -- * * Small blocks are allocations of up to (SMALL_BLOCK_NUM+1)*4 Bytes, currently * 68 Bytes. Such block are allocated from larger memory blocks ('slabs') * sized rough multiples of 4KByte. The size is determined such that the slab * holds at least 100 blocks. * * A slab consists of a header, followed by the blocks themselves. Free blocks * within a slab are linked together in a free list. When a slab is freshly * created, it is filled in from the end and the allocator keeps an external * pointer to the end of the yet unused space in the slab. * * For every possible block size, the allocator maintains three lists of * slabs: a list of fully used slabs (single-ended), a list of partially * used slabs (double-ended) and a list of fully free slabs (double-ended). * In addition, the allocator maintains for each block size a 'fresh slab' * which is only partially initialized. #ifdef MALLOC_ORDER_SLAB_FREELISTS * The partially used slabs in the lists are rougly ordered by the ascending * number of free blocks - 'roughly' meaning that whenever the freeing of * a block causes a slab block to have twice the number of free blocks than * the next one, the block is moved to its proper position in the list. * * This approach, as compared to a straight-forward FIFO handling , leads * to a better relation of fully used to partially used slabs. #endif * * When allocating a new block, the allocator first tries to allocated the * block from the freelist of the first partly filled slab. If there is none, * it satisfies the request from the fresh slab (allocating one if necessary). * * When a block is freed, the allocator determines which slab it belongs to * and links it into the freelist. If it is the first free block for the slab, * the slab is also moved at the beginning of the slab list for this block * size. If the block was the last used block in the slab, the whole slab * is tagged with a timestamp and moved into a separate list. * * The backend (and the GC) in regular intervals call mem_consolidate() which * will scan the lists of free slabs and remove those with a timestamp older * than SLAB_RETENTION_TIME (currently 60 seconds). The intention is to * keep a load-depending cache of free slabs around to avoid thrashing * in the large allocator. * * In order to determine the slab for a block, the distance to the slab begin * is stored as number (counting in words) in the size field of the block. * In addition, bit 27 (0x08000000) is set to mark the block as 'small'. * * This allocation system intents to achieve these goals: * - Allocations of small blocks of equal size are clustered for reduced * fragmentation. This applies in particular for allocations from the * free lists, which are partly decoupled from the order in which the * blocks are freed. * - No blocks of sizes not used by the driver are ever generated. * - Unused small block memory can be returned to the overall memory pool. * * -- Large Blocks -- * * Large blocks are allocated from the system - if large allocation is * too small (less than 256 KByte), the allocator allocates a 256 KByte * block and enters the 'unnecessary' extra memory into the freelist. * Large blocks are stored with boundary tags: the size field without flags * is replicated in the last word of the block. * * The free large blocks are stored in an AVL tree for fast retrieval * of best fits. The AVL structures are stored in the user area of the blocks. * #ifdef USE_AVL_FREELIST * An AVL node is created for every distinct size of a large free block; * if additional blocks of the same size are freed, they are kept in * a double-linked list hanging of the node. Intention of this setup * is to decrease the size of the AVL tree (and thus increase the * locality large block lookups) in the case of multiple blocks of the * same size. * * The double-links are not really required, as only the first block on the * freelist or the node itself are selected for allocation, but it's * algorithmically simple to implement, and we have the memory for the * second pointer. #ifdef MALLOC_ORDER_LARGE_FREELISTS * The freelist is ordered by the starting address of the blocks (with * the likely exception of the block holding the AVL node), in an attempt * to reduce high heap fragmentation. #endif #else * A new AVL node is created for every free large block. #endif * #ifdef SBRK_OK && MALLOC_SBRK * Memory is allocated from the system with sbrk() and brk(), which puts * the whole heap under our control. * * In this mode, several functions like amalloc() are compiled as malloc(), * implementing all libc memory functions. #else * malloc() is used to allocate a new block of memory. If this block borders * previous ones, the blocks are joined. * * The allocated block (modulo joints) is tagged at both ends with fake * "allocated" blocks of which cover the unallocated areas - large_malloc() * will perceive this as a fragmented heap. #endif * * -- Privilege Level -- * * Memory is managed at three privilege levels: USER, MASTER and SYSTEM. * For every level, a special memory reserve is held by the backend. * If the system runs out of memory (or if the max_malloced limit has * been reached), the following steps are taken: * * - free the user reserve, then try again. * - if MASTER privilege: free the master reserve, then try again. * - if SYSTEM privilege: free the system reserve, then try again. * - if !SYSTEM privilege: set out_of_memory and return NULL * - if SYSTEM privilege: dump the lpc backtrace and exit(2) resp. fatal(). * * If any of the reserves is freed, the gc_request flag is set. * * * -- Block Structure -- * * The 'zeroeth' word in each large block is used to hold the size of the * block (incl. all overhead). This extra word is necessary as on some * systems large blocks can span half of the address space; however, it * is necessary only for large blocks. * * The first word in each allocated block is used for the 'flag&size' field * which holds the flags for a block, plust for small blocks the size (incl. * all overhead): * * THIS_BLOCK: small: set if this block is not allocated * large: set if this block is allocated * PREV_BLOCK: large: set if the previous block is allocated * M_GC_FREE : set in allocated blocks if the GC may free this block if * found lost * M_REF : set if this block is referenced (used during a GC) * M_SMALL : set in small blocks * * Because of the PREV_BLOCK flag, all large allocations (either the * slabs for the small block allocator, or the esbrk() chunks for the * large block allocator) are initialized with a fake permanently allocatod * block at the end of the chunk, consisting of not much more than the * flag&size field with THIS_BLOCK cleared. * #ifdef MALLOC_CHECK * The second word holds the magic word, which for small blocks is determined * also by the size of the block. #endif * * When accessing these header fields, the code in general keeps a pointer * to the flag&size word. * * The user (aka payload) area of free blocks is used to manage the various * free lists. * * Small free blocks: * * The M_SIZE field in fact holds the offset to the slab holding this * block (this also holds true for the small allocated blocks). * * The first word of the user area holds the 'next' link of the free * list, which is NULL for the last block in the list. * * Large free blocks: * * The first words of the user area hold the AVL tree node structure. * * The last word in the user area holds the size of the block in words. * #ifdef HAVE_MADVISE * TODO: Not tested for in configure, not documented. #endif *--------------------------------------------------------------------------- */ #include "driver.h" #include "typedefs.h" #ifdef HAVE_MADVISE # include # define MADVISE(new,old) madvise(new,old,MADV_RANDOM) #else # define MADVISE(new,old) NOOP #endif #include "slaballoc.h" #include "mstrings.h" #include "stdstrings.h" #include "svalue.h" #ifdef MALLOC_EXT_STATISTICS #include "array.h" #include "backend.h" #endif /* MALLOC_EXT_STATISTICS */ #include "../mudlib/sys/debug_info.h" /* Defines required by the xalloc.c wrapper */ #define MEM_ALIGN (SIZEOF_CHAR_P) /* If possible, request memory using sbrk(). But if using pthreads, do * not replace malloc() with our routines, even if the system allows it, * as slaballoc is not threadsafe. */ #if defined(SBRK_OK) # ifdef MALLOC_SBRK # if MALLOC_ALIGN == 4 # define REPLACE_MALLOC # else # undef SBRK_OK # endif # else # undef SBRK_OK # endif #endif #define MEM_MAIN_THREADSAFE /* #undef NO_MEM_BLOCK_SIZE */ #define SINT (SIZEOF_CHAR_P) /* Initialiser macros for the tables */ #define INIT4 0, 0, 0, 0 #define INIT8 INIT4, INIT4 #define INIT12 INIT8, INIT4 #define INIT16 INIT8, INIT8 #define INIT24 INIT16, INIT8 #define INIT32 INIT16, INIT16 /*-------------------------------------------------------------------------*/ /* A handy macro to statically determine the number of * elements in an array. */ #define NELEM(a) (sizeof (a) / sizeof (a)[0]) /* #undef DEBUG_AVL */ /* Define this to debug the AVL tree. */ /* The extra slaballoc header fields. */ #define M_LSIZE (-1) /* (word_t) Size in word_t (large blocks only) */ #define M_SIZE (0) /* (word_t) Size in word_t, plus some flags */ #ifdef MALLOC_CHECK # define M_OVERHEAD (2) # define M_MAGIC (1) /* (word_t) The magic word */ #else # define M_OVERHEAD (1) #endif /* MALLOC_CHECK */ #define ML_OVERHEAD (M_OVERHEAD+1) /* The overhead for large blocks. */ #define M_LINK M_OVERHEAD /* Index of the 'next' link for the small free lists */ #define BLOCK_NEXT(block) ((word_t *)(block[M_LINK])) /* The 'next' link of free block , properly typed. */ #define SET_BLOCK_NEXT(block,link) ((block)[M_LINK] = (word_t)(link)) /* Set the 'next' link of free blocks. */ #define T_OVERHEAD (M_OVERHEAD + XM_OVERHEAD) /* Total overhead: it is used by slaballoc to make smart decisions * about when to split blocks. */ #define TL_OVERHEAD (ML_OVERHEAD + XM_OVERHEAD) /* Total overhead for large blocks. */ #define SLAB_RETENTION_TIME (60) /* Retention time of a fully free slab, in seconds. * If set as 0, slabs are deallocated immediately. */ #define SMALL_BLOCK_MIN (2) /* Minimum size of a small block in words */ #define SMALL_BLOCK_NUM (16) /* Number of different small block sizes. */ #define INIT_SMALL_BLOCK INIT16 /* The proper initializer macro for all tables sized SMALL_BLOCK_NUM. */ /* Derived values */ #define SMALL_BLOCK_MIN_BYTES (SMALL_BLOCK_MIN * SINT) /* Minimum payload size of a small block. */ #define SMALL_BLOCK_MAX (SMALL_BLOCK_NUM-1 + T_OVERHEAD + SMALL_BLOCK_MIN) /* The maximum size (incl. overhead) of a small block in words */ #define SMALL_BLOCK_MAX_BYTES ((SMALL_BLOCK_NUM-1 + SMALL_BLOCK_MIN) * SINT) /* Maximum payload size of a small block. */ #define DESIRED_SLAB_SIZE 0x01000 /* 4 KByte */ /* Desired slab size (or a multiple thereof). */ #ifdef SBRK_OK # define CHUNK_SIZE 0x40000 #else /* It seems to be advantagous to be just below a power of two * make sure that the resulting chunk sizes are SINT aligned. */ # define CHUNK_SIZE (0x40000 - SINT - EXTERN_MALLOC_OVERHEAD) #endif /* Bitflags for the size field: * TODO: Assumes a 32-Bit word_t. */ #define PREV_BLOCK 0x80000000 /* Previous block is allocated */ #define THIS_BLOCK 0x40000000 /* This block is allocated */ #define M_REF 0x20000000 /* Block is referenced */ #define M_GC_FREE 0x10000000 /* GC may free this block */ #define M_SMALL 0x08000000 /* Small block */ #define M_MASK 0x07ffffff /* Mask for the size, measured in word_t's */ #ifdef MALLOC_CHECK /* The magic words for free and allocated small blocks. * Every small block size should get its own magic words, but * if there are more sizes than words, the words of lower sizes * are reused. * * There are enough words for 64 different sizes. */ static word_t sfmagic[] = { 0xde8f7d2d , 0xbd89a4b6 , 0xb667bbec , 0x475fae0a , 0x39fc5582 , 0x32041f46 , 0x624a92f0 , 0x16dd36e2 , 0x7be9c1bd , 0x088aa102 , 0x3d38509b , 0x746b9fbe , 0x2d04417f , 0x775d4351 , 0x53c48d96 , 0x02b26e0b , 0x418fedcf , 0x19dbc19e , 0x78512adb , 0x1a1f5e2b , 0x307d7761 , 0x6584c1f0 , 0x24e3c36f , 0x2232310f , 0x2dac5ceb , 0x106e8b5a , 0x5a05a938 , 0x5e6392b6 , 0x66b90348 , 0x75264901 , 0x4174f402 , 0x618b18a4 , 0x3a6ab4bf , 0x3c4bc289 , 0x2657a9a9 , 0x4e68589b , 0x09648aa6 , 0x3fc489bb , 0x1c1b715c , 0x054e4c63 , 0x484f2abd , 0x5953c1f8 , 0x79b9ec22 , 0x75536c3d , 0x50b10549 , 0x4d7e79b8 , 0x7805da48 , 0x1240f318 , 0x675a3b56 , 0x70570523 , 0x2c605143 , 0x17d7b2b7 , 0x55dbc713 , 0x514414b2 , 0x3a09e3c7 , 0x038823ff , 0x61b2a00c , 0x140f8cff , 0x61ebb6b5 , 0x486ba354 , 0x2360aab8 , 0x29f6bbf8 , 0x43a08abf , 0x5fac6d41 }; static word_t samagic[] = { 0x34706efd , 0xfc2a5830 , 0x4e62041a , 0x2713945e , 0xab58d8ab , 0xa372eeab , 0x71093c08 , 0xed2e6cc9 , 0x504e65a2 , 0x1208e35b , 0x6910f7e7 , 0x1012ef5d , 0x2e2454b7 , 0x6e5f444b , 0x58621a1b , 0x077816af , 0x6819306d , 0x4db58658 , 0x58291bf8 , 0x3597aa25 , 0x45bb60a0 , 0x6a6a0f11 , 0x1cf1e57c , 0x361265c4 , 0x16ca6054 , 0x34c99833 , 0x0bee2cd7 , 0x680e7507 , 0x6ed37bfa , 0x0f7650d6 , 0x49c11513 , 0x02e308f9 , 0x7162078c , 0x122cb868 , 0x0c18defa , 0x14c2b244 , 0x3c237460 , 0x4fb969b9 , 0x746f1f85 , 0x0c71da02 , 0x61c24d14 , 0x5d80176d , 0x1c84c960 , 0x0fe6a1cc , 0x4bdf5bb8 , 0x74e6e37b , 0x175eb87b , 0x33f88c25 , 0x429c69d3 , 0x6f87d474 , 0x6990364a , 0x0857ca73 , 0x59f1e385 , 0x06821bc6 , 0x3e6a3037 , 0x70bc43d9 , 0x3b4bb3fa , 0x4a585d0f , 0x58cab8e0 , 0x2a1f2ff4 , 0x59ceade5 , 0x228bcdf4 , 0x2d0238ee , 0x4b30b571 }; #define LFMAGIC 0xdfaff2ee /* Magic word for free large blocks */ #define LAMAGIC 0xf460146e /* Magic word for allocated large blocks */ #endif /* MALLOC_CHECK */ /*-------------------------------------------------------------------------*/ /* Debugging macros */ /* Define this macro to get a log of all allocation requests which can't be * immediately satisfied from a freelist. The log is written to * gcollect_outfd. */ /* #define DEBUG_MALLOC_ALLOCS */ #ifdef DEBUG_MALLOC_ALLOCS # define ulog(s) \ write(gcollect_outfd, s, strlen(s)) # define ulog1f(s,t) \ dprintf1(gcollect_outfd, s, (p_int)(t)) # define ulog2f(s,t1,t2) \ dprintf2(gcollect_outfd, s, (p_int)(t1), (p_int)(t2)) # define ulog3f(s,t1,t2,t3) \ dprintf3(gcollect_outfd, s, (p_int)(t1), (p_int)(t2), (p_int)(t3)) # define ulog4f(s,t1,t2,t3,t4) \ dprintf4(gcollect_outfd, s, (p_int)(t1), (p_int)(t2), (p_int)(t3), (p_int)t4) #else # define ulog(s) (void)0 # define ulog1f(s,t) (void)0 # define ulog2f(s,t1,t2) (void)0 # define ulog3f(s,t1,t2,t3) (void)0 # define ulog4f(s,t1,t2,t3,t4) (void)0 #endif /*-------------------------------------------------------------------------*/ static size_t slaballoc_size; /* Size of the current mem_alloc() request. * It is used to print a diagnostic when the memory is running out. */ /* --- Small Block types --- */ /* --- struct slab_s: Slab header * This structure describes a slab, the block space follows immediately * afterwards. */ typedef struct slab_s mslab_t; struct slab_s { #ifdef MALLOC_CHECK word_t magic; /* Magic 'small' word based on .size. */ #endif size_t size; /* Block size (incl. overhead) for this slab in * bytes. * This entry is required for e.g. realloc(), * because otherwise it would be impossible * to determine the size of a block given its * address. * TODO: Split the size field of a block in half * TODO:: to hold the block size as well as the * TODO:: offset to the start of the slab. Then * TODO:: this entry can be moved into the * TODO:: slabentry_t. */ mslab_t * prev, * next; /* Link pointers for slab list. */ unsigned long numAllocated; /* Number of blocks allocated. */ word_t * freeList; /* List of free blocks. */ word_t blocks[1]; /* First word of the allocatable memory. */ }; #define SLAB_SIZE(slab, ix) \ ( sizeof(struct slab_s) - SINT + slabtable[ix].numBlocks * (slab)->size ) /* Actual size of a given slab. */ /*--- struct slabentry_s: Slab table entry * This structure manages the slabs for a given block size. */ typedef struct slabentry_s slabentry_t; struct slabentry_s { mslab_t * first; /* Head of the slab list. */ mslab_t * last; /* Last slab in the list. */ mslab_t * fullSlabs; /* Head of the full slab list. */ mslab_t * firstFree; /* Head of the free slab list. */ mslab_t * lastFree; /* Last free slab in the list. */ mslab_t * fresh; /* NULL, or the fresh slab. */ word_t * freshMem; /* NULL, or the end of the free mem in the fresh * slab. */ unsigned long numSlabs; /* Total Number of slabs for this size. */ unsigned long numFullSlabs; /* Number of full slabs in the list. */ unsigned long numFreeSlabs; /* Number of free slabs in the free list. */ unsigned long numBlocks; /* Desired number of blocks per slab. */ }; #define INIT_SLABENTRY { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } #define INIT_SLABENTRY2 INIT_SLABENTRY, INIT_SLABENTRY #define INIT_SLABENTRY4 INIT_SLABENTRY2, INIT_SLABENTRY2 #define INIT_SLABENTRY8 INIT_SLABENTRY4, INIT_SLABENTRY4 #define INIT_SLABENTRY16 INIT_SLABENTRY8, INIT_SLABENTRY8 /* --- Small Block variables --- */ static slabentry_t slabtable[SMALL_BLOCK_NUM] = { INIT_SLABENTRY16 }; /* List of slabs of the various sizes. */ /* --- Large Block variables --- */ static word_t *heap_start = NULL; /* First address on the heap. */ static word_t *heap_end = NULL; /* Current end address of the heap. */ /* --- Statistics --- */ static t_stat small_alloc_stat = {0,0}; /* Number and size of small block allocations (incl overhead). */ static t_stat small_free_stat = {0,0}; /* Number and size of small blocks deallocations (incl overhead). */ static t_stat small_slab_stat = {0,0}; /* Number and size of allocated slabs (incl. fully unused slabs). */ static t_stat small_slab_free_stat = {0,0}; /* Number and size of allocated but completely unused slabs. */ static t_stat large_free_stat = {0,0}; /* Number and size of free large blocks. */ static t_stat large_alloc_stat = {0,0}; /* Number and size of allocated large blocks. */ static t_stat large_wasted_stat = {0,0}; /* Number and size of unusable memory frags around the large blocks. */ static t_stat sbrk_stat; /* Number and size of allocated heap blocks. */ static t_stat perm_alloc_stat = {0,0}; /* Number and size of permanent allocations functions (incl overhead). This * figure is a subset of {small,large}_alloc_stat. */ static unsigned long malloc_increment_size_calls = 0; /* Number of calls to malloc_increment_size(). */ static unsigned long malloc_increment_size_success = 0; /* Number of successfull calls to malloc_increment_size(). */ static unsigned long malloc_increment_size_total = 0; /* Total memory allocated through to malloc_increment_size(). */ #ifdef USE_AVL_FREELIST static unsigned long num_avl_nodes = 0; /* Number of nodes in the AVL tree managing the large free blocks. */ #endif /* USE_AVL_FREELIST */ /*-------------------------------------------------------------------------*/ /* Forward declarations */ static char *esbrk(word_t, size_t * pExtra); static char *large_malloc(word_t, Bool); #define large_malloc_int(size, force_m) large_malloc(size, force_m) static void large_free(char *); #ifdef MALLOC_EXT_STATISTICS /*=========================================================================*/ /* EXTENDED STATISTICS */ /* Extended statistics, giving a better overview about what is allocated * how often. */ /*-------------------------------------------------------------------------*/ /* --- struct extstat_s: Statistics for one block size */ typedef struct extstat_s { unsigned long max_alloc; /* Max number of blocks allocated */ unsigned long cur_alloc; /* Current number of blocks allocated */ unsigned long max_free; /* Max number of free blocks */ unsigned long cur_free; /* Current number of free blocks */ unsigned long num_xalloc; /* Number of xalloc() requests since last avg */ unsigned long num_xfree; /* Number of xfree() requests since last avg */ double savg_xalloc; /* Sum of xalloc() requests/second */ double savg_xfree; /* Sum of xfree() requests/second */ /* On every call to mem_update_stats(), num_x{alloc,free} is averaged * into a number of requests/second and then added to savg_. The * total average is then later calculated using savg_ and the number * of mem_update_stats() calls - this way overflows are avoided. */ unsigned long tot_sm_free; /* Total number of small frees requests. */ unsigned long num_sm_free; /* Number of small free's since last avg. */ unsigned long num_smf_steps; /* Number of small free search steps since last * avg. */ double avg_sm_free; /* Avg. number of free steps/small free req. */ statistic_t stat_free; /* Weighted small free search steps statistics. */ unsigned long tot_sm_alloc; /* Total number of small allocs requests. */ unsigned long num_sm_alloc; /* Number of small alloc's since last avg. */ unsigned long num_sma_steps; /* Number of small alloc search steps since last * avg. */ double avg_sm_alloc; /* Avg. number of alloc steps/small alloc req. */ /* On every call to mem_update_stats(), num_sm_free/num_smf_steps * is averaged into avg_sm_free. Ditto for _alloc. */ } extstat_t; static unsigned long num_update_calls = 0; /* Number of mem_update_stats() calls. */ static mp_int last_update_time = 0; /* Timestamp of last call to mem_update_stats(). */ #define SIZE_EXTSTATS (SMALL_BLOCK_NUM+2) #define EXTSTAT_SLABS (SMALL_BLOCK_NUM) #define EXTSTAT_LARGE (SMALL_BLOCK_NUM+1) static extstat_t extstats[SIZE_EXTSTATS]; /* The statistics array. * [EXTSTAT_SLABS] is for the slabs combined; the semantics here are * a bit different: while cur/max give the number of slabs in the * used resp. free lists, the number of allocs/frees counts the * actual memory (de)allocations of slabs in the large pool - not * the transfers between the full and free lists. * [EXTSTAT_LARGE] is for the large blocks. */ /*-------------------------------------------------------------------------*/ static INLINE void extstat_update_max (extstat_t * pStat) { if (pStat->cur_alloc > pStat->max_alloc) pStat->max_alloc = pStat->cur_alloc; if (pStat->cur_free > pStat->max_free) pStat->max_free = pStat->cur_free; } /* extstat_update_max() */ #endif /* MALLOC_EXT_STATISTICS */ /*=========================================================================*/ /* ASSOCIATED ROUTINES */ /*-------------------------------------------------------------------------*/ static INLINE size_t mem_block_total_size (POINTER p) /* Return the size of block

(including internal overhead) in bytes. */ { word_t *q = ((word_t *) p) - M_OVERHEAD; if (q[M_SIZE] & M_SMALL) { mslab_t * slab = (mslab_t*)(q - (q[M_SIZE] & M_MASK)); return slab->size; } return q[M_LSIZE]*SINT; } /* mem_block_total_size() */ /*-------------------------------------------------------------------------*/ static INLINE size_t mem_block_size (POINTER p) /* Return the size of block

(sans internal overhead) in bytes. */ { word_t * q = ((word_t *)p) - M_OVERHEAD; if (q[M_SIZE] & M_SMALL) { return mem_block_total_size(p) - M_OVERHEAD*SINT; } return mem_block_total_size(p) - ML_OVERHEAD*SINT; } /* mem_block_size() */ /*-------------------------------------------------------------------------*/ static INLINE void mem_mark_permanent (POINTER p) /* Mark the allocated block at

as permanent, ie. it won't be subject * to the GC. */ { word_t * q = (word_t *)p; if (q[-M_OVERHEAD] & M_GC_FREE) { q[-M_OVERHEAD] &= ~M_GC_FREE; count_up(&perm_alloc_stat, mem_block_total_size(q)); } } /* mem_mark_permanent() */ /*-------------------------------------------------------------------------*/ static INLINE void mem_mark_collectable (POINTER p) /* Mark the allocated block at

as non-permant, ie. it is subject * to the GC. */ { word_t * q = (word_t *)p; if (!(q[-M_OVERHEAD] & M_GC_FREE)) { q[-M_OVERHEAD] |= (M_REF|M_GC_FREE); count_back(&perm_alloc_stat, mem_block_total_size(q)); } } /* mem_mark_collectable() */ /*-------------------------------------------------------------------------*/ static INLINE size_t mem_overhead (void) /* Return the size of each block's overhead in bytes. */ { return M_OVERHEAD*SINT; } /* mem_overhead() */ /*-------------------------------------------------------------------------*/ static INLINE unsigned short getNumBlocks (int ix) /* Return the desired number of blocks per slab for slabs in slabtable[ix]. * * The number is determined such that the slab is roughly a multiple of * DESIRED_SLAB_SIZE large, and can hold at least 100 objects. */ { unsigned short numBlocks; if (0 == (numBlocks = slabtable[ix].numBlocks)) { size_t xsize = (ix+SMALL_BLOCK_MIN+M_OVERHEAD)*SINT; #ifdef SLABALLOC_DYNAMIC_SLABS size_t minSlabObjectSize = DESIRED_SLAB_SIZE / 128; size_t sizeMultiplier = 1+ (xsize-1) / minSlabObjectSize; size_t totalSlabSize = DESIRED_SLAB_SIZE * sizeMultiplier; size_t effSlabSize = totalSlabSize - sizeof(mslab_t) - SINT; #else size_t totalSlabSize = DESIRED_SLAB_SIZE; size_t effSlabSize = totalSlabSize - sizeof(mslab_t) - SINT; #endif numBlocks = (unsigned short)(effSlabSize / xsize); slabtable[ix].numBlocks = numBlocks; } return numBlocks; } /* getNumBlocks() */ /*-------------------------------------------------------------------------*/ #ifdef MALLOC_EXT_STATISTICS void mem_update_stats (void) /* Update whatever extended statistics the allocator has. Called every * backend cycle or so to allow for the calculation of averages over time. */ { int i; double time_diff; if (!last_update_time) last_update_time = boot_time; if (current_time <= last_update_time) return; time_diff = current_time - last_update_time; for (i = 0; i < SIZE_EXTSTATS; ++i) { double sum; extstats[i].savg_xalloc += (double)extstats[i].num_xalloc / time_diff; extstats[i].savg_xfree += (double)extstats[i].num_xfree / time_diff; extstats[i].num_xalloc = 0; extstats[i].num_xfree = 0; if (extstats[i].num_sm_free) { update_statistic(& extstats[i].stat_free, extstats[i].num_smf_steps); sum = (double)extstats[i].tot_sm_free; extstats[i].tot_sm_free += extstats[i].num_sm_free; sum /= (double)extstats[i].tot_sm_free; sum *= extstats[i].avg_sm_free; sum += (double)extstats[i].num_smf_steps / (double)extstats[i].tot_sm_free; extstats[i].avg_sm_free = sum; extstats[i].num_sm_free = 0; extstats[i].num_smf_steps = 0; } if (extstats[i].num_sm_alloc) { sum = (double)extstats[i].tot_sm_alloc; extstats[i].tot_sm_alloc += extstats[i].num_sm_alloc; sum /= (double)extstats[i].tot_sm_alloc; sum *= extstats[i].avg_sm_alloc; sum += (double)extstats[i].num_sma_steps / (double)extstats[i].tot_sm_alloc; extstats[i].avg_sm_alloc = sum; extstats[i].num_sm_alloc = 0; extstats[i].num_sma_steps = 0; } } num_update_calls++; last_update_time = current_time; } /* mem_update_stats() */ #endif /* MALLOC_EXT_STATISTICS */ /*-------------------------------------------------------------------------*/ void mem_dump_data (strbuf_t *sbuf) /* For the status commands and functions: add the slaballoc statistic * to the buffer . */ { t_stat sbrk_st, perm_st, xalloc_st; #if defined(REPLACE_MALLOC) t_stat clib_st; #endif t_stat l_alloc, l_free, l_wasted; t_stat s_alloc, s_free, s_slab, s_free_slab; unsigned long s_overhead; /* Get a snapshot of the statistics - strbuf_add() might do further * allocations while we're reading them. */ #ifdef MALLOC_EXT_STATISTICS mem_update_stats(); #endif /* MALLOC_EXT_STATISTICS */ sbrk_st = sbrk_stat; #if defined(REPLACE_MALLOC) clib_st = clib_alloc_stat; #endif xalloc_st = xalloc_stat; perm_st = perm_alloc_stat; l_alloc = large_alloc_stat; l_alloc.size *= SINT; l_free = large_free_stat; l_free.size *= SINT; l_wasted = large_wasted_stat; s_alloc = small_alloc_stat; s_free = small_free_stat; s_slab = small_slab_stat; s_slab.size += s_slab.counter * M_OVERHEAD * SINT; s_free_slab = small_slab_free_stat; s_overhead = s_slab.counter * (sizeof(mslab_t) - SINT + M_OVERHEAD * SINT); # define dump_stat(str,stat) strbuf_addf(sbuf, str,stat.counter,stat.size) strbuf_add(sbuf, "Type Count Space (bytes)\n"); dump_stat("xallocs: %8lu %10lu\n\n", xalloc_st); dump_stat("sbrk requests: %8lu %10lu (a)\n",sbrk_st); dump_stat("large blocks: %8lu %10lu (b)\n",l_alloc); strbuf_addf(sbuf , "large net avail: %10lu\n" , l_alloc.size - l_alloc.counter * ML_OVERHEAD * SINT ); dump_stat("large free blocks: %8lu %10lu (c)\n",l_free); dump_stat("large wasted: %8lu %10lu (d)\n\n",l_wasted); dump_stat("small slabs: %8lu %10lu (e)\n",s_slab); dump_stat("small blocks: %8lu %10lu (f)\n",s_alloc); strbuf_addf(sbuf , "small net avail: %10lu\n" , s_alloc.size - s_alloc.counter * M_OVERHEAD * SINT ); dump_stat("small free blocks: %8lu %10lu (g)\n",s_free); dump_stat("small free slabs: %8lu %10lu \n",s_free_slab); strbuf_addf(sbuf , "small overhead: %10lu (h)\n" , s_overhead ); dump_stat("\npermanent blocks: %8lu %10lu\n", perm_st); #ifdef SBRK_OK dump_stat("clib allocations: %8lu %10lu\n", clib_st); #else strbuf_addf(sbuf, "clib allocations: n/a n/a\n"); #endif strbuf_add(sbuf, "\n"); #ifdef USE_AVL_FREELIST strbuf_addf(sbuf, "AVL nodes: %8lu -\n", num_avl_nodes); strbuf_add(sbuf, "\n"); #endif /* USE_AVL_FREELIST */ #undef dump_stat strbuf_addf(sbuf, "malloc_increment_size: calls %lu success %lu total %lu\n\n", malloc_increment_size_calls, malloc_increment_size_success, malloc_increment_size_total ); strbuf_addf(sbuf , "Total storage: (b+c+d) %10lu should equal (a) %10lu\n" , l_alloc.size + l_free.size + l_wasted.size , sbrk_st.size ); strbuf_addf(sbuf , "Total small storage: (f+g+h) %10lu should equal (e) %10lu\n" , s_alloc.size + s_free.size + s_overhead , s_slab.size ); strbuf_addf(sbuf , "Total storage in use: (b-g-h) %10lu net available: %10lu\n" , l_alloc.size - s_free.size - s_overhead , l_alloc.size - s_free.size - s_overhead - l_alloc.counter * ML_OVERHEAD * SINT - s_alloc.counter * M_OVERHEAD * SINT - xalloc_st.counter * XM_OVERHEAD_SIZE ); strbuf_addf(sbuf , "Total storage unused: (c+d+g) %10lu\n\n" , l_free.size + l_wasted.size + s_free.size ); } /* mem_dump_data() */ /*-------------------------------------------------------------------------*/ void mem_dump_extdata (strbuf_t *sbuf) /* For the status commands and functions: add the extended slaballoc statistic * to the buffer . */ { #ifdef MALLOC_EXT_STATISTICS unsigned int i; strbuf_add(sbuf, "Detailed Block Statistics:\n\n" ); for (i = 0; i < SIZE_EXTSTATS; ++i) { if (i > 0) strbuf_add(sbuf, "\n"); if (i < SMALL_BLOCK_NUM) strbuf_addf(sbuf, " Size %3u: ", (i + SMALL_BLOCK_MIN) * SINT); else if (i == EXTSTAT_SLABS) strbuf_addf(sbuf, " Slabs: "); else strbuf_addf(sbuf, " Large: "); strbuf_addf(sbuf, "Alloc: %7.1lf /s - %7lu / %7lu cur/max\n" , num_update_calls ? extstats[i].savg_xalloc / num_update_calls : 0.0 , extstats[i].cur_alloc, extstats[i].max_alloc ); strbuf_addf(sbuf, " " "Free: %7.1lf /s - %7lu / %7lu cur/max\n" , num_update_calls ? extstats[i].savg_xfree / num_update_calls : 0.0 , extstats[i].cur_free, extstats[i].max_free ); if (i < SMALL_BLOCK_NUM) { strbuf_addf(sbuf, " " "Slabs: %4lu partial, %4lu full, %4lu free = %4lu total\n" , slabtable[i].numSlabs - slabtable[i].numFullSlabs - slabtable[i].numFreeSlabs , slabtable[i].numFullSlabs , slabtable[i].numFreeSlabs , slabtable[i].numSlabs ); { /* Determined the number of objects such that the slab * as mem_alloc() does it. */ unsigned long numObjects = getNumBlocks(i); unsigned long avgNumObjects = extstats[i].cur_alloc / (slabtable[i].numSlabs - slabtable[i].numFreeSlabs); strbuf_addf(sbuf, " " "Avg. %4lu of %4lu (%6.2lf%%) objects per slab allocated\n" , avgNumObjects , numObjects , 100.0 * (double)avgNumObjects / (double)numObjects ); } #ifdef MALLOC_ORDER_SLAB_FREELISTS strbuf_addf(sbuf, " " "Small free: %7.2lf search steps / freed block - %6.1lf steps/s\n" , extstats[i].avg_sm_free , extstats[i].stat_free.weighted_avg ); #endif /* MALLOC_ORDER_LARGE_FREELISTS */ } /* if (i < SMALL_BLOCK_NUM) */ #ifdef MALLOC_ORDER_SLAB_FREELISTS if (EXTSTAT_LARGE == i) { strbuf_addf(sbuf, " " "Large free: %7.2lf search steps / freed block - %6.1lf steps/s\n" , extstats[i].avg_sm_free , extstats[i].stat_free.weighted_avg ); } #endif /* MALLOC_ORDER_LARGE_FREELISTS */ } /* Print slaballoc options */ { char * optstrings[] = { "Slaballoc options: " # if defined(MALLOC_CHECK) , "MALLOC_CHECK" # endif # if defined(MALLOC_TRACE) , "MALLOC_TRACE" # endif # if defined(MALLOC_LPC_TRACE) , "MALLOC_LPC_TRACE" # endif # if defined(MALLOC_SBRK_TRACE) , "MALLOC_SBRK_TRACE" # endif # if defined(SLABALLOC_DYNAMIC_SLABS) , "SLABALLOC_DYNAMIC_SLABS" # endif # if defined(MALLOC_ORDER_LARGE_FREELISTS) , "MALLOC_ORDER_LARGE_FREELISTS" # endif # if defined(MALLOC_ORDER_SLAB_FREELISTS) , "MALLOC_ORDER_SLAB_FREELISTS" # endif # if defined(MALLOC_EXT_STATISTICS) , "MALLOC_EXT_STATISTICS" # endif # if defined(USE_AVL_FREELIST) , "USE_AVL_FREELIST" # endif }; size_t nStrings = sizeof(optstrings) / sizeof(optstrings[0]); size_t iInitial = strlen(optstrings[0]); size_t curlen = 0; if (nStrings > 1) { strbuf_add(sbuf, "\n"); strbuf_add(sbuf, optstrings[0]); curlen = iInitial; for (i = 1; i < nStrings; i++) { curlen += strlen(optstrings[i]) + 2; if (curlen > 78) { strbuf_addf(sbuf, "\n%*s", (int)iInitial, " "); curlen = iInitial + strlen(optstrings[i]) + 2; } strbuf_add(sbuf, optstrings[i]); if (i < nStrings-1) strbuf_add(sbuf, ", "); } strbuf_add(sbuf, ".\n"); } } /* print options */ #else strbuf_add(sbuf, "No detailed blocks statistics available.\n"); #endif /* MALLOC_EXT_STATISTICS */ } /* mem_dump_extdata() */ /*-------------------------------------------------------------------------*/ void mem_dinfo_data (svalue_t *svp, int value) /* Fill in the data for debug_info(DINFO_DATA, DID_MEMORY) into the * svalue-block svp. */ { unsigned long small_overhead; #define ST_NUMBER(which,code) \ if (value == -1) svp[which].u.number = code; \ else if (value == which) svp->u.number = code if (value == -1) put_ref_string(svp+DID_MEM_NAME, STR_SLABALLOC); else if (value == DID_MEM_NAME) put_ref_string(svp, STR_SLABALLOC); small_overhead = small_slab_stat.counter * (sizeof(mslab_t) - SINT + M_OVERHEAD * SINT); ST_NUMBER(DID_MEM_SBRK, sbrk_stat.counter); ST_NUMBER(DID_MEM_SBRK_SIZE, sbrk_stat.size); ST_NUMBER(DID_MEM_LARGE, large_alloc_stat.counter); ST_NUMBER(DID_MEM_LARGE_SIZE, large_alloc_stat.size * SINT); ST_NUMBER(DID_MEM_LFREE, large_free_stat.counter); ST_NUMBER(DID_MEM_LFREE_SIZE, large_free_stat.size * SINT); ST_NUMBER(DID_MEM_LWASTED, large_wasted_stat.counter); ST_NUMBER(DID_MEM_LWASTED_SIZE, large_wasted_stat.size); ST_NUMBER(DID_MEM_SLAB, small_slab_stat.counter); ST_NUMBER(DID_MEM_SLAB_SIZE, small_slab_stat.size + small_slab_stat.counter * M_OVERHEAD * SINT); ST_NUMBER(DID_MEM_SLAB_FREE, small_slab_free_stat.counter); ST_NUMBER(DID_MEM_SLAB_FREE_SIZE, small_slab_free_stat.size); ST_NUMBER(DID_MEM_SMALL, small_alloc_stat.counter); ST_NUMBER(DID_MEM_SMALL_SIZE, small_alloc_stat.size); ST_NUMBER(DID_MEM_SFREE, small_free_stat.counter); ST_NUMBER(DID_MEM_SFREE_SIZE, small_free_stat.size); ST_NUMBER(DID_MEM_SMALL_OVERHEAD_SIZE, small_overhead); ST_NUMBER(DID_MEM_MINC_CALLS, malloc_increment_size_calls); ST_NUMBER(DID_MEM_MINC_SUCCESS, malloc_increment_size_success); ST_NUMBER(DID_MEM_MINC_SIZE, malloc_increment_size_total); #if defined(REPLACE_MALLOC) ST_NUMBER(DID_MEM_CLIB, clib_alloc_stat.counter); ST_NUMBER(DID_MEM_CLIB_SIZE, clib_alloc_stat.size); #else ST_NUMBER(DID_MEM_CLIB, 0); ST_NUMBER(DID_MEM_CLIB_SIZE, 0); #endif ST_NUMBER(DID_MEM_PERM, perm_alloc_stat.counter); ST_NUMBER(DID_MEM_PERM_SIZE, perm_alloc_stat.size); ST_NUMBER(DID_MEM_OVERHEAD, T_OVERHEAD * SINT); ST_NUMBER(DID_MEM_ALLOCATED, large_alloc_stat.size * SINT - small_free_stat.size - small_overhead); ST_NUMBER(DID_MEM_USED, large_alloc_stat.size * SINT - small_free_stat.size - small_overhead - large_alloc_stat.counter * ML_OVERHEAD * SINT - small_alloc_stat.counter * M_OVERHEAD * SINT - xalloc_stat.counter * XM_OVERHEAD_SIZE ); ST_NUMBER(DID_MEM_TOTAL_UNUSED, large_free_stat.size * SINT + large_wasted_stat.size + small_free_stat.size ); #ifdef USE_AVL_FREELIST ST_NUMBER(DID_MEM_AVL_NODES, num_avl_nodes); #else ST_NUMBER(DID_MEM_AVL_NODES, large_free_stat.counter); #endif /* USE_AVL_FREELIST */ #ifdef MALLOC_EXT_STATISTICS do { vector_t * top; /* Array holding the sub vectors */ int i; Bool deallocate = MY_FALSE; mem_update_stats(); top = allocate_array(SIZE_EXTSTATS); if (!top) break; for (i = 0; i < SIZE_EXTSTATS; ++i) { vector_t *sub = allocate_array(DID_MEM_ES_MAX); if (!sub) { deallocate = MY_TRUE; break; } put_number(&sub->item[DID_MEM_ES_MAX_ALLOC], extstats[i].max_alloc); put_number(&sub->item[DID_MEM_ES_CUR_ALLOC], extstats[i].cur_alloc); put_number(&sub->item[DID_MEM_ES_MAX_FREE], extstats[i].max_free); put_number(&sub->item[DID_MEM_ES_CUR_FREE], extstats[i].cur_free); if (num_update_calls) { put_float(&sub->item[DID_MEM_ES_AVG_XALLOC], extstats[i].savg_xalloc / num_update_calls); put_float(&sub->item[DID_MEM_ES_AVG_XFREE], extstats[i].savg_xfree / num_update_calls); } if (i < SMALL_BLOCK_NUM) { put_number(&sub->item[DID_MEM_ES_FULL_SLABS], slabtable[i].numFullSlabs); put_number(&sub->item[DID_MEM_ES_FREE_SLABS], slabtable[i].numFreeSlabs); put_number(&sub->item[DID_MEM_ES_TOTAL_SLABS], slabtable[i].numSlabs); } put_array(top->item + i, sub); } if (deallocate) { free_array(top); ST_NUMBER(DID_MEM_EXT_STATISTICS, 0); } else { if (value == -1) put_array(svp+DID_MEM_EXT_STATISTICS, top); else if (value == DID_MEM_EXT_STATISTICS) put_array(svp, top); } } while(0); #else ST_NUMBER(DID_MEM_EXT_STATISTICS, 0); #endif /* MALLOC_EXT_STATISTICS */ #undef ST_NUMBER } /* mem_dinfo_data() */ /*-------------------------------------------------------------------------*/ #ifdef CHECK_MAPPING_TOTAL mp_int available_memory(void) /* Return the amount of memory actually used by the driver. */ { return large_alloc_stat.size * SINT - small_free_stat.size - large_alloc_stat.counter * ML_OVERHEAD * SINT - small_alloc_stat.counter * M_OVERHEAD * SINT; } /* available_memory() */ #endif /* CHECK_MAPPING_TOTAL */ /*=========================================================================*/ /* SMALL BLOCKS */ /*-------------------------------------------------------------------------*/ #define SIZE_INDEX(size) \ ((size)/SINT - T_OVERHEAD - SMALL_BLOCK_MIN) /* Index to the proper array entry for a small * block of (including overhead). */ #define SIZE_MOD_INDEX(size, table) \ (((size)/SINT - T_OVERHEAD - SMALL_BLOCK_MIN) % NELEM(table)) /* Index to the proper array entry for a small * block of (including overhead), limited to the size of . */ #define INDEX_SIZE(ix) \ ((ix + T_OVERHEAD + SMALL_BLOCK_MIN) * SINT) /* Determine block (including overhead) for a slab held * in table index . */ /*-------------------------------------------------------------------------*/ /* Macro MAKE_SMALL_CHECK(block, size) * Macro MAKE_SMALL_CHECK_UNCHECKED(block, size) * If MALLOC_CHECK is defined, fill in the CHECK information * in the small block of size (in bytes incl overhead). * The _UNCHECKED macro is like the basic macro, except that it doesn't * check the M_MAGIC word before setting it. */ #ifdef MALLOC_CHECK # define MAKE_SMALL_CHECK(block, size) do { \ if (block[M_MAGIC] != sfmagic[SIZE_MOD_INDEX(size, sfmagic)] ) \ { \ in_malloc = 0; \ fatal("allocation from free list for %lu bytes: " \ "block %p (user %p) magic match failed, " \ "expected %08lx, found %08lx\n" \ , (unsigned long) size, block, block+T_OVERHEAD \ , sfmagic[SIZE_MOD_INDEX(size, sfmagic)] \ , block[M_MAGIC]); \ } \ block[M_MAGIC] = samagic[SIZE_MOD_INDEX(size, samagic)]; \ } while(0) # define MAKE_SMALL_CHECK_UNCHECKED(block, size) do { \ block[M_MAGIC] = samagic[SIZE_MOD_INDEX(size, samagic)]; \ } while(0) #else # define MAKE_SMALL_CHECK(block, size) (void)0 # define MAKE_SMALL_CHECK_UNCHECKED(block, size) (void)0 #endif /*-------------------------------------------------------------------------*/ #ifdef MALLOC_ORDER_SLAB_FREELISTS static void keep_small_order (mslab_t * slab, int ix) /* Slab is a partially used slab in slabtable entry . * Check it's position and move it to a more approriate place * if necessary. */ { /* The threshold is if this slab has twice more free blocks than the next * one. */ if (slab->next && (slabtable[ix].numBlocks - slab->numAllocated) > 2 * (slabtable[ix].numBlocks - slab->next->numAllocated) ) { /* Find new insertion position (pointing to the slab after * which to insert). * Since keeping a total order has linear complexity, we just * keep a local order by switch the current slab and its successor. * Kinda like a fuzzy incremental bubble sort :-) */ mslab_t * point = slab->next; #ifdef MALLOC_EXT_STATISTICS extstats[ix].num_smf_steps++; #endif /* MALLOC_EXT_STATISTICS */ ulog4f("slaballoc: need reorder: this %x free %d, next %x free %d\n" , slab, slabtable[ix].numBlocks - slab->numAllocated , slab->next, slabtable[ix].numBlocks - slab->next->numAllocated ); #ifdef DEBUG_MALLOC_ALLOCS { mslab_t * p = slabtable[ix].first; ulog3f("slaballoc: [%d]: %x (%d)" , ix, p, slabtable[ix].numBlocks - p->numAllocated ); while (p->next != NULL) { mslab_t * prev = p; p = p->next; ulog2f(" -> %x (%d)" , p, slabtable[ix].numBlocks - p->numAllocated ); if (prev != p->prev) fatal("Inconsistent freelist pointer: prev %p should be %p\n" , p->prev, prev ); } ulog("\n"); } #endif while (point->next && point->next->numAllocated > slab->numAllocated ) { point = point->next; #ifdef MALLOC_EXT_STATISTICS extstats[ix].num_smf_steps++; #endif /* MALLOC_EXT_STATISTICS */ } #ifdef DEBUG_MALLOC_ALLOCS if (point->next) ulog4f("slaballoc: point %x free %d, point->next %x free %d\n" , point, slabtable[ix].numBlocks - point->numAllocated , point->next, slabtable[ix].numBlocks - point->next->numAllocated ); else ulog2f("slaballoc: point %x free %d, no next\n" , point, slabtable[ix].numBlocks - point->numAllocated ); #endif /* DEBUG_MALLOC_ALLOCS */ /* Unlink slab */ if (slab->next) slab->next->prev = slab->prev; if (slab->prev) slab->prev->next = slab->next; if (slabtable[ix].first == slab) slabtable[ix].first = slab->next; if (slabtable[ix].last == slab) slabtable[ix].last = slab->prev; /* Relink slab */ slab->next = point->next; slab->prev = point; if (point->next) point->next->prev = slab; point->next = slab; if (slabtable[ix].last == point) slabtable[ix].last = slab; #ifdef DEBUG_MALLOC_ALLOCS { mslab_t * p = slabtable[ix].first; ulog3f("slaballoc: [%d]: %x (%d)" , ix, p, slabtable[ix].numBlocks - p->numAllocated ); while (p->next != NULL) { mslab_t * prev = p; p = p->next; ulog2f(" -> %x (%d)" , p, slabtable[ix].numBlocks - p->numAllocated ); if (prev != p->prev) fatal("Inconsistent freelist pointer: prev %p should be %p\n" , p->prev, prev ); } ulog("\n"); } #endif } #ifdef DEBUG_MALLOC_ALLOCS else if (slab->next) ulog4f("slaballoc: no reorder: this %x free %d, next %x free %d\n" , slab, slabtable[ix].numBlocks - slab->numAllocated , slab->next, slabtable[ix].numBlocks - slab->next->numAllocated ); else ulog2f("slaballoc: no reorder: this %x free %d, no next\n" , slab, slabtable[ix].numBlocks - slab->numAllocated ); #endif /* DEBUG_MALLOC_ALLOCS */ } /* keep_small_order() */ #endif /* MALLOC_ORDER_SLAB_FREELISTS */ /*-------------------------------------------------------------------------*/ static void insert_partial_slab (mslab_t * slab, int ix) /* Insert the given into the partially-used list of slabtable entry * . * * No statistics or unlinking are handled, this is just the insertion itself. */ { ulog4f("slaballoc: insert partial slab %x into [%d]: first %x, last %x\n" , slab, ix, slabtable[ix].first, slabtable[ix].last ); if (NULL == slabtable[ix].first) { slabtable[ix].first = slabtable[ix].last = slab; slab->next = slab->prev = NULL; } else { #ifdef MALLOC_ORDER_SLAB_FREELISTS /* Insert at front and let the ordering mechanism kick in * later. */ slab->next = slabtable[ix].first; slab->prev = NULL; slabtable[ix].first->prev = slab; slabtable[ix].first = slab; keep_small_order(slab, ix); #else /* Insert at end. */ slab->prev = slabtable[ix].last; slab->next = NULL; slabtable[ix].last->next = slab; slabtable[ix].last = slab; #endif /* MALLOC_ORDER_SLAB_FREELISTS */ #if 0 /* If no order is desired: Insert at back for FIFO handling. */ slab->prev = slabtable[ix].last; slab->next = NULL; slabtable[ix].last->next = slab; slabtable[ix].last = slab; #endif } } /* insert_partial_slab() */ /*-------------------------------------------------------------------------*/ static void free_slab (mslab_t * slab, int ix) /* Free the which belongs to slabtable entry . * The slab has already been unlinked, all there is left to do is * adjusting the generic statistics and deallocating the memory. */ { ulog2f("slaballoc: deallocate slab %x [%d]\n", slab, ix); slabtable[ix].numSlabs--; count_back(&small_slab_stat, SLAB_SIZE(slab, ix)); count_back_n(&small_free_stat, slabtable[ix].numBlocks, slab->size); #ifdef MALLOC_EXT_STATISTICS extstats[SIZE_INDEX(slab->size)].cur_free -= slabtable[ix].numBlocks; extstats[EXTSTAT_SLABS].cur_free--; extstats[EXTSTAT_SLABS].num_xfree++; extstat_update_max(extstats + EXTSTAT_SLABS); #endif /* MALLOC_EXT_STATISTICS */ large_free((char*)slab); } /* free_slab() */ /*-------------------------------------------------------------------------*/ static POINTER mem_alloc (size_t size) /* Allocate a memory block for bytes at the source :. * Result is the pointer the memory block, or NULL when out of memory. */ { word_t *block = NULL; int ix; #if defined(HAVE_MADVISE) size_t orig_size = size; #endif assert_stack_gap(); slaballoc_size = size; if (size == 0) { size++; } /* TODO: For the following test, see SIZET_limits in port.h */ if (size >= ULONG_MAX - (T_OVERHEAD+SMALL_BLOCK_MIN)*SINT || size >= (M_MASK + 1 - (T_OVERHEAD+SMALL_BLOCK_MIN))*SINT ) { in_malloc = 0; if (malloc_privilege == MALLOC_SYSTEM) fatal("Malloc size exceeds numerical limit.\n"); debug_message("Malloc size exceeds numerical limit.\n"); return NULL; } if (in_malloc++) { in_malloc = 0; writes(1,"Multiple threads in slaballoc()\n"); fatal("Multiple threads in slaballoc()\n"); /* NOTREACHED */ return NULL; } if (size > SMALL_BLOCK_MAX_BYTES + XM_OVERHEAD_SIZE) { void * rc = large_malloc(size, MY_FALSE); in_malloc--; return rc; } /* --- It's a small block --- */ /* Get the block size rounded to the next multiple of a word * and with the overhead. */ if (size < SMALL_BLOCK_MIN_BYTES + XM_OVERHEAD_SIZE) size = SMALL_BLOCK_MIN_BYTES + XM_OVERHEAD_SIZE; ulog2f("slaballoc: alloc (%d + %d)", size, M_OVERHEAD*SINT); size = (size+M_OVERHEAD*SINT+SINT-1) & ~(SINT-1); ix = SIZE_INDEX(size); ulog2f(" %d bytes -> [%d]\n", size, ix); /* Update statistics */ count_up(&small_alloc_stat,size); #ifdef MALLOC_EXT_STATISTICS extstats[SIZE_INDEX(size)].num_xalloc++; extstats[SIZE_INDEX(size)].cur_alloc++; #endif /* MALLOC_EXT_STATISTICS */ /* Try allocating the block from the first partially used slab. */ if (NULL != slabtable[ix].first) { /* Yup, we can. */ mslab_t * slab = slabtable[ix].first; ulog4f("slaballoc: block %x from freelist (next %x), slab %x free %d\n" , slab->freeList, BLOCK_NEXT(slab->freeList), slab, slabtable[ix].numBlocks - slab->numAllocated ); block = slab->freeList; slab->freeList = BLOCK_NEXT(slab->freeList); count_back(&small_free_stat, slab->size); #ifdef MALLOC_EXT_STATISTICS extstats[ix].cur_free--; extstats[ix].num_sm_alloc++; #endif /* MALLOC_EXT_STATISTICS */ slab->numAllocated++; /* If the slab is now fully used, move it into the full slab list. */ if (NULL == slab->freeList) { /* Unlink from partially-used list */ ulog3f("slaballoc: fully used: move from partial (next %x, last %x) to full (%x)\n" , slab->next, slabtable[ix].last, slabtable[ix].fullSlabs ); if (slabtable[ix].first == slabtable[ix].last) { /* Only one slab in the list */ slabtable[ix].first = slabtable[ix].last = NULL; } else { /* At least two slabs in the list. */ slabtable[ix].first = slab->next; slabtable[ix].first->prev = NULL; } /* Link into fully-used list */ slab->prev = NULL; slab->next = slabtable[ix].fullSlabs; if (slab->next) slab->next->prev = slab; slabtable[ix].fullSlabs = slab; slabtable[ix].numFullSlabs++; } /* Setup the block (M_SIZE) is mostly ok. */ MAKE_SMALL_CHECK(block, size); block[M_SIZE] |= (M_GC_FREE|M_REF); block[M_SIZE] &= ~THIS_BLOCK; block += M_OVERHEAD; MADVISE(block, orig_size); #ifdef MALLOC_EXT_STATISTICS extstat_update_max(extstats + SIZE_INDEX(size)); #endif /* MALLOC_EXT_STATISTICS */ in_malloc--; return (POINTER)block; } /* If there is no fresh slab, allocate one. */ if (NULL == slabtable[ix].fresh) { mslab_t * slab; /* If there are slabs in the fully-free list, use * the oldest one from there. Otherwise, allocated * from the large memory pool. */ if (NULL != slabtable[ix].firstFree) { /* Unlink the last (oldest) slab from the free list and hold it. */ slab = slabtable[ix].lastFree; ulog3f("slaballoc: get fresh %x from free list (prev %x, first %x)\n" , slab, slab->prev, slabtable[ix].firstFree ); if (slabtable[ix].firstFree == slabtable[ix].lastFree) { slabtable[ix].firstFree = slabtable[ix].lastFree = NULL; } else { slabtable[ix].lastFree = slab->prev; slabtable[ix].lastFree->next = NULL; } slabtable[ix].numFreeSlabs--; count_back(&small_slab_free_stat, SLAB_SIZE(slab, ix)); #ifdef MALLOC_EXT_STATISTICS extstats[EXTSTAT_SLABS].cur_free--; #endif /* MALLOC_EXT_STATISTICS */ } else { /* Allocate a new slab from the large memory pool. */ unsigned short numObjects = getNumBlocks(ix); size_t slabSize = sizeof(struct slab_s) - SINT + numObjects * size; slab = (mslab_t*)large_malloc_int(slabSize, MY_FALSE); if (slab == NULL) { dprintf1(2, "Low on MEMORY: Failed to allocated small slab " "of %d bytes.\n" , (p_int)(slabSize) ); in_malloc--; return NULL; } ulog4f("slaballoc: allocated fresh %x size %d, objects %d size %d\n" , slab, slabSize, numObjects, size ); slab->size = size; slabtable[ix].numSlabs++; count_up(&small_slab_stat, slabSize); count_up_n(&small_free_stat, numObjects, size); #ifdef MALLOC_EXT_STATISTICS extstats[ix].cur_free += numObjects; extstats[EXTSTAT_SLABS].num_xalloc++; #endif /* MALLOC_EXT_STATISTICS */ } /* (Re)initialize the remaining slab fields. */ #ifdef MALLOC_CHECK slab->magic = samagic[SIZE_MOD_INDEX(slab->size, samagic)]; #endif slab->next = slab->prev = NULL; slab->freeList = NULL; slab->numAllocated = 0; slabtable[ix].fresh = slab; slabtable[ix].freshMem = slab->blocks + (slabtable[ix].numBlocks * slab->size / SINT); ulog3f("slaballoc: fresh %x mem %x..%x\n" , slab, slab->blocks, slabtable[ix].freshMem ); #ifdef MALLOC_EXT_STATISTICS extstats[EXTSTAT_SLABS].cur_alloc++; extstat_update_max(extstats + EXTSTAT_SLABS); #endif /* MALLOC_EXT_STATISTICS */ } /* Allocate a new block from the fresh slab's memory. */ { word_t wsize = size / SINT; slabtable[ix].freshMem -= wsize; block = slabtable[ix].freshMem; slabtable[ix].fresh->numAllocated++; ulog3f("slaballoc: freshmem %x from %x..%x\n" , block, slabtable[ix].fresh->blocks, ((char *)slabtable[ix].fresh->blocks) + (slabtable[ix].numBlocks * slabtable[ix].fresh->size) ); block[M_SIZE] = (word_t)(block - (word_t *)(slabtable[ix].fresh)) | (M_SMALL|M_GC_FREE|M_REF); MAKE_SMALL_CHECK_UNCHECKED(block, size); block += M_OVERHEAD; MADVISE(block, orig_size); count_back(&small_free_stat, size); #ifdef MALLOC_EXT_STATISTICS extstats[ix].cur_free--; #endif /* MALLOC_EXT_STATISTICS */ } /* We have the block allocated, now check if the fresh slab has been used up. * If yes, move the fresh slab into the appropriate slab: front if some * of the blocks have already been freed, back if the slab is full. */ if (slabtable[ix].freshMem == slabtable[ix].fresh->blocks) { mslab_t * slab = slabtable[ix].fresh; ulog2f("slaballoc: fresh %x full, freelist %x\n" , slab, slab->freeList ); slabtable[ix].fresh = NULL; slabtable[ix].freshMem = NULL; if (NULL != slab->freeList) { insert_partial_slab(slab, ix); } else { /* Insert in fully-used list */ ulog2f("slaballoc: fresh %x into full list (first %x)\n" , slab, slabtable[ix].fullSlabs ); slab->next = slabtable[ix].fullSlabs; if (NULL != slabtable[ix].fullSlabs) slabtable[ix].fullSlabs->prev = slab; slabtable[ix].fullSlabs = slab; slabtable[ix].numFullSlabs++; } } #ifdef MALLOC_EXT_STATISTICS extstat_update_max(extstats + ix); #endif /* MALLOC_EXT_STATISTICS */ /* Done! */ in_malloc--; return (POINTER)block; } /* mem_alloc() */ /*-------------------------------------------------------------------------*/ static void sfree (POINTER ptr) /* Return the memoryblock to the allocator. * This function does the actual freeing, without checking for mutexes * and stack gaps; it is also used internally by the allocator to free * the memory reserves. */ { word_t *block; word_t ix; mslab_t *slab; Bool isFirstFree; if (!ptr) return; mem_mark_collectable(ptr); /* Get the real block address and type */ block = (word_t *) ptr; block -= M_OVERHEAD; ulog4f("slaballoc: free %x, %s %d, small %d\n" , block , (block[M_SIZE] & M_SMALL) ? "offset" : "size" , block[M_SIZE] & M_MASK , !!(block[M_SIZE] & M_SMALL) ); if (!(block[M_SIZE] & M_SMALL)) { /* It's a big block */ large_free(ptr); return; } /* It's a small block: put it into the slab's free list */ slab = (mslab_t*)(block - (block[M_SIZE] & M_MASK)); count_back(&small_alloc_stat, slab->size); ix = SIZE_INDEX(slab->size); ulog4f("slaballoc: -> slab %x [%d], freelist %x, %d free\n" , slab, ix, slab->freeList, slabtable[ix].numBlocks - slab->numAllocated ); #ifdef MALLOC_EXT_STATISTICS extstats[ix].num_xfree++; extstats[ix].cur_alloc--; extstats[ix].num_sm_free++; #endif /* MALLOC_EXT_STATISTICS */ #ifdef MALLOC_CHECK if (slab->magic == sfmagic[ix % NELEM(sfmagic)]) { in_malloc = 0; fatal("mem_free: block %p size %"PRIuPINT" (user %p) freed in free slab %p\n" , block, (ix * SINT), ptr, slab); } if (slab->magic != samagic[ix % NELEM(samagic)]) { in_malloc = 0; fatal("mem_free: block %p magic match failed for slab %p: " "size %"PRIuPINT", expected %"PRIuPINT", found %"PRIuPINT"\n" , block, slab, (ix * SINT), (p_uint)samagic[ix], slab->magic); } if (block[M_MAGIC] == sfmagic[ix % NELEM(sfmagic)]) { in_malloc = 0; fatal("mem_free: block %p size %"PRIuPINT" (user %p) freed twice\n" , block, (ix * SINT), ptr); } if (block[M_MAGIC] != samagic[ix % NELEM(samagic)]) { in_malloc = 0; fatal("mem_free: block %p magic match failed: " "size %"PRIuPINT", expected %"PRIuPINT", found %"PRIuPINT"\n" , block, (ix * SINT), (p_uint)samagic[ix], block[M_MAGIC]); } #endif /* Insert the block into the slab's freelist */ isFirstFree = (slab->freeList == NULL); #ifdef MALLOC_EXT_STATISTICS extstats[ix].cur_free++; extstat_update_max(extstats + ix); #endif /* MALLOC_EXT_STATISTICS */ block[M_SIZE] |= (THIS_BLOCK|M_REF); block[M_SIZE] &= ~(M_GC_FREE); #ifdef MALLOC_CHECK block[M_MAGIC] = sfmagic[SIZE_MOD_INDEX(slab->size, sfmagic)]; #endif SET_BLOCK_NEXT(block, slab->freeList); slab->freeList = block; slab->numAllocated--; count_up(&small_free_stat, slab->size); /* If this slab is not the fresh slab, handle possible list movements. */ if (slab != slabtable[ix].fresh) { if (isFirstFree) { /* First free block: move slab into the partially-used * list. */ ulog2f("slaballoc: first free: unlink %x from full (first %x)\n" , slab, slabtable[ix].fullSlabs ); if (slabtable[ix].fullSlabs == slab) slabtable[ix].fullSlabs = slab->next; if (slab->next) slab->next->prev = slab->prev; if (slab->prev) slab->prev->next = slab->next; slabtable[ix].numFullSlabs--; insert_partial_slab(slab, ix); } else if (slab->numAllocated == 0) { /* Slab completely free: unlink from list and move over * to free list (deallocate if retention time is 0). */ ulog3f("slaballoc: all free: unlink %x from partial (first %x, last %x)\n" , slab, slabtable[ix].first, slabtable[ix].last ); if (slab->next) slab->next->prev = slab->prev; if (slab->prev) slab->prev->next = slab->next; if (slabtable[ix].first == slab) slabtable[ix].first = slab->next; if (slabtable[ix].last == slab) slabtable[ix].last = slab->prev; # if SLAB_RETENTION_TIME > 0 ulog3f("slaballoc: current time %d, free (first %x, last %x)\n" , current_time, slabtable[ix].firstFree, slabtable[ix].lastFree ); #ifdef MALLOC_CHECK slab->magic = sfmagic[SIZE_MOD_INDEX(slab->size, sfmagic)]; #endif slab->blocks[0] = (mp_int)current_time; slab->prev = NULL; slab->next = slabtable[ix].firstFree; if (slab->next) slab->next->prev = slab; else slabtable[ix].lastFree = slab; slabtable[ix].firstFree = slab; slabtable[ix].numFreeSlabs++; count_up(&small_slab_free_stat, SLAB_SIZE(slab, ix)); #ifdef MALLOC_EXT_STATISTICS extstats[EXTSTAT_SLABS].cur_alloc--; extstats[EXTSTAT_SLABS].cur_free++; extstat_update_max(extstats + EXTSTAT_SLABS); #endif /* MALLOC_EXT_STATISTICS */ # else free_slab(slab, ix); # endif } #ifdef MALLOC_ORDER_SLAB_FREELISTS else { /* Another block freed in the slab: check if its position * in the list needs to be updated. */ keep_small_order(slab, ix); } #endif /* MALLOC_ORDER_SLAB_FREELISTS */ } /* if (not fresh slab) */ #ifdef DEBUG_MALLOC_ALLOCS else ulog("slaballoc: is fresh slab\n"); #endif /* DEBUG_MALLOC_ALLOCS */ } /* sfree() */ /*-------------------------------------------------------------------------*/ static void mem_free (POINTER ptr) /* Return the memoryblock to the allocator. * This is a wrapper for sfree() which performs checks for stack-gaps * and such. */ { if (!ptr) return; assert_stack_gap(); if (in_malloc++) { in_malloc = 0; writes(1, "Multiple threads in mem_free()\n"); fatal("Multiple threads in mem_free()\n"); /* NOTREACHED */ return; } sfree(ptr); in_malloc--; } /* mem_free() */ /*-------------------------------------------------------------------------*/ static POINTER mem_realloc (POINTER p, size_t size) /* Reallocate block

to the new size of and return the pointer. * The memory is not aligned and subject to GC. */ { POINTER t; size_t old_size; old_size = mem_block_size(p); if (old_size >= size) return p; t = mem_alloc(size); if (t == NULL) return NULL; memcpy(t, p, old_size); mem_free(p); return t; } /* mem_realloc() */ /*=========================================================================*/ /* LARGE BLOCKS */ /*-------------------------------------------------------------------------*/ #define l_next_block(p) ((p) + (p)[M_LSIZE] ) #define l_prev_block(p) ((p) - (*((p)-2)) ) /* Address the previous resp. this block. */ #define l_prev_free(p) (!(*(p) & PREV_BLOCK)) #define l_next_free(p) (!(*l_next_block(p) & THIS_BLOCK)) /* Check if the previous resp. the next block is free. */ /*-------------------------------------------------------------------------*/ /* Extra types and definitions for the AVL routines */ #if defined (sun) || defined(AMIGA) || defined(__linux__) || defined(__BEOS__) /* there is a type signed char */ typedef signed char balance_t; # define BALANCE_T_BITS 8 #else typedef short balance_t; # define BALANCE_T_BITS 16 #endif #if defined(sparc) || defined(AMIGA) /* try to avoid multiple shifts, because these are costly */ # define NO_BARREL_SHIFT #endif struct free_block { word_t size; struct free_block *parent, *left, *right; balance_t balance; #ifdef USE_AVL_FREELIST struct free_block * prev; /* prev free block in freelist * NULL for the AVL node */ struct free_block * next; /* next free block in freelist */ #endif /* USE_AVL_FREELIST */ short align_dummy; }; /* Prepare two nodes for the free tree that will never be removed, * so that we can always assume that the tree is and remains non-empty. */ extern struct free_block dummy2; /* forward */ static struct free_block dummy = { /* size */ 0 , /* parent */ &dummy2, /* left */ 0, /* right */ 0, /* balance */ 0 #ifdef USE_AVL_FREELIST , /* prev */ 0, /* next */ 0 #endif /* USE_AVL_FREELIST */ , /* align_dummy */ 0 }; struct free_block dummy2 = { /* size */ 0 , /* parent */ 0, /* left */ &dummy, /* right */ 0, /* balance */ -1 #ifdef USE_AVL_FREELIST , /* prev */ 0, /* next */ 0 #endif /* USE_AVL_FREELIST */ , /* align_dummy */ 0 }; static struct free_block *free_tree = &dummy2; #ifdef DEBUG_AVL static Bool inconsistency = MY_FALSE; /*-------------------------------------------------------------------------*/ static void _check_next (struct free_block *p) /* DEBUG_AVL: check this node

and both subnodes. */ { if (!p) return; { word_t *q; q = ((word_t *)p)-ML_OVERHEAD; q += *q; } _check_next(p->left); _check_next(p->right); } /* _check_next() */ /*-------------------------------------------------------------------------*/ static void check_next(void) /* DEBUG_AVL: check the free_tree. */ { _check_next(free_tree); } /* check_next() */ /*-------------------------------------------------------------------------*/ static int check_avl (struct free_block *parent, struct free_block *p) /* DEBUG_AVL: check the consistency of the AVL-(sub)tree in node

. * Return the size of the subtree, and set inconsistency to TRUE * if it is inconsistent. */ { int left, right; if (!p) return 0; left = check_avl(p, p->left ); right = check_avl(p, p->right); if (p->balance != right - left || p->balance < -1 || p->balance > 1) { writes (2, "Inconsistency in avl node: invalid balance!\n"); dprintf1(2, " node:%x\n",(p_uint)p); dprintf1(2, " size: %d\n", p->size); dprintf1(2, " left node:%x\n",(p_uint)p->left); dprintf1(2, " left height: %d\n",left ); dprintf1(2, " right node:%x\n",(p_uint)p->right); dprintf1(2, " right height: %d\n",right); dprintf1(2, " alleged balance: %d\n",p->balance); inconsistency = MY_TRUE; } if (p->parent != parent) { writes (2, "Inconsistency in avl node: invalid parent!\n"); dprintf1(2, " node:%x\n",(p_uint)p); dprintf1(2, " size: %d\n", p->size); dprintf1(2, " parent: %x\n", (p_uint)parent); dprintf1(2, " parent size: %d\n", parent->size); dprintf1(2, " alleged parent: %x\n", (p_uint)p->parent); dprintf1(2, " alleged parent size: %d\n", p->parent->size); dprintf1(2, " left height: %d\n",left ); dprintf1(2, " right height: %d\n",right); dprintf1(2, " alleged balance: %d\n",p->balance); inconsistency = MY_TRUE; } return left > right ? left+1 : right+1; } /* check_avl() */ /*-------------------------------------------------------------------------*/ static int do_check_avl(void) /* DEBUG_AVL: Check the free_tree for consistency. * Return 0 on success, otherwise abort the driver. */ { check_avl(0, free_tree); if (inconsistency) { fflush(stderr); fflush(stdout); in_malloc = 0; fatal("Inconsistency could crash the driver\n"); } return 0; } /* do_check_avl() */ /*-------------------------------------------------------------------------*/ static Bool contains (struct free_block *p, struct free_block *q) /* DEBUG_AVL: Check if free_block is contained in the sub-tree at * and below

. */ { return p == q || (p && (contains(p->left, q) || contains(p->right, q))); } /* contains() */ /*-------------------------------------------------------------------------*/ static int check_free_block (void *m) /* DEBUG_AVL: Check if free memory at is contained in the free_tree. * Return 0 on success, otherwise abort the driver. */ { word_t *p; int size; p = (word_t *)(((char *)m)-M_OVERHEAD*SINT); size = p[M_LSIZE]; if (!(*(p+size) & THIS_BLOCK)) { if (!contains(free_tree, (struct free_block *)(p+size+T_OVERHEAD)) ) { in_malloc = 0; fatal("Memory at %p, size: %"PRIuPINT" (user: %p) was not found in the free_tree\n", p, size, m); } } return 0; } /* check_free_block() */ #else /* !DEBUG_AVL */ #define do_check_avl() 0 #endif /* DEBUG_AVL */ /*-------------------------------------------------------------------------*/ static void remove_from_free_list (word_t *ptr) /* Remove the memory block from the free list. */ { struct free_block *p, *q, *r, *s, *t; #ifdef MALLOC_CHECK if (ptr[M_MAGIC] != LFMAGIC) { in_malloc = 0; fatal("remove_from_free_list: block %p, " "magic match failed: expected %"PRIuPINT", " "found %"PRIuPINT"\n" , ptr, (p_uint)LFMAGIC, ptr[M_MAGIC] ); } #endif p = (struct free_block *)(ptr+M_OVERHEAD); count_back(&large_free_stat, p->size); #ifdef MALLOC_EXT_STATISTICS extstats[EXTSTAT_LARGE].cur_free--; #endif /* MALLOC_EXT_STATISTICS */ #ifdef USE_AVL_FREELIST /* Unlink from AVL freelist */ if (p->prev) p->prev->next = p->next; if (p->next) p->next->prev = p->prev; /* If the block is not the AVL node itself, we're done */ if (p->prev) return; /*

is the AVL node itself, but if there is another block free of * the same size, just transfer over the node. */ if (p->next) { struct free_block *next = p->next; if (p == free_tree) { #ifdef DEBUG if (p->parent) { fatal("(remove_from_free_list) Node %p (size %"PRIuPINT ") is the AVL tree root, but has a parent\n", p, p->size); } #endif free_tree = p->next; } else { #ifdef DEBUG if (!p->parent) { fatal("(remove_from_free_list) Node %p (size %"PRIuPINT ") has neither a parent nor is it the AVL tree root.\n", p, p->size); } #endif if (p->parent->left == p) p->parent->left = p->next; else p->parent->right = p->next; } /* We must not clobber p->next->next when copying the node! */ p->next = next->next; *next = *p; /* Now adjust the parent pointer of the sub-nodes to the new * parent node. */ if (p->left) p->left->parent = next; if (p->right) p->right->parent = next; return; } /* It's the AVL itself, and there is no other free block of the same * size: remove the node from the tree. */ num_avl_nodes--; #endif /* USE_AVL_FREELIST */ #ifdef DEBUG_AVL dprintf1(2, "node:%x\n",(p_uint)p); dprintf1(2, "size:%d\n",p->size); #endif if (p->left) { if ( NULL != (q = p->right) ) { /* two childs */ s = q; do { r = q; q = r->left; } while(q != NULL); if (r == s) { r->left = s = p->left; s->parent = r; if ( NULL != (r->parent = s = p->parent) ) { if (p == s->left) { s->left = r; } else { s->right = r; } } else { free_tree = r; } r->balance = p->balance; p = r; goto balance_right; } else { t = r->parent; if ( NULL != (t->left = s = r->right) ) { s->parent = t; } r->balance = p->balance; r->left = s = p->left; s->parent = r; r->right = s = p->right; s->parent = r; if ( NULL != (r->parent = s = p->parent) ) { if (p == s->left) { s->left = r; } else { s->right = r; } } else { free_tree = r; } p = t; goto balance_left; } } else /* no right child, but left child */ { /* We set up the free list in a way so that there will remain at least two nodes, and the avl property ensures that the left child is a leaf ==> there is a parent */ s = p; p = s->parent; r = s->left; r->parent = p; if (s == p->left) { p->left = r; goto balance_left; } else { p->right = r; goto balance_right; } } } else /* no left child */ { /* We set up the free list in a way so that there is a node left of all used nodes, so there is a parent */ s = p; p = s->parent; if ( NULL != (q = r = s->right) ) { r->parent = p; } if (s == p->left) { p->left = r; goto balance_left; } else { p->right = r; goto balance_right; } } balance_q: r = p; p = q; if (r == p->right) { balance_t b; balance_right: b = p->balance; if (b > 0) { p->balance = 0; if (NULL != (q = p->parent)) goto balance_q; return; } else if (b < 0) { r = p->left; b = r->balance; if (b <= 0) { /* R-Rotation */ #ifdef DEBUG_AVL dprintf1(2, "r->balance: %d\n", r->balance); #endif if ( NULL != (p->left = s = r->right) ) { s->parent = p; } r->right = p; s = p->parent; p->parent = r; b += 1; r->balance = b; b = -b; #ifdef DEBUG_AVL dprintf1(2, "node r: %x\n", (p_uint)r); dprintf1(2, "r->balance: %d\n", r->balance); dprintf1(2, "node p: %x\n", (p_uint)p); p->balance = b; dprintf1(2, "p->balance: %d\n", p->balance); dprintf1(2, "r-height: %d\n", check_avl(r->parent, r)); #endif if ( NULL != (r->parent = s) ) { if ( 0 != (p->balance = b) ) { if (p == s->left) { s->left = r; return; } else { s->right = r; return; } } if (p == s->left) { /* left from parent */ goto balance_left_s; } else { /* right from parent */ p = s; p->right = r; goto balance_right; } } p->balance = b; free_tree = r; return; } else /* r->balance == +1 */ { /* LR-Rotation */ balance_t b2; t = r->right; b = t->balance; if ( NULL != (p->left = s = t->right) ) { s->parent = p; } if ( NULL != (r->right = s = t->left) ) { s->parent = r; } t->left = r; t->right = p; r->parent = t; s = p->parent; p->parent = t; #ifdef NO_BARREL_SHIFT b = -b; b2 = b >> 1; r->balance = b2; b -= b2; p->balance = b; #else b2 = (unsigned char)b >> 7; p->balance = b2; b2 = -b2 -b; r->balance = b2; #endif t->balance = 0; #ifdef DEBUG_AVL dprintf1(2, "t-height: %d\n", check_avl(t->parent, t)); #endif if ( NULL != (t->parent = s) ) { if (p == s->left) { p = s; s->left = t; goto balance_left; } else { p = s; s->right = t; goto balance_right; } } free_tree = t; return; } } else /* p->balance == 0 */ { p->balance = -1; return; } } else /* r == p->left */ { balance_t b; goto balance_left; balance_left_s: p = s; s->left = r; balance_left: b = p->balance; if (b < 0) { p->balance = 0; if ( NULL != (q = p->parent) ) goto balance_q; return; } else if (b > 0) { r = p->right; b = r->balance; if (b >= 0) { /* L-Rotation */ #ifdef DEBUG_AVL dprintf1(2, "r->balance: %d\n", r->balance); #endif if ( NULL != (p->right = s = r->left) ) { s->parent = p; } /* subtree relocated */ r->left = p; s = p->parent; p->parent = r; b -= 1; r->balance = b; b = -b; #ifdef DEBUG_AVL /* balances calculated */ dprintf1(2, "node r: %x\n", (p_uint)r); dprintf1(2, "r->balance: %d\n", r->balance); dprintf1(2, "node p: %x\n", (p_uint)p); p->balance = b; dprintf1(2, "p->balance: %d\n", p->balance); dprintf1(2, "r-height: %d\n", check_avl(r->parent, r)); #endif if ( NULL != (r->parent = s) ) { if ( 0 != (p->balance = b) ) { if (p == s->left) { s->left = r; return; } else { s->right = r; return; } } if (p == s->left) { /* left from parent */ goto balance_left_s; } else { /* right from parent */ p = s; p->right = r; goto balance_right; } } p->balance = b; free_tree = r; return; } else /* r->balance == -1 */ { /* RL-Rotation */ balance_t b2; t = r->left; b = t->balance; if ( NULL != (p->right = s = t->left) ) { s->parent = p; } if ( NULL != (r->left = s = t->right) ) { s->parent = r; } t->right = r; t->left = p; r->parent = t; s = p->parent; p->parent = t; #ifdef NO_BARREL_SHIFT b = -b; b2 = b >> 1; p->balance = b2; b -= b2; r->balance = b; #else b2 = (unsigned char)b >> 7; r->balance = b2; b2 = -b2 -b; p->balance = b2; #endif t->balance = 0; if ( NULL != (t->parent = s) ) { if (p == s->left) { p = s; s->left = t; goto balance_left; } else { s->right = t; p = s; goto balance_right; } } free_tree = t; return; } } else /* p->balance == 0 */ { p->balance++; return; } } } /* remove_from_free_list() */ /*-------------------------------------------------------------------------*/ static void add_to_free_list (word_t *ptr) /* Add the memory block to the free list. */ { word_t size; struct free_block *p, *q, *r; /* When there is a distinction between data and address registers and/or accesses, gcc will choose data type for q, so an assignmnt to q will faciliate branching */ #ifdef MALLOC_CHECK ptr[M_MAGIC] = LFMAGIC; #endif size = ptr[M_LSIZE]; #ifdef DEBUG_AVL dprintf1(2, "size:%d\n",size); #endif q = (struct free_block *)size; /* this assignment is just a hint for * register choice */ r = (struct free_block *)(ptr+M_OVERHEAD); count_up(&large_free_stat, size); #ifdef MALLOC_EXT_STATISTICS extstats[EXTSTAT_LARGE].cur_free++; extstat_update_max(extstats+EXTSTAT_LARGE); #endif /* MALLOC_EXT_STATISTICS */ r->size = size; r->parent = NULL; r->left = 0; r->right = 0; r->balance = 0; #ifdef USE_AVL_FREELIST r->prev = NULL; r->next = NULL; #endif /* USE_AVL_FREELIST */ q = free_tree; for ( ; ; /*p = q*/) { p = q; #ifdef DEBUG_AVL dprintf1(2, "checked node size %d\n",p->size); #endif #ifdef USE_AVL_FREELIST if (size == p->size) { /* We can attach the block to an existing node */ #ifdef MALLOC_ORDER_LARGE_FREELISTS struct free_block * tail = p; /* Find the proper node after which to insert */ if (p->next != NULL) { while (tail->next && tail->next < r) { tail = tail->next; #ifdef MALLOC_EXT_STATISTICS extstats[EXTSTAT_LARGE].num_sm_free++; #endif /* MALLOC_EXT_STATISTICS */ } } r->next = tail->next; r->prev = tail; if (r->next) r->next->prev = r; tail->next = r; #else r->next = p->next; r->prev = p; if (r->next) r->next->prev = r; p->next = r; #endif /* MALLOC_ORDER_LARGE_FREELISTS */ return; } #endif /* USE_AVL_FREELIST */ if (size < p->size) { if ( NULL != (q = p->left) ) { continue; } /* add left */ p->left = r; break; } else /* >= */ { if ( NULL != (q = p->right) ) { continue; } /* add right */ p->right = r; break; } } /* new leaf */ r->parent = p; #ifdef USE_AVL_FREELIST num_avl_nodes++; #endif /* USE_AVL_FREELIST */ #ifdef DEBUG_AVL dprintf2(2, "p %x->balance:%d\n",p, p->balance); #endif do { struct free_block *s; if (r == p->left) { balance_t b; if ( !(b = p->balance) ) { #ifdef DEBUG_AVL dprintf1(2, "p: %x\n", p); dprintf1(2, " p->size: %d\n", p->size); dprintf1(2, " p->balance: %d := -1\n", p->balance); dprintf1(2, " p->right-h: %d\n", check_avl(p, p->right)); dprintf1(2, " p->left -h: %d\n", check_avl(p, p->left )); #endif /* growth propagation from left side */ p->balance = -1; } else if (b < 0) { #ifdef DEBUG_AVL dprintf2(2, "p %x->balance:%d\n",p, p->balance); #endif if (r->balance < 0) { /* R-Rotation */ if ( NULL != (p->left = s = r->right) ) { s->parent = p; } r->right = p; p->balance = 0; r->balance = 0; s = p->parent; p->parent = r; if ( NULL != (r->parent = s) ) { if ( s->left == p) { s->left = r; } else { s->right = r; } } else { free_tree = r; } } else /* r->balance == +1 */ { /* LR-Rotation */ balance_t b2; struct free_block *t = r->right; #ifdef DEBUG_AVL dprintf1(2, "t = %x\n",(p_uint)t); dprintf1(2, "r->balance:%d\n",r->balance); #endif if ( NULL != (p->left = s = t->right) ) { s->parent = p; } /* relocated right subtree */ t->right = p; if ( NULL != (r->right = s = t->left) ) { s->parent = r; } /* relocated left subtree */ t->left = r; b = t->balance; #ifdef NO_BARREL_SHIFT b = -b; b2 = b >> 1; r->balance = b2; b -= b2; p->balance = b; #else b2 = (unsigned char)b >> 7; p->balance = b2; b2 = -b2 -b; r->balance = b2; #endif t->balance = 0; /* balances calculated */ s = p->parent; p->parent = t; r->parent = t; if ( NULL != (t->parent = s) ) { if ( s->left == p) { s->left = t; } else { s->right = t; } } else { free_tree = t; } #ifdef DEBUG_AVL dprintf1(2, "p->balance:%d\n",p->balance); dprintf1(2, "r->balance:%d\n",r->balance); dprintf1(2, "t->balance:%d\n",t->balance); /* LR-Rotation completed */ #endif } break; } else /* p->balance == +1 */ { p->balance = 0; /* growth of left side balanced the node */ break; } } else /* r == p->right */ { balance_t b; if ( !(b = p->balance) ) { #ifdef DEBUG_AVL dprintf1(2, "p: %x\n", p); dprintf1(2, " p->size: %d\n", p->size); dprintf1(2, " p->balance: %d += 1\n", p->balance); dprintf1(2, " p->right-h: %d\n", check_avl(p, p->right)); dprintf1(2, " p->left -h: %d\n", check_avl(p, p->left )); #endif /* growth propagation from right side */ p->balance++; } else if (b > 0) { if (r->balance > 0) { /* L-Rotation */ if ( NULL != (p->right = s = r->left) ) { s->parent = p; } r->left = p; p->balance = 0; r->balance = 0; s = p->parent; p->parent = r; if ( NULL != (r->parent = s) ) { if ( s->left == p) { s->left = r; } else { s->right = r; } } else { free_tree = r; } } else /* r->balance == -1 */ { /* RL-Rotation */ balance_t b2; struct free_block *t = r->left; #ifdef DEBUG_AVL dprintf1(2, "t = %x\n",(p_uint)t); dprintf1(2, "r->balance:%d\n",r->balance); #endif if ( NULL != (p->right = s = t->left) ) { s->parent = p; } /* relocated left subtree */ t->left = p; if ( NULL != (r->left = s = t->right) ) { s->parent = r; } /* relocated right subtree */ t->right = r; b = t->balance; #ifdef NO_BARREL_SHIFT b = -b; b2 = b >> 1; p->balance = b2; b -= b2; r->balance = b; #else b2 = (unsigned char)b >> 7; r->balance = b2; b2 = -b2 -b; p->balance = b2; #endif t->balance = 0; s = p->parent; p->parent = t; r->parent = t; if ( NULL != (t->parent = s) ) { if ( s->left == p) { s->left = t; } else { s->right = t; } } else { free_tree = t; } /* RL-Rotation completed */ } break; } else /* p->balance == -1 */ { #ifdef DEBUG_AVL dprintf1(2, "p: %x\n", p); dprintf1(2, " p->balance: %d\n", p->balance); dprintf1(2, " p->right-h: %d\n", check_avl(p, p->right)); dprintf1(2, " p->left -h: %d\n", check_avl(p, p->left )); #endif p->balance = 0; /* growth of right side balanced the node */ break; } } r = p; p = p->parent; } while ( NULL != (q = p) ); } /* add_to_free_list() */ /*-------------------------------------------------------------------------*/ static void build_block (word_t *ptr, word_t size) /* Mark the memory block of size words(!) as unallocated. * Also used to initialize new memory blocks received from the system. */ { word_t tmp; tmp = (*ptr & PREV_BLOCK); ptr[M_LSIZE] = size; *(ptr+size-2) = size; /* copy the size information */ *(ptr) = tmp | M_MASK; /* marks this block as free */ *(ptr+size) &= ~PREV_BLOCK; /* unset 'previous block' flag in next block */ } /* build_block() */ /*-------------------------------------------------------------------------*/ static void mark_block (word_t *ptr) /* Mark the memory block at as allocated, used, and freeable * by the GC. */ { *l_next_block(ptr) |= PREV_BLOCK; *ptr |= THIS_BLOCK | M_GC_FREE | M_REF; } /* mark_block() */ /*-------------------------------------------------------------------------*/ static word_t * add_large_free (word_t *ptr, word_t block_size) /* The large memory block with size is free: * add it to the free list after trying to coagulate it with adjacent * ones. * Result is the resulting pointer to the (possibly coagulated) free block. */ { /* If the next block is free, coagulate */ if (!(*(ptr+block_size) & THIS_BLOCK)) { remove_from_free_list(ptr+block_size); block_size += (ptr+block_size)[M_LSIZE]; } /* If the previous block is free, coagulate */ if (l_prev_free(ptr)) { remove_from_free_list(l_prev_block(ptr)); block_size += l_prev_block(ptr)[M_LSIZE]; ptr = l_prev_block(ptr); } /* Mark the block as free and add it to the freelist */ build_block(ptr, block_size); add_to_free_list(ptr); return ptr; } /* add_large_free() */ /*-------------------------------------------------------------------------*/ static char * large_malloc ( word_t size, Bool force_more) /* Allocate a large or bytes. * * If is TRUE, the function first tries to allocate * more memory directly from the system. If that fails, it tries a normal * allocation. This feature may be used to allocate small chunks for the * small block allocator (currently it is not). * * If memory from the system is allocated and * is not TRUE (or it is in the retry), the allocator allocates at least * CHUNK_SIZE bytes. Any extra is immediately entered into the freelist. * * Return the pointer to the allocated memory block, or NULL when out * of memory (not when running in SYSTEM privilege). * * If the system runs out of memory, the following steps are taken: * * - if is TRUE, it is set to FALSE and the allocation is tried * again, this time from the freelist. * - free the user reserve, then try again. * - if MASTER privilege: free the master reserve, then try again. * - if SYSTEM privilege: free the system reserve, then try again. * - if !SYSTEM privilege: set out_of_memory and return NULL * - if SYSTEM privilege: dump the lpc backtrace and exit(2) resp. fatal(). * * If any of the reserves is freed, the gc_request flag is set. */ { word_t real_size; word_t *ptr; #if defined(HAVE_MADVISE) || defined(DEBUG) || defined(DEBUG_MALLOC_ALLOCS) size_t orig_size = size; #endif size = (size + SINT*ML_OVERHEAD + SINT-1) / SINT; /* incl. overhead */ retry: ptr = NULL; if (!force_more) { /* Find the best fit in the AVL tree */ struct free_block *p, *q, *r; word_t minsplit; word_t tempsize; ptr += M_OVERHEAD; /* 'NULL' including overhead */ minsplit = size + SMALL_BLOCK_MAX + 1; /* The split-off block must still count as 'large' */ q = free_tree; for ( ; ; ) { p = q; #ifdef DEBUG_AVL dprintf1(2, "checked node size %d\n",p->size); #endif tempsize = p->size; if (minsplit < tempsize) { ptr = (word_t*)p; /* remember this fit */ if ( NULL != (q = p->left) ) { continue; } /* We don't need that much, but that's the best fit we have */ break; } else if (size > tempsize) { if ( NULL != (q = p->right) ) { continue; } break; } else /* size <= tempsize <= minsplit */ { if (size == tempsize) { ptr = (word_t*)p; break; } /* size < tempsize */ if ( NULL != (q = p->left) ) { r = p; /* if r is used in the following loop instead of p, * gcc will handle q very inefficient throughout the * function large_malloc() */ for (;;) { p = q; tempsize = p->size; if (size < tempsize) { if ( NULL != (q = p->left) ) { continue; } break; } else if (size > tempsize ) { if ( NULL != (q = p->right) ) { continue; } break; } else { ptr = (word_t*)p; goto found_fit; } } p = r; } tempsize = p->size; if (minsplit > tempsize) { if ( NULL != (q = p->right) ) { for (;;) { p = q; tempsize = p->size; if (minsplit <= tempsize) { ptr = (word_t*)p; /* remember this fit */ if ( NULL != (q = p->left) ) { continue; } break; } else /* minsplit > tempsize */ { if ( NULL != (q = p->right) ) { continue; } break; } } /* end inner for */ break; } break; /* no new fit */ } /* minsplit == tempsize ==> best non-exact fit */ ptr = (word_t*)p; break; } } /* end outer for */ found_fit: ptr -= M_OVERHEAD; } /* if (!force_more) */ if (!ptr) { /* No match, allocate more memory */ word_t chunk_size, block_size; size_t extra = 0; /* extra memory allocated by esbrk() */ block_size = size*SINT; /* If force_more is true (read: we have to allocate a small chunk) * or if the if the requested block would leave only a 'small' * block or no block in the usual CHUNK_SIZEd chunk, then allocate * exactly the block requested. Otherwise allocate a CHUNK_SIZEd * chunk, of which the unneeded part is entered into the freelist. */ if (force_more || block_size > CHUNK_SIZE - SMALL_BLOCK_MAX_BYTES - T_OVERHEAD*SINT ) { chunk_size = block_size; } else { chunk_size = CHUNK_SIZE; } /* Some systems like Darwin don't like odd sbrk()/malloc()s, therefore * we round up to the next multiple - 64 seems to work fine. That of * course might give an overhang of less than SMALL_BLOCK_MAX_BYTES, * so we have to add that and its own 64-Byte-fudge factor then, too. */ # define ALLOC_MULTIPLE 63 /* Ok, the (multiple-1) */ if ((chunk_size & ALLOC_MULTIPLE) != 0) { # if SMALL_BLOCK_MAX_BYTES > ALLOC_MULTIPLE chunk_size += SMALL_BLOCK_MAX_BYTES + 2 * ALLOC_MULTIPLE; # else chunk_size += ALLOC_MULTIPLE; # endif chunk_size &= ~ALLOC_MULTIPLE; } if (force_more) ulog3f("lmalloc(%d / %d): Forced allocate new chunk of %d bytes\n" , orig_size, block_size, chunk_size); else ulog3f("lmalloc(%d / %d): Allocate new chunk of %d bytes\n" , orig_size, block_size, chunk_size); /* Get more bytes from the system */ { if (max_malloced > 0 && (mp_int)(sbrk_stat.size + chunk_size) > max_malloced && ( (mp_int)(sbrk_stat.size + (heap_start ? 0 : SINT)) >= max_malloced || (chunk_size = max_malloced - sbrk_stat.size - (heap_start?0:SINT) ) < block_size ) ) { static const char mess[] = "MAX_MALLOCED limit reached.\n"; writes(2, mess); ptr = NULL; } else { ptr = (word_t *)esbrk(chunk_size, &extra); } } if (ptr == NULL) { /* Out of memory - try to recover */ ulog2f("lmalloc(%d / %d): Didn't get the memory from the system.\n" , orig_size, block_size); if (force_more) { /* The system is out of memory, but maybe we have some left * in the freelist. */ force_more = MY_FALSE; goto retry; } /* Give up */ return NULL; } /* Enough of the scary words - we got our memory block */ chunk_size += extra; block_size = chunk_size / SINT; /* Add block to free memory. */ ptr = add_large_free(ptr, block_size); } /* end of creating a new chunk */ /* ptr is now a pointer to a free block in the free list */ #ifdef USE_AVL_FREELIST /* If there is more than one free block for this size, take * the first block from the freelist to avoid copying around * the AVL node information. */ { struct free_block *p = (struct free_block *)(ptr + M_OVERHEAD); if (p->next) ptr = ((word_t *)(p->next)) - M_OVERHEAD; } #endif remove_from_free_list(ptr); real_size = ptr[M_LSIZE]; if (real_size - size) { /* split block pointed to by ptr into two blocks */ ptr[size] = 0; /* Init the header word */ build_block(ptr+size, real_size-size); #ifdef DEBUG if (real_size - size <= SMALL_BLOCK_MAX) { dprintf2(2,"DEBUG: lmalloc(%d / %d): " , orig_size, size * SINT); dprintf2(2 , "Split off block of %d bytes, small limit is %d bytes.\n" , (p_int)(real_size - size) * SINT , (p_int)SMALL_BLOCK_MAX * SINT); #ifdef DEBUG_MALLOC_ALLOCS if (gcollect_outfd != 2) { dprintf2(gcollect_outfd ,"DEBUG: lmalloc(%d / %d): " , orig_size, size * SINT); dprintf2(gcollect_outfd , "Split off block of %d bytes, small limit is %d bytes.\n" , (p_int)(real_size - size) * SINT , (p_int)SMALL_BLOCK_MAX * SINT); } #endif } #endif # ifndef SBRK_OK /* When we allocate a new chunk, it might differ slightly in size from * the desired size. */ if (real_size - size <= SMALL_BLOCK_MAX) { mark_block(ptr+size); *(ptr+size) &= ~M_GC_FREE; /* Hands off, GC! */ count_up(&large_wasted_stat, (*(ptr+size) & M_MASK) * SINT); } else # endif { /* At this point, it shouldn't happen that the split-off * block is too small to be allocated as a small block. */ add_to_free_list(ptr+size); } build_block(ptr, size); } /* The block at ptr is all ours */ mark_block(ptr); count_up(&large_alloc_stat, size); #ifdef MALLOC_EXT_STATISTICS extstats[EXTSTAT_LARGE].num_xalloc++; extstats[EXTSTAT_LARGE].cur_alloc++; extstat_update_max(extstats+EXTSTAT_LARGE); #endif /* MALLOC_EXT_STATISTICS */ #ifdef MALLOC_CHECK ptr[M_MAGIC] = LAMAGIC; #endif MADVISE(ptr+M_OVERHEAD, orig_size); return (char *) (ptr + M_OVERHEAD); } /* large_malloc() */ /*-------------------------------------------------------------------------*/ static void large_free (char *ptr) /* Free the large memory block . If possible, coagulate it with * neighbouring free blocks into one. */ { word_t size, *p; /* Get the real block pointer */ p = (word_t *) ptr; p -= M_OVERHEAD; size = p[M_LSIZE]; count_back(&large_alloc_stat, size); #ifdef MALLOC_EXT_STATISTICS extstats[EXTSTAT_LARGE].num_xfree++; extstats[EXTSTAT_LARGE].cur_alloc--; #endif /* MALLOC_EXT_STATISTICS */ #ifdef MALLOC_CHECK if (p[M_MAGIC] == LFMAGIC) { in_malloc = 0; fatal("large_free: block %p size %"PRIuPINT", (user %p) freed twice\n" , p, (size * SINT), ptr); } if (p[M_MAGIC] != LAMAGIC) { in_malloc = 0; fatal("large_free(%p): block %p magic match failed: size %"PRIuPINT", " "expected %"PRIuPINT", found %"PRIuPINT"\n" , ptr, p, (size * SINT) , (p_uint)LAMAGIC , p[M_MAGIC] ); } #endif (void)add_large_free(p, size); } /* large_free() */ /*-------------------------------------------------------------------------*/ static char * esbrk (word_t size, size_t * pExtra) /* Allocate a block of bytes from the system and return its pointer. * If esbrk() allocated a larger block, the difference to is * returned in *. * #ifdef SBRK_OK * It is system dependent how sbrk() aligns data, so we simpy use brk() * to insure that we have enough. * At the end of the newly expanded heap we create a fake allocated * block of 0 bytes so that large_malloc() knows where to stop. #else * Use malloc() to allocate a new block of memory. If this block borders * to the previous one, both blocks are joined. * The allocated block (modulo joints) is tagged at both ends with fake * "allocated" blocks of which cover the unallocated areas - large_malloc() * will perceive this as a fragmented heap. #endif */ { #ifdef SBRK_OK #ifdef SunOS4 extern char *sbrk(); extern int brk(); #endif mdb_log_sbrk(size); *pExtra = 0; if (!heap_end) { /* First call: allocate the first fake block */ heap_start = heap_end = (word_t *)sbrk(0); if (!esbrk(2*SINT, pExtra)) { in_malloc = 0; fatal("Couldn't malloc anything\n"); } *heap_start = 2; *(heap_start+1) = PREV_BLOCK | M_MASK; count_up(&large_wasted_stat, 2*SINT); assert_stack_gap(); } /* Get the new block */ if ((int)brk((char *)heap_end + size) == -1) return NULL; count_up(&sbrk_stat, size); heap_end = (word_t*)((char *)heap_end + size); heap_end[-1] = THIS_BLOCK | M_MASK; heap_end[-2] = M_MASK; return (char *)(heap_end - 1) - size; /* overlap old memory block */ #else /* not SBRK_OK */ char *block; word_t *p; /* start of the fake block */ const int overhead = TL_OVERHEAD; size_t overlap = 0; /* How much extra memory esbrk() could recycle from joining * the new allocation with the old one. */ mdb_log_sbrk(size); size += overhead * SINT; /* for the extra fake "allocated" block */ block = malloc(size); if (!block) { *pExtra = 0; return NULL; } assert_stack_gap(); /* p points to the start of the fake allocated block used * as sentinel/bridge */ p = (word_t *)(block + size) - overhead + 1; #ifdef MALLOC_TRACE p[M_OVERHEAD+XM_FILE] = (word_t)"sentinel/bridge"; p[M_OVERHEAD+XM_LINE] = 0; #endif #ifdef MALLOC_LPC_TRACE p[M_OVERHEAD+XM_OBJ] = 0; p[M_OVERHEAD+XM_PROG] = 0; p[M_OVERHEAD+XM_PC] = 0; #endif if (!heap_end) { /* First call: create the inital fake block */ heap_start = (word_t*)block; heap_end = (word_t*)(block + size); *((word_t *)block+1) = PREV_BLOCK; p[M_LSIZE] = overhead; p[M_SIZE] = THIS_BLOCK | M_MASK; /* no M_GC_FREE */ overlap = 0; } else { /* Try to join with the existing heap */ if (block < (char *)heap_start) { /* New block before the known heap */ *((word_t *)block+1) = PREV_BLOCK|M_MASK; /* Lower sentinel */ if (block + size == (char *)heap_start) { /* We can join with the existing heap */ p[overhead] &= ~PREV_BLOCK; overlap = SINT; count_back(&large_wasted_stat, overlap); } else { /* Separate from the heap */ p[M_LSIZE] = (heap_start - p + 1); p[M_SIZE] = THIS_BLOCK | M_MASK; /* no M_GC_FREE */ overlap = 0; } heap_start = (word_t *)block; } else if (block >= (char *)heap_end) { /* New block after the known heap */ p[M_SIZE] = THIS_BLOCK | M_MASK; /* no M_GC_FREE */ p[M_LSIZE] = overhead; if (block == (char *)heap_end) { /* We can join with the existing heap */ heap_end = (word_t *)(block + size); block -= overhead; overlap = overhead * SINT; count_back(&large_wasted_stat, overlap); } else { /* Separate from the heap */ p = (word_t *)heap_end - overhead + 1; p[M_SIZE] = (p[M_SIZE] & (PREV_BLOCK|THIS_BLOCK|M_GC_FREE)) | M_MASK; p[M_LSIZE] = (word_t *)block - p + 1; heap_end = (word_t *)(block + size); *((word_t *)block+1) = PREV_BLOCK; overlap = 0; } } else { /* We got a block within the known heap. * Try to find it in the fake blocks created earlier. * This is slow, but it shouldn't happen too often. */ word_t *prev, *next; /* Go to the block right before the one we got */ next = heap_start; do { prev = next; next = prev + *prev; } while (next < (word_t *)block); overlap = 0; if ((word_t *)block == prev + overhead) { /* Our block directly follows the one we found */ block -= overhead; overlap += overhead * SINT; count_back(&large_wasted_stat, overhead * SINT); } else { /* We have to create a new bridge block */ prev++; prev[M_SIZE] = (prev[M_SIZE] & (PREV_BLOCK|THIS_BLOCK|M_GC_FREE)) | M_MASK; prev[M_LSIZE] = (word_t*)block - prev + 1; #ifdef MALLOC_TRACE prev[M_OVERHEAD+XM_FILE] = (word_t)"block"; prev[M_OVERHEAD+XM_LINE] = 0; #endif #ifdef MALLOC_LPC_TRACE prev[M_OVERHEAD+XM_OBJ] = 0; prev[M_OVERHEAD+XM_PROG] = 0; prev[M_OVERHEAD+XM_PC] = 0; #endif *((word_t *)block+1) = PREV_BLOCK | M_MASK; } if (next - p == overhead) { /* Our block directly preceedes the next one */ *(next+1) &= ~PREV_BLOCK; overlap += overhead * SINT; count_back(&large_wasted_stat, overhead * SINT); } else { /* We have to create a new bridge block */ p[M_SIZE] = THIS_BLOCK | M_MASK; /* no M_GC_FREE */ p[M_LSIZE] = next - p + 1; } } } count_up(&sbrk_stat, size); count_up(&large_wasted_stat, overhead * SINT); *pExtra = overlap; return block + SINT; #endif /* !SBRK_OK */ } /* esbrk() */ /*-------------------------------------------------------------------------*/ static INLINE void * mem_increment_size (void *vp, size_t size) /* Try to extent the allocation block for to hold more bytes. * If this is not possible, return NULL, otherwise return a pointer * to the start of the block extension. */ { char *p = vp; word_t *start, *start2, *start3, old_size, next; const word_t wsize = (size + SINT - 1)/ SINT; malloc_increment_size_calls++; start = (word_t*)p - M_OVERHEAD; if (start[M_SIZE] & M_SMALL) { /* Can't extend small blocks. */ return NULL; } /* Extend a large block */ old_size = start[M_LSIZE]; start2 = &start[old_size]; if (start2[M_SIZE] & THIS_BLOCK) return NULL; /* no following free block */ next = start2[M_LSIZE]; if (next == wsize) { /* Next block fits perfectly */ remove_from_free_list(start2); start2[next] |= PREV_BLOCK; start[M_LSIZE] += wsize; malloc_increment_size_success++; malloc_increment_size_total += (start2 - start) - M_OVERHEAD; count_add(&large_alloc_stat, wsize); return start2+M_LSIZE; } if (next > wsize + SMALL_BLOCK_MAX + 1) { /* Split the next block, it is large enough */ remove_from_free_list(start2); start2[next-2] -= wsize; start3 = start2 + wsize; start3[M_SIZE] = M_MASK | PREV_BLOCK; start3[M_LSIZE] = (next-wsize); add_to_free_list(start3); start[M_LSIZE] += wsize; malloc_increment_size_success++; malloc_increment_size_total += (start2 - start) - M_OVERHEAD; count_add(&large_alloc_stat, wsize); return start2+M_LSIZE; } /* No success */ return NULL; } /* mem_increment_size() */ /*=========================================================================*/ /* GARBAGE COLLECTOR */ /*-------------------------------------------------------------------------*/ static INLINE void mem_clear_ref (POINTER p) /* GC Support: Clear the 'referenced' flag for block

. */ { ((word_t *)(p))[-M_OVERHEAD] &= ~M_REF; } /* mem_clear_ref() */ /*-------------------------------------------------------------------------*/ static INLINE void mem_mark_ref (POINTER p) /* GC Support: Set the 'referenced' flag for block

. */ { ((word_t *)(p))[-M_OVERHEAD] |= M_REF; } /* mem_mark_ref() */ /*-------------------------------------------------------------------------*/ static INLINE Bool mem_test_ref (POINTER p) /* GC Support: Check the memory block marker for

, return TRUE if _not_ * set. */ { return !( ((word_t *)(p))[-M_OVERHEAD] & M_REF ); } /* mem_test_ref() */ /*-------------------------------------------------------------------------*/ Bool mem_is_freed (void *p, p_uint minsize) /* Check if block for the allocation

is a free block of at least * . Blocks outside the heap always qualify. * The function might return false for joined blocks. */ { word_t *block; word_t i; block = (word_t *) p; block -= M_OVERHEAD; if (block < heap_start || block + M_OVERHEAD >= heap_end) return MY_TRUE; if (block[M_SIZE] & M_SMALL) { i = ((mslab_t *)(block - (block[M_SIZE] & M_MASK)))->size / SINT; } else { i = block[M_LSIZE]; } if (i < M_OVERHEAD + ((minsize + SINT - 1) / SINT) || block + i >= heap_end) return MY_TRUE; if (i >= SMALL_BLOCK_MAX) { /* Large block */ word_t* block2; block2 = block + i; return !(block[M_SIZE] & THIS_BLOCK) #ifdef MALLOC_CHECK || block[M_MAGIC] != LAMAGIC #endif /* MALLOC_CHECK */ || !(*block2 & PREV_BLOCK); } /* Small block */ return (block[M_SIZE] & THIS_BLOCK) != 0; } /* mem_is_freed() */ /*-------------------------------------------------------------------------*/ static void mem_clear_slab_memory_flags ( const char * tag , mslab_t * slab, int ix, word_t * startp) /* Set the flags for the memory of in table[]: the slab and the * contained free list block are marked as referenced, the others as * unreferenced. If is not NULL, it denotes the starting memory * address for the scan, otherwise ->blocks is the starting address * (this is used for marking the fresh slab). */ { word_t * p; if (NULL != startp) p = startp; else p = slab->blocks; ulog2f("slaballoc: clear in %s slab %x,", tag, slab); ulog3f(" mem %x/%x..%x\n" , slab->blocks, startp, ((char *)slab->blocks) + (slabtable[ix].numBlocks * slab->size) ); mem_mark_ref(slab); while (p < slab->blocks + slabtable[ix].numBlocks * slab->size / SINT) { /* ulog1f("slaballoc: clear block %x\n", p); */ *p &= ~M_REF; p += slab->size / SINT; } for (p = slab->freeList; p != NULL; p = BLOCK_NEXT(p)) { /* ulog1f("slaballoc: mark free block %x\n", p); */ *p |= M_REF; } } /* mem_clear_slab_memory_flags() */ /*-------------------------------------------------------------------------*/ void mem_clear_ref_flags (void) /* Walk through all allocated blocks and clear the M_REF flag in preparation * for a GC. */ { word_t *p, *last; int i; /* Clear the large blocks */ last = heap_end - TL_OVERHEAD; for (p = heap_start; p < last; ) { p[1] &= ~M_REF; if (p + *p > heap_end) { in_malloc = 0; fatal("pointer larger than brk: %p + %"PRIuPINT" = %p > %p\n" , p, (p_uint)(*p), p + *p , last); } p += *p; } /* Now mark the memory used for the small slabs as ref'd, * clear all contained small blocks, then mark all blocks in the free * lists as ref'd. */ for (i = 0; i < SMALL_BLOCK_NUM; ++i) { mslab_t * slab; ulog1f("slaballoc: clear_ref [%d]\n", i); /* Mark the fresh slab. */ if (slabtable[i].fresh) { mem_clear_slab_memory_flags( "fresh", slabtable[i].fresh, i , slabtable[i].freshMem); } /* Mark the partially used slabs. */ for (slab = slabtable[i].first; slab != NULL; slab = slab->next) { mem_clear_slab_memory_flags("partial", slab, i, NULL); } /* Mark the fully used slabs. */ for (slab = slabtable[i].fullSlabs; slab != NULL; slab = slab->next) { mem_clear_slab_memory_flags("full", slab, i, NULL); } /* Mark the fully free slabs. */ for (slab = slabtable[i].firstFree; slab != NULL; slab = slab->next) { ulog1f("slaballoc: clear in free slab %x\n" , slab); mem_mark_ref(slab); /* No internal freelist to scan */ } } /* We set M_REF for small free blocks that early for two reasons: * - if it's referenced anywhere else, we get a fatal. * - if the block gets malloced by the swapper in pass 5, it needs * M_REF to be already set. */ } /* mem_clear_ref_flags() */ /*-------------------------------------------------------------------------*/ static mp_int mem_free_unrefed_slab_memory ( const char * tag , mslab_t * slab, int ix, word_t * startp) /* Scan the memory of in table[] for unreferenced blocks and free * them. If is not NULL, it denotes the starting memory address for * the scan, otherwise ->blocks is the starting address (this is used * for scanning the fresh slab). * Return the number of successfully recovered blocks. * * Note that this may move the slab into a different slab list. */ { mp_int success = 0; word_t * p; if (NULL != startp) p = startp; else p = slab->blocks; ulog2f("slaballoc: free unref in %s slab %x,", tag, slab); ulog3f(" mem %x/%x..%x\n" , slab->blocks, startp, ((char *)slab->blocks) + (slabtable[ix].numBlocks * slab->size) ); while (p < slab->blocks + slabtable[ix].numBlocks * slab->size / SINT) { if ((*p & (M_REF|M_GC_FREE)) == M_GC_FREE) { /* Unref'd small blocks are definitely lost */ success++; count_back(&xalloc_stat, slab->size - (T_OVERHEAD * SINT)); dprintf2(gcollect_outfd, "freeing small block 0x%x (user 0x%x)" , (p_uint)p, (p_uint)(p+M_OVERHEAD)); #ifdef MALLOC_TRACE dprintf2(gcollect_outfd, " %s %d" , p[XM_FILE+M_OVERHEAD], p[XM_LINE+M_OVERHEAD]); #endif writes(gcollect_outfd, "\n"); #ifdef MALLOC_LPC_TRACE write_lpc_trace(gcollect_outfd, p + M_OVERHEAD, MY_FALSE); #endif print_block(gcollect_outfd, p + M_OVERHEAD); /* Recover the block */ *p |= M_REF; sfree(p+M_OVERHEAD); } #if 0 && defined(DEBUG_MALLOC_ALLOCS) else ulog1f("slaballoc: block %x is ref'd\n", p); #endif /* DEBUG_MALLOC_ALLOCS */ p += slab->size / SINT; } return success; } /* mem_free_unrefed_slab_memory() */ /*-------------------------------------------------------------------------*/ void mem_free_unrefed_memory (void) /* The GC marked all used memory as REF'd, now recover all blocks which * are allocated, but haven't been marked. */ { word_t *p, *last; int i; mp_int success = 0; /* Scan the heap for lost large blocks */ last = heap_end - TL_OVERHEAD; for (p = heap_start; p < last; ) { word_t size, flags; size = *p; flags = p[1]; if ( (flags & (M_REF|THIS_BLOCK|M_GC_FREE)) == (THIS_BLOCK|M_GC_FREE) ) { /* Large block marked as in use (THIS_BLOCK), but is not * referenced (no M_REF) - recover it. */ word_t size2, flags2; success++; count_back(&xalloc_stat, mem_block_size(p+ML_OVERHEAD)); #if defined(MALLOC_TRACE) || defined(MALLOC_LPC_TRACE) dprintf1(gcollect_outfd, "freeing large block 0x%x", (p_uint)p); #endif #ifdef MALLOC_TRACE dprintf3(gcollect_outfd, " %s %d size 0x%x\n", p[XM_FILE+ML_OVERHEAD], p[XM_LINE+ML_OVERHEAD], size & M_MASK ); #endif #ifdef MALLOC_LPC_TRACE write_lpc_trace(gcollect_outfd, p + ML_OVERHEAD, MY_FALSE); #endif print_block(gcollect_outfd, p + ML_OVERHEAD); size2 = p[size]; flags2 = p[size + 1]; large_free((char *)(p+ML_OVERHEAD)); if ( !(flags2 & THIS_BLOCK) ) size += size2; } p += size; } if (success) { dprintf1(gcollect_outfd, "%d large blocks freed\n", success); } /* Scan the small chunks for lost small blocks. * Remember that small blocks in the free-lists are marked as ref'd. */ success = 0; for (i = 0; i < SMALL_BLOCK_NUM; ++i) { mslab_t * slab, *next; ulog1f("slaballoc: free_unrefed [%d]\n" , i); /* Search the fresh slab for unreferenced blocks. */ if (slabtable[i].fresh) { success += mem_free_unrefed_slab_memory("fresh", slabtable[i].fresh , i, slabtable[i].freshMem); } /* Search the partially used slabs for unreferenced blocks. * Note that this may move individual slabs into a different list. */ for (slab = slabtable[i].first; slab != NULL; slab = next) { next = slab->next; success += mem_free_unrefed_slab_memory("partial", slab, i, NULL); } /* Search the fully used slabs for unreferenced blocks. * Note that this may move individual slabs into a different list. */ for (slab = slabtable[i].fullSlabs; slab != NULL; slab = next) { next = slab->next; success += mem_free_unrefed_slab_memory("full", slab, i, NULL); } } if (success) { dprintf1(gcollect_outfd, "%d small blocks freed\n", success); } } /* mem_free_unrefed_memory() */ /*-------------------------------------------------------------------------*/ #if 0 static void mem_dump_slab_memory (int fd, const char * tag, int tableIx , mslab_t * slab, word_t * startp) /* Dump the memory of . * If is not NULL, it denotes the starting memory address * for the dump, otherwise ->blocks is the starting address (this is * used for dumping the fresh slab). */ { word_t * p; dprintf4(fd, "\n--- %s Slab: %x .. %x size %x" , (p_uint)tag , (p_uint)slab, (p_uint)(slab + SLAB_SIZE(slab, tableIx)) - 1 , (p_uint)SLAB_SIZE(slab, tableIx) ); dprintf2(fd, " (%d/%d byte blocks)\n" , (p_int)(tableIx + SMALL_BLOCK_MIN) * SINT , (p_int)(tableIx + T_OVERHEAD+SMALL_BLOCK_MIN) * SINT ); /* No trace information for slabs */ if (NULL != startp) p = startp; else p = slab->blocks; while (p <= slab->blocks + slabtable[tableIx].numBlocks * slab->size / SINT) { word_t size = *p; if (!(*p & THIS_BLOCK)) { dprintf4(fd, "%x .. %x %s offset %x " , (p_uint)p, (p_uint)(p + (size&M_MASK)) - 1 , (p_uint)((size & M_GC_FREE) ? " " : "P") , (p_uint)(size & M_MASK) * SINT ); #ifdef MALLOC_TRACE if (p[XM_FILE+M_OVERHEAD]) dprintf2(fd, ": %s %d " , p[XM_FILE+M_OVERHEAD], q[XM_LINE+M_OVERHEAD] ); else writes(fd, ": - - "); #endif #ifdef MALLOC_LPC_TRACE if (p[M_OVERHEAD + XM_OBJ]) { writes(fd, ": "); write_lpc_trace(fd, p + M_OVERHEAD, MY_TRUE); } else #endif writes(fd, "\n"); } p += slab->size / SINT; } } /* mem_dump_slab_memory() */ #endif /*-------------------------------------------------------------------------*/ static Bool mem_identify_slab (int fd, const char * tag, int tableIx , mslab_t * list, mslab_t * slab) /* Check if is member of the list starting at . * If yes, print it's information preceeded by and return TRUE. * Return FALSE otherwise. */ { Bool isSlab = MY_FALSE; for ( NOOP ; !isSlab && list != NULL ; list = list->next ) { if (slab == list) { dprintf4(fd, ": %s slab %x (%d/%d bytes)\n" , (p_int)tag , (p_int) slab , (p_int)(tableIx + SMALL_BLOCK_MIN) * SINT , (p_int)(tableIx + T_OVERHEAD+SMALL_BLOCK_MIN) * SINT ); isSlab = MY_TRUE; break; } } return isSlab; } /* mem_identify_slab() */ /*-------------------------------------------------------------------------*/ Bool mem_dump_memory (int fd) /* Print the location, size, and (if available) the TRACE information * of all memory blocks to file , and return TRUE. * If the allocator doesn't support this operation, print nothing * and return FALSE. * * If is -1, just return TRUE or FALSE (this is used to check if * the allocator supports memory dumps). */ { word_t *p, *q, *last; int ix; if (fd < 0) return MY_TRUE; writes(fd, "\n--- Large Blocks\n"); /* Dump the heap blocks */ last = heap_end - TL_OVERHEAD; for (p = heap_start; p < last; ) { word_t size, flags; size = *p; flags = p[1]; if ( flags & THIS_BLOCK ) { Bool isSlab = MY_FALSE; dprintf4(fd, "%x .. %x %s size %x " , (p_uint)p, (p_uint)(p + size) - 1 , (p_uint)((flags & M_GC_FREE) ? " " : "P") , (p_uint)size * SINT ); q = p + ML_OVERHEAD; for (ix = 0; !isSlab && ix < SMALL_BLOCK_NUM; ++ix) { if ((word_t *)slabtable[ix].fresh == q) { dprintf2(fd, ": fresh slab (%d/%d bytes)\n" , (p_int)(ix + SMALL_BLOCK_MIN) * SINT , (p_int)(ix + T_OVERHEAD+SMALL_BLOCK_MIN) * SINT ); isSlab = MY_TRUE; } if (!isSlab) isSlab = mem_identify_slab(fd, "partial", ix , slabtable[ix].first , (mslab_t*)q); if (!isSlab) isSlab = mem_identify_slab(fd, "full", ix , slabtable[ix].fullSlabs , (mslab_t*)q); if (!isSlab) isSlab = mem_identify_slab(fd, "free", ix , slabtable[ix].firstFree , (mslab_t*)q); } if (!isSlab) { #ifdef MALLOC_TRACE if (p[XM_FILE+ML_OVERHEAD]) dprintf2(fd, ": %s %d " , p[XM_FILE+ML_OVERHEAD], p[XM_LINE+ML_OVERHEAD] ); else writes(fd, ": - -"); #endif #ifdef MALLOC_LPC_TRACE if (p[ML_OVERHEAD + XM_OBJ]) { writes(fd, ": "); write_lpc_trace(fd, p + ML_OVERHEAD, MY_TRUE); } else #endif writes(fd, "\n"); } } p += size; } #if 0 /* Dump the slabs and their small blocks. */ for (ix = 0; ix < SMALL_BLOCK_NUM; ++ix) { mslab_t * slab; if (NULL != (slab = slabtable[ix].fresh)) { mem_dump_slab_memory(fd, "Fresh", ix, slab, slabtable[ix].freshMem); } for (slab = slabtable[ix].first; slab != NULL; slab = slab->next) mem_dump_slab_memory(fd, "Partial", ix, slab, NULL); for (slab = slabtable[ix].fullSlabs; slab != NULL; slab = slab->next) mem_dump_slab_memory(fd, "Full", ix, slab, NULL); for (slab = slabtable[ix].firstFree; slab != NULL; slab = slab->next) mem_dump_slab_memory(fd, "Free", ix, slab, NULL); } #endif return MY_TRUE; } /* mem_dump_memory() */ /*-------------------------------------------------------------------------*/ void mem_consolidate (Bool force) /* Consolidate the memory. * * If is TRUE, all fully free slabs are deallocated. * If is FALSE, only the free slabs older than SLAB_RETENTION_TIME * are deallocated. */ { int ix; ulog1f("slaballoc: consolidate (%d)\n", force); for (ix = 0; ix < SMALL_BLOCK_NUM; ++ix) { ulog1f("slaballoc: consolidate [%d]\n", ix); if (force) { mslab_t * slab; while (NULL != (slab = slabtable[ix].firstFree)) { slabtable[ix].firstFree = slab->next; count_back(&small_slab_free_stat, SLAB_SIZE(slab, ix)); free_slab(slab, ix); } slabtable[ix].numFreeSlabs = 0; slabtable[ix].lastFree = NULL; } #if SLAB_RETENTION_TIME > 0 else { mslab_t * slab; if ( NULL != (slab = slabtable[ix].lastFree) && current_time - (mp_int)slab->blocks[0] > SLAB_RETENTION_TIME ) { ulog3f("slaballoc: free slab %x (prev %x) delta-t %d\n" , slab, slab->prev, current_time - (mp_int)slab->blocks[0]); slabtable[ix].lastFree = slab->prev; if (slab->prev) slab->prev->next = NULL; else slabtable[ix].firstFree = NULL; slabtable[ix].numFreeSlabs--; count_back(&small_slab_free_stat, SLAB_SIZE(slab, ix)); free_slab(slab, ix); } #ifdef DEBUG_MALLOC_ALLOCS else if (NULL != slab) { ulog3f("slaballoc: keep slab %x (prev %x) delta-t %d\n" , slab, slab->prev, current_time - (mp_int)slab->blocks[0]); } #endif /* DEBUG_MALLOC_ALLOCS */ } #endif /* SLAB_RETENTION_TIME > 0 */ } #ifdef MALLOC_EXT_STATISTICS extstat_update_max(extstats + EXTSTAT_SLABS); #endif /* MALLOC_EXT_STATISTICS */ } /* mem_consolidate() */ /***************************************************************************/