#define JEMALLOC_ARENA_C_

#include "jemalloc/internal/jemalloc_internal.h"

/******************************************************************************/

/* Data. */

ssize_t opt_decay_time = DECAY_TIME_DEFAULT;

static ssize_t decay_time_default;

const arena_bin_info_t arena_bin_info[NBINS] = {

#define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \

{reg_size, slab_size, nregs, BITMAP_INFO_INITIALIZER(nregs)},

#define BIN_INFO_bin_no(reg_size, slab_size, nregs)

#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \

lg_delta_lookup) \

BIN_INFO_bin_##bin((1U<<lg_grp) + (ndelta<<lg_delta), \

(pgs << LG_PAGE), (pgs << LG_PAGE) / ((1U<<lg_grp) + \

(ndelta<<lg_delta)))

SIZE_CLASSES

#undef BIN_INFO_bin_yes

#undef BIN_INFO_bin_no

#undef SC

};

/******************************************************************************/

/*

* Function prototypes for static functions that are referenced prior to

* definition.

*/

static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena,

size_t ndirty_limit);

static void arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena,

extent_t *slab, arena_bin_t *bin);

static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena,

extent_t *slab, arena_bin_t *bin);

/******************************************************************************/

static size_t

arena_extent_dirty_npages(const extent_t *extent)

{

return (extent_size_get(extent) >> LG_PAGE);

}

static extent_t *

arena_extent_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena,

extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,

size_t alignment, bool *zero, bool slab)

{

bool commit = true;

malloc_mutex_assert_owner(tsdn, &arena->lock);

return (extent_alloc_cache(tsdn, arena, r_extent_hooks, new_addr, usize,

pad, alignment, zero, &commit, slab));

}

extent_t *

arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,

extent_hooks_t **r_extent_hooks, void *new_addr, size_t size,

size_t alignment, bool *zero)

{

extent_t *extent;

malloc_mutex_lock(tsdn, &arena->lock);

extent = arena_extent_cache_alloc_locked(tsdn, arena, r_extent_hooks,

new_addr, size, 0, alignment, zero, false);

malloc_mutex_unlock(tsdn, &arena->lock);

return (extent);

}

static void

arena_extent_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena,

extent_hooks_t **r_extent_hooks, extent_t *extent)

{

malloc_mutex_assert_owner(tsdn, &arena->lock);

extent_dalloc_cache(tsdn, arena, r_extent_hooks, extent);

arena_maybe_purge(tsdn, arena);

}

void

arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,

extent_hooks_t **r_extent_hooks, extent_t *extent)

{

malloc_mutex_lock(tsdn, &arena->lock);

arena_extent_cache_dalloc_locked(tsdn, arena, r_extent_hooks, extent);

malloc_mutex_unlock(tsdn, &arena->lock);

}

void

arena_extent_cache_maybe_insert(tsdn_t *tsdn, arena_t *arena, extent_t *extent,

bool cache)

{

malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);

if (cache) {

extent_ring_insert(&arena->extents_dirty, extent);

arena->ndirty += arena_extent_dirty_npages(extent);

}

}

void

arena_extent_cache_maybe_remove(tsdn_t *tsdn, arena_t *arena, extent_t *extent,

bool dirty)

{

malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);

if (dirty) {

extent_ring_remove(extent);

assert(arena->ndirty >= arena_extent_dirty_npages(extent));

arena->ndirty -= arena_extent_dirty_npages(extent);

}

}

JEMALLOC_INLINE_C void *

arena_slab_reg_alloc(tsdn_t *tsdn, extent_t *slab,

const arena_bin_info_t *bin_info)

{

void *ret;

arena_slab_data_t *slab_data = extent_slab_data_get(slab);

size_t regind;

assert(slab_data->nfree > 0);

assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info));

regind = bitmap_sfu(slab_data->bitmap, &bin_info->bitmap_info);

ret = (void *)((uintptr_t)extent_addr_get(slab) +

(uintptr_t)(bin_info->reg_size * regind));

slab_data->nfree--;

return (ret);

}

#ifndef JEMALLOC_JET

JEMALLOC_INLINE_C

#endif

size_t

arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr)

{

size_t diff, regind;

/* Freeing a pointer outside the slab can cause assertion failure. */

assert((uintptr_t)ptr >= (uintptr_t)extent_addr_get(slab));

assert((uintptr_t)ptr < (uintptr_t)extent_past_get(slab));

/* Freeing an interior pointer can cause assertion failure. */

assert(((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)) %

(uintptr_t)arena_bin_info[binind].reg_size == 0);

/* Avoid doing division with a variable divisor. */

diff = (size_t)((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab));

switch (binind) {

#define REGIND_bin_yes(index, reg_size) \

case index: \

regind = diff / (reg_size); \

assert(diff == regind * (reg_size)); \

break;

#define REGIND_bin_no(index, reg_size)

#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \

lg_delta_lookup) \

REGIND_bin_##bin(index, (1U<<lg_grp) + (ndelta<<lg_delta))

SIZE_CLASSES

#undef REGIND_bin_yes

#undef REGIND_bin_no

#undef SC

default: not_reached();

}

assert(regind < arena_bin_info[binind].nregs);

return (regind);

}

JEMALLOC_INLINE_C void

arena_slab_reg_dalloc(tsdn_t *tsdn, extent_t *slab,

arena_slab_data_t *slab_data, void *ptr)

{

szind_t binind = slab_data->binind;

const arena_bin_info_t *bin_info = &arena_bin_info[binind];

size_t regind = arena_slab_regind(slab, binind, ptr);

assert(slab_data->nfree < bin_info->nregs);

/* Freeing an unallocated pointer can cause assertion failure. */

assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind));

bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind);

slab_data->nfree++;

}

static void

arena_nactive_add(arena_t *arena, size_t add_pages)

{

arena->nactive += add_pages;

}

static void

arena_nactive_sub(arena_t *arena, size_t sub_pages)

{

assert(arena->nactive >= sub_pages);

arena->nactive -= sub_pages;

}

static void

arena_large_malloc_stats_update(arena_t *arena, size_t usize)

{

szind_t index = size2index(usize);

szind_t hindex = (index >= NBINS) ? index - NBINS : 0;

cassert(config_stats);

arena->stats.nmalloc_large++;

arena->stats.allocated_large += usize;

arena->stats.lstats[hindex].nmalloc++;

arena->stats.lstats[hindex].nrequests++;

arena->stats.lstats[hindex].curlextents++;

}

static void

arena_large_malloc_stats_update_undo(arena_t *arena, size_t usize)

{

szind_t index = size2index(usize);

szind_t hindex = (index >= NBINS) ? index - NBINS : 0;

cassert(config_stats);

arena->stats.nmalloc_large--;

arena->stats.allocated_large -= usize;

arena->stats.lstats[hindex].nmalloc--;

arena->stats.lstats[hindex].nrequests--;

arena->stats.lstats[hindex].curlextents--;

}

static void

arena_large_dalloc_stats_update(arena_t *arena, size_t usize)

{

szind_t index = size2index(usize);

szind_t hindex = (index >= NBINS) ? index - NBINS : 0;

cassert(config_stats);

arena->stats.ndalloc_large++;

arena->stats.allocated_large -= usize;

arena->stats.lstats[hindex].ndalloc++;

arena->stats.lstats[hindex].curlextents--;

}

static void

arena_large_reset_stats_cancel(arena_t *arena, size_t usize)

{

szind_t index = size2index(usize);

szind_t hindex = (index >= NBINS) ? index - NBINS : 0;

cassert(config_stats);

arena->stats.ndalloc_large++;

arena->stats.lstats[hindex].ndalloc--;

}

static void

arena_large_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize)

{

arena_large_dalloc_stats_update(arena, oldusize);

arena_large_malloc_stats_update(arena, usize);

}

static extent_t *

arena_extent_alloc_large_hard(tsdn_t *tsdn, arena_t *arena,

extent_hooks_t **r_extent_hooks, size_t usize, size_t alignment, bool *zero)

{

extent_t *extent;

bool commit = true;

extent = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL, usize,

large_pad, alignment, zero, &commit, false);

if (extent == NULL) {

/* Revert optimistic stats updates. */

malloc_mutex_lock(tsdn, &arena->lock);

if (config_stats) {

arena_large_malloc_stats_update_undo(arena, usize);

arena->stats.mapped -= usize;

}

arena_nactive_sub(arena, (usize + large_pad) >> LG_PAGE);

malloc_mutex_unlock(tsdn, &arena->lock);

}

return (extent);

}

extent_t *

arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,

size_t alignment, bool *zero)

{

extent_t *extent;

extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;

malloc_mutex_lock(tsdn, &arena->lock);

/* Optimistically update stats. */

if (config_stats) {

arena_large_malloc_stats_update(arena, usize);

arena->stats.mapped += usize;

}

arena_nactive_add(arena, (usize + large_pad) >> LG_PAGE);

extent = arena_extent_cache_alloc_locked(tsdn, arena, &extent_hooks,

NULL, usize, large_pad, alignment, zero, false);

malloc_mutex_unlock(tsdn, &arena->lock);

if (extent == NULL) {

extent = arena_extent_alloc_large_hard(tsdn, arena,

&extent_hooks, usize, alignment, zero);

}

return (extent);

}

void

arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,

bool locked)

{

extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;

if (!locked)

malloc_mutex_lock(tsdn, &arena->lock);

else

malloc_mutex_assert_owner(tsdn, &arena->lock);

if (config_stats) {

arena_large_dalloc_stats_update(arena,

extent_usize_get(extent));

arena->stats.mapped -= extent_size_get(extent);

}

arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE);

arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, extent);

if (!locked)

malloc_mutex_unlock(tsdn, &arena->lock);

}

void

arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,

size_t oldusize)

{

size_t usize = extent_usize_get(extent);

size_t udiff = oldusize - usize;

malloc_mutex_lock(tsdn, &arena->lock);

if (config_stats) {

arena_large_ralloc_stats_update(arena, oldusize, usize);

arena->stats.mapped -= udiff;

}

arena_nactive_sub(arena, udiff >> LG_PAGE);

malloc_mutex_unlock(tsdn, &arena->lock);

}

void

arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,

size_t oldusize)

{

size_t usize = extent_usize_get(extent);

size_t udiff = usize - oldusize;

malloc_mutex_lock(tsdn, &arena->lock);

if (config_stats) {

arena_large_ralloc_stats_update(arena, oldusize, usize);

arena->stats.mapped += udiff;

}

arena_nactive_add(arena, udiff >> LG_PAGE);

malloc_mutex_unlock(tsdn, &arena->lock);

}

static void

arena_decay_deadline_init(arena_t *arena)

{

/*

* Generate a new deadline that is uniformly random within the next

* epoch after the current one.

*/

nstime_copy(&arena->decay.deadline, &arena->decay.epoch);

nstime_add(&arena->decay.deadline, &arena->decay.interval);

if (arena->decay.time > 0) {

nstime_t jitter;

nstime_init(&jitter, prng_range_u64(&arena->decay.jitter_state,

nstime_ns(&arena->decay.interval)));

nstime_add(&arena->decay.deadline, &jitter);

}

}

static bool

arena_decay_deadline_reached(const arena_t *arena, const nstime_t *time)

{

return (nstime_compare(&arena->decay.deadline, time) <= 0);

}

static size_t

arena_decay_backlog_npages_limit(const arena_t *arena)

{

static const uint64_t h_steps[] = {

#define STEP(step, h, x, y) \

h,

SMOOTHSTEP

#undef STEP

};

uint64_t sum;

size_t npages_limit_backlog;

unsigned i;

/*

* For each element of decay_backlog, multiply by the corresponding

* fixed-point smoothstep decay factor. Sum the products, then divide

* to round down to the nearest whole number of pages.

*/

sum = 0;

for (i = 0; i < SMOOTHSTEP_NSTEPS; i++)

sum += arena->decay.backlog[i] * h_steps[i];

npages_limit_backlog = (size_t)(sum >> SMOOTHSTEP_BFP);

return (npages_limit_backlog);

}

static void

arena_decay_backlog_update_last(arena_t *arena)

{

size_t ndirty_delta = (arena->ndirty > arena->decay.nunpurged) ?

arena->ndirty - arena->decay.nunpurged : 0;

arena->decay.backlog[SMOOTHSTEP_NSTEPS-1] = ndirty_delta;

}

static void

arena_decay_backlog_update(arena_t *arena, uint64_t nadvance_u64)

{

if (nadvance_u64 >= SMOOTHSTEP_NSTEPS) {

memset(arena->decay.backlog, 0, (SMOOTHSTEP_NSTEPS-1) *

sizeof(size_t));

} else {

size_t nadvance_z = (size_t)nadvance_u64;

assert((uint64_t)nadvance_z == nadvance_u64);

memmove(arena->decay.backlog, &arena->decay.backlog[nadvance_z],

(SMOOTHSTEP_NSTEPS - nadvance_z) * sizeof(size_t));

if (nadvance_z > 1) {

memset(&arena->decay.backlog[SMOOTHSTEP_NSTEPS -

nadvance_z], 0, (nadvance_z-1) * sizeof(size_t));

}

}

arena_decay_backlog_update_last(arena);

}

static void

arena_decay_epoch_advance_helper(arena_t *arena, const nstime_t *time)

{

uint64_t nadvance_u64;

nstime_t delta;

assert(arena_decay_deadline_reached(arena, time));

nstime_copy(&delta, time);

nstime_subtract(&delta, &arena->decay.epoch);

nadvance_u64 = nstime_divide(&delta, &arena->decay.interval);

assert(nadvance_u64 > 0);

/* Add nadvance_u64 decay intervals to epoch. */

nstime_copy(&delta, &arena->decay.interval);

nstime_imultiply(&delta, nadvance_u64);

nstime_add(&arena->decay.epoch, &delta);

/* Set a new deadline. */

arena_decay_deadline_init(arena);

/* Update the backlog. */

arena_decay_backlog_update(arena, nadvance_u64);

}

static void

arena_decay_epoch_advance_purge(tsdn_t *tsdn, arena_t *arena)

{

size_t ndirty_limit = arena_decay_backlog_npages_limit(arena);

if (arena->ndirty > ndirty_limit)

arena_purge_to_limit(tsdn, arena, ndirty_limit);

arena->decay.nunpurged = arena->ndirty;

}

static void

arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, const nstime_t *time)

{

arena_decay_epoch_advance_helper(arena, time);

arena_decay_epoch_advance_purge(tsdn, arena);

}

static void

arena_decay_init(arena_t *arena, ssize_t decay_time)

{

arena->decay.time = decay_time;

if (decay_time > 0) {

nstime_init2(&arena->decay.interval, decay_time, 0);

nstime_idivide(&arena->decay.interval, SMOOTHSTEP_NSTEPS);

}

nstime_init(&arena->decay.epoch, 0);

nstime_update(&arena->decay.epoch);

arena->decay.jitter_state = (uint64_t)(uintptr_t)arena;

arena_decay_deadline_init(arena);

arena->decay.nunpurged = arena->ndirty;

memset(arena->decay.backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t));

}

static bool

arena_decay_time_valid(ssize_t decay_time)

{

if (decay_time < -1)

return (false);

if (decay_time == -1 || (uint64_t)decay_time <= NSTIME_SEC_MAX)

return (true);

return (false);

}

ssize_t

arena_decay_time_get(tsdn_t *tsdn, arena_t *arena)

{

ssize_t decay_time;

malloc_mutex_lock(tsdn, &arena->lock);

decay_time = arena->decay.time;

malloc_mutex_unlock(tsdn, &arena->lock);

return (decay_time);

}

bool

arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time)

{

if (!arena_decay_time_valid(decay_time))

return (true);

malloc_mutex_lock(tsdn, &arena->lock);

/*

* Restart decay backlog from scratch, which may cause many dirty pages

* to be immediately purged. It would conceptually be possible to map

* the old backlog onto the new backlog, but there is no justification

* for such complexity since decay_time changes are intended to be

* infrequent, either between the {-1, 0, >0} states, or a one-time

* arbitrary change during initial arena configuration.

*/

arena_decay_init(arena, decay_time);

arena_maybe_purge(tsdn, arena);

malloc_mutex_unlock(tsdn, &arena->lock);

return (false);

}

static void

arena_maybe_purge_helper(tsdn_t *tsdn, arena_t *arena)

{

nstime_t time;

/* Purge all or nothing if the option is disabled. */

if (arena->decay.time <= 0) {

if (arena->decay.time == 0)

arena_purge_to_limit(tsdn, arena, 0);

return;

}

nstime_init(&time, 0);

nstime_update(&time);

if (unlikely(!nstime_monotonic() && nstime_compare(&arena->decay.epoch,

&time) > 0)) {

/*

* Time went backwards. Move the epoch back in time and

* generate a new deadline, with the expectation that time

* typically flows forward for long enough periods of time that

* epochs complete. Unfortunately, this strategy is susceptible

* to clock jitter triggering premature epoch advances, but

* clock jitter estimation and compensation isn't feasible here

* because calls into this code are event-driven.

*/

nstime_copy(&arena->decay.epoch, &time);

arena_decay_deadline_init(arena);

} else {

/* Verify that time does not go backwards. */

assert(nstime_compare(&arena->decay.epoch, &time) <= 0);

}

/*

* If the deadline has been reached, advance to the current epoch and

* purge to the new limit if necessary. Note that dirty pages created

* during the current epoch are not subject to purge until a future

* epoch, so as a result purging only happens during epoch advances.

*/

if (arena_decay_deadline_reached(arena, &time))

arena_decay_epoch_advance(tsdn, arena, &time);

}

void

arena_maybe_purge(tsdn_t *tsdn, arena_t *arena)

{

malloc_mutex_assert_owner(tsdn, &arena->lock);

/* Don't recursively purge. */

if (arena->purging)

return;

arena_maybe_purge_helper(tsdn, arena);

}

static size_t

arena_dirty_count(tsdn_t *tsdn, arena_t *arena)

{

extent_t *extent;

size_t ndirty = 0;

malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);

for (extent = qr_next(&arena->extents_dirty, qr_link); extent !=

&arena->extents_dirty; extent = qr_next(extent, qr_link))

ndirty += extent_size_get(extent) >> LG_PAGE;

return (ndirty);

}

static size_t

arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,

size_t ndirty_limit, extent_t *purge_extents_sentinel)

{

extent_t *extent, *next;

size_t nstashed = 0;

malloc_mutex_lock(tsdn, &arena->extents_mtx);

/* Stash extents according to ndirty_limit. */

for (extent = qr_next(&arena->extents_dirty, qr_link); extent !=

&arena->extents_dirty; extent = next) {

size_t npages;

bool zero, commit;

UNUSED extent_t *textent;

npages = extent_size_get(extent) >> LG_PAGE;

if (arena->ndirty - (nstashed + npages) < ndirty_limit)

break;

next = qr_next(extent, qr_link);

/* Allocate. */

zero = false;

commit = false;

textent = extent_alloc_cache_locked(tsdn, arena, r_extent_hooks,

extent_base_get(extent), extent_size_get(extent), 0, PAGE,

&zero, &commit, false);

assert(textent == extent);

assert(zero == extent_zeroed_get(extent));

extent_ring_remove(extent);

extent_ring_insert(purge_extents_sentinel, extent);

nstashed += npages;

}

malloc_mutex_unlock(tsdn, &arena->extents_mtx);

return (nstashed);

}

static size_t

arena_purge_stashed(tsdn_t *tsdn, arena_t *arena,

extent_hooks_t **r_extent_hooks, extent_t *purge_extents_sentinel)

{

UNUSED size_t nmadvise;

size_t npurged;

extent_t *extent, *next;

if (config_stats)

nmadvise = 0;

npurged = 0;

for (extent = qr_next(purge_extents_sentinel, qr_link); extent !=

purge_extents_sentinel; extent = next) {

if (config_stats)

nmadvise++;

npurged += extent_size_get(extent) >> LG_PAGE;

next = qr_next(extent, qr_link);

extent_ring_remove(extent);

extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, extent);

}

if (config_stats) {

arena->stats.nmadvise += nmadvise;

arena->stats.purged += npurged;

}

return (npurged);

}

/*

* ndirty_limit: Purge as many dirty extents as possible without violating the

* invariant: (arena->ndirty >= ndirty_limit)

*/

static void

arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit)

{

extent_hooks_t *extent_hooks = extent_hooks_get(arena);

size_t npurge, npurged;

extent_t purge_extents_sentinel;

arena->purging = true;

/*

* Calls to arena_dirty_count() are disabled even for debug builds

* because overhead grows nonlinearly as memory usage increases.

*/

if (false && config_debug) {

size_t ndirty = arena_dirty_count(tsdn, arena);

assert(ndirty == arena->ndirty);

}

extent_init(&purge_extents_sentinel, arena, NULL, 0, 0, 0, false, false,

false, false);

npurge = arena_stash_dirty(tsdn, arena, &extent_hooks, ndirty_limit,

&purge_extents_sentinel);

if (npurge == 0)

goto label_return;

npurged = arena_purge_stashed(tsdn, arena, &extent_hooks,

&purge_extents_sentinel);

assert(npurged == npurge);

if (config_stats)

arena->stats.npurge++;

label_return:

arena->purging = false;

}

void

arena_purge(tsdn_t *tsdn, arena_t *arena, bool all)

{

malloc_mutex_lock(tsdn, &arena->lock);

if (all)

arena_purge_to_limit(tsdn, arena, 0);

else

arena_maybe_purge(tsdn, arena);

malloc_mutex_unlock(tsdn, &arena->lock);

}

static void

arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab)

{

extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;

arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE);

arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, slab);

}

static void

arena_bin_slabs_nonfull_insert(arena_bin_t *bin, extent_t *slab)

{

assert(extent_slab_data_get(slab)->nfree > 0);

extent_heap_insert(&bin->slabs_nonfull, slab);

}

static void

arena_bin_slabs_nonfull_remove(arena_bin_t *bin, extent_t *slab)

{

extent_heap_remove(&bin->slabs_nonfull, slab);

}

static extent_t *

arena_bin_slabs_nonfull_tryget(arena_bin_t *bin)

{

extent_t *slab = extent_heap_remove_first(&bin->slabs_nonfull);

if (slab == NULL)

return (NULL);

if (config_stats)

bin->stats.reslabs++;

return (slab);

}

static void

arena_bin_slabs_full_insert(arena_bin_t *bin, extent_t *slab)

{

assert(extent_slab_data_get(slab)->nfree == 0);

extent_ring_insert(&bin->slabs_full, slab);

}

static void

arena_bin_slabs_full_remove(extent_t *slab)

{

extent_ring_remove(slab);

}

void

arena_reset(tsd_t *tsd, arena_t *arena)

{

unsigned i;

extent_t *extent;

/*

* Locking in this function is unintuitive. The caller guarantees that

* no concurrent operations are happening in this arena, but there are

* still reasons that some locking is necessary:

*

* - Some of the functions in the transitive closure of calls assume

* appropriate locks are held, and in some cases these locks are

* temporarily dropped to avoid lock order reversal or deadlock due to

* reentry.

* - mallctl("epoch", ...) may concurrently refresh stats. While

* strictly speaking this is a "concurrent operation", disallowing

* stats refreshes would impose an inconvenient burden.

*/

/* Large allocations. */

malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);

for (extent = ql_last(&arena->large, ql_link); extent != NULL; extent =

ql_last(&arena->large, ql_link)) {

void *ptr = extent_base_get(extent);

size_t usize;

malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);

if (config_stats || (config_prof && opt_prof))

usize = isalloc(tsd_tsdn(tsd), extent, ptr);

/* Remove large allocation from prof sample set. */

if (config_prof && opt_prof)

prof_free(tsd, extent, ptr, usize);

large_dalloc(tsd_tsdn(tsd), extent);

malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);

/* Cancel out unwanted effects on stats. */

if (config_stats)

arena_large_reset_stats_cancel(arena, usize);

}

malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);

malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);

/* Bins. */

for (i = 0; i < NBINS; i++) {

extent_t *slab;

arena_bin_t *bin = &arena->bins[i];

malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);

if (bin->slabcur != NULL) {

slab = bin->slabcur;

bin->slabcur = NULL;

malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);

arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);

malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);

}

while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) !=

NULL) {

malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);

arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);

malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);

}

for (slab = qr_next(&bin->slabs_full, qr_link); slab !=

&bin->slabs_full; slab = qr_next(&bin->slabs_full,

qr_link)) {

arena_bin_slabs_full_remove(slab);

malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);

arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);

malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);

}

if (config_stats) {

bin->stats.curregs = 0;

bin->stats.curslabs = 0;

}

malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);

}

assert(!arena->purging);

arena->nactive = 0;

malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);

}

static extent_t *

arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,

extent_hooks_t **r_extent_hooks, const arena_bin_info_t *bin_info)

{

extent_t *slab;

bool zero, commit;

zero = false;

commit = true;

malloc_mutex_unlock(tsdn, &arena->lock);

slab = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL,

bin_info->slab_size, 0, PAGE, &zero, &commit, true);

malloc_mutex_lock(tsdn, &arena->lock);

return (slab);

}

static extent_t *

arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,

const arena_bin_info_t *bin_info)

{

extent_t *slab;

arena_slab_data_t *slab_data;

extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;

bool zero = false;

slab = arena_extent_cache_alloc_locked(tsdn, arena, &extent_hooks, NULL,

bin_info->slab_size, 0, PAGE, &zero, true);

if (slab == NULL) {

slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks,

bin_info);

if (slab == NULL)

return (NULL);

}

assert(extent_slab_get(slab));

arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE);

/* Initialize slab internals. */

slab_data = extent_slab_data_get(slab);

slab_data->binind = binind;

slab_data->nfree = bin_info->nregs;

bitmap_init(slab_data->bitmap, &bin_info->bitmap_info);

if (config_stats)

arena->stats.mapped += extent_size_get(slab);

return (slab);

}

static extent_t *

arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin,

szind_t binind)

{

extent_t *slab;

const arena_bin_info_t *bin_info;

/* Look for a usable slab. */

slab = arena_bin_slabs_nonfull_tryget(bin);

if (slab != NULL)

return (slab);

/* No existing slabs have any space available. */

bin_info = &arena_bin_info[binind];

/* Allocate a new slab. */

malloc_mutex_unlock(tsdn, &bin->lock);

/******************************/

malloc_mutex_lock(tsdn, &arena->lock);

slab = arena_slab_alloc(tsdn, arena, binind, bin_info);

malloc_mutex_unlock(tsdn, &arena->lock);

/********************************/

malloc_mutex_lock(tsdn, &bin->lock);

if (slab != NULL) {

if (config_stats) {

bin->stats.nslabs++;

bin->stats.curslabs++;

}

return (slab);

}

/*

* arena_slab_alloc() failed, but another thread may have made

* sufficient memory available while this one dropped bin->lock above,

* so search one more time.

*/

slab = arena_bin_slabs_nonfull_tryget(bin);

if (slab != NULL)

return (slab);

return (NULL);

}

/* Re-fill bin->slabcur, then call arena_slab_reg_alloc(). */

static void *

arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin,

szind_t binind)

{

const arena_bin_info_t *bin_info;

extent_t *slab;

bin_info = &arena_bin_info[binind];

if (bin->slabcur != NULL) {

arena_bin_slabs_full_insert(bin, bin->slabcur);

bin->slabcur = NULL;

}

slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind);

if (bin->slabcur != NULL) {

/*