Returns whether or not the free attempt was successful. */
| 3062 | |
| 3063 | /* Returns whether or not the free attempt was successful. */ |
| 3064 | JEMALLOC_ALWAYS_INLINE |
| 3065 | bool free_fastpath(void *ptr, size_t size, bool size_hint) { |
| 3066 | tsd_t *tsd = tsd_get(false); |
| 3067 | /* The branch gets optimized away unless tsd_get_allocates(). */ |
| 3068 | if (unlikely(tsd == NULL)) { |
| 3069 | return false; |
| 3070 | } |
| 3071 | /* |
| 3072 | * The tsd_fast() / initialized checks are folded into the branch |
| 3073 | * testing (deallocated_after >= threshold) later in this function. |
| 3074 | * The threshold will be set to 0 when !tsd_fast. |
| 3075 | */ |
| 3076 | assert(tsd_fast(tsd) || |
| 3077 | *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) == 0); |
| 3078 | |
| 3079 | emap_alloc_ctx_t alloc_ctx; |
| 3080 | if (!size_hint) { |
| 3081 | bool err = emap_alloc_ctx_try_lookup_fast(tsd, |
| 3082 | &arena_emap_global, ptr, &alloc_ctx); |
| 3083 | |
| 3084 | /* Note: profiled objects will have alloc_ctx.slab set */ |
| 3085 | if (unlikely(err || !alloc_ctx.slab || |
| 3086 | free_fastpath_nonfast_aligned(ptr, |
| 3087 | /* check_prof */ false))) { |
| 3088 | return false; |
| 3089 | } |
| 3090 | assert(alloc_ctx.szind != SC_NSIZES); |
| 3091 | } else { |
| 3092 | /* |
| 3093 | * Check for both sizes that are too large, and for sampled / |
| 3094 | * special aligned objects. The alignment check will also check |
| 3095 | * for null ptr. |
| 3096 | */ |
| 3097 | if (unlikely(size > SC_LOOKUP_MAXCLASS || |
| 3098 | free_fastpath_nonfast_aligned(ptr, |
| 3099 | /* check_prof */ true))) { |
| 3100 | return false; |
| 3101 | } |
| 3102 | alloc_ctx.szind = sz_size2index_lookup(size); |
| 3103 | /* Max lookup class must be small. */ |
| 3104 | assert(alloc_ctx.szind < SC_NBINS); |
| 3105 | /* This is a dead store, except when opt size checking is on. */ |
| 3106 | alloc_ctx.slab = true; |
| 3107 | } |
| 3108 | /* |
| 3109 | * Currently the fastpath only handles small sizes. The branch on |
| 3110 | * SC_LOOKUP_MAXCLASS makes sure of it. This lets us avoid checking |
| 3111 | * tcache szind upper limit (i.e. tcache_maxclass) as well. |
| 3112 | */ |
| 3113 | assert(alloc_ctx.slab); |
| 3114 | |
| 3115 | uint64_t deallocated, threshold; |
| 3116 | te_free_fastpath_ctx(tsd, &deallocated, &threshold); |
| 3117 | |
| 3118 | size_t usize = sz_index2size(alloc_ctx.szind); |
| 3119 | uint64_t deallocated_after = deallocated + usize; |
| 3120 | /* |
| 3121 | * Check for events and tsd non-nominal (fast_threshold will be set to |
no test coverage detected