* Replenish an alloc bucket and possibly restore an old one. Called in * a critical section. Returns in a critical section. * * A false return value indicates an allocation failure. * A true return value indicates success and the caller should retry. */
| 3478 | * A true return value indicates success and the caller should retry. |
| 3479 | */ |
| 3480 | static __noinline bool |
| 3481 | cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) |
| 3482 | { |
| 3483 | uma_bucket_t bucket; |
| 3484 | int curdomain, domain; |
| 3485 | bool new; |
| 3486 | |
| 3487 | CRITICAL_ASSERT(curthread); |
| 3488 | |
| 3489 | /* |
| 3490 | * If we have run out of items in our alloc bucket see |
| 3491 | * if we can switch with the free bucket. |
| 3492 | * |
| 3493 | * SMR Zones can't re-use the free bucket until the sequence has |
| 3494 | * expired. |
| 3495 | */ |
| 3496 | if ((cache_uz_flags(cache) & UMA_ZONE_SMR) == 0 && |
| 3497 | cache->uc_freebucket.ucb_cnt != 0) { |
| 3498 | cache_bucket_swap(&cache->uc_freebucket, |
| 3499 | &cache->uc_allocbucket); |
| 3500 | return (true); |
| 3501 | } |
| 3502 | |
| 3503 | /* |
| 3504 | * Discard any empty allocation bucket while we hold no locks. |
| 3505 | */ |
| 3506 | bucket = cache_bucket_unload_alloc(cache); |
| 3507 | critical_exit(); |
| 3508 | |
| 3509 | if (bucket != NULL) { |
| 3510 | KASSERT(bucket->ub_cnt == 0, |
| 3511 | ("cache_alloc: Entered with non-empty alloc bucket.")); |
| 3512 | bucket_free(zone, bucket, udata); |
| 3513 | } |
| 3514 | |
| 3515 | /* |
| 3516 | * Attempt to retrieve the item from the per-CPU cache has failed, so |
| 3517 | * we must go back to the zone. This requires the zdom lock, so we |
| 3518 | * must drop the critical section, then re-acquire it when we go back |
| 3519 | * to the cache. Since the critical section is released, we may be |
| 3520 | * preempted or migrate. As such, make sure not to maintain any |
| 3521 | * thread-local state specific to the cache from prior to releasing |
| 3522 | * the critical section. |
| 3523 | */ |
| 3524 | domain = PCPU_GET(domain); |
| 3525 | if ((cache_uz_flags(cache) & UMA_ZONE_ROUNDROBIN) != 0 || |
| 3526 | VM_DOMAIN_EMPTY(domain)) |
| 3527 | domain = zone_domain_highest(zone, domain); |
| 3528 | bucket = cache_fetch_bucket(zone, cache, domain); |
| 3529 | if (bucket == NULL && zone->uz_bucket_size != 0 && !bucketdisable) { |
| 3530 | bucket = zone_alloc_bucket(zone, udata, domain, flags); |
| 3531 | new = true; |
| 3532 | } else { |
| 3533 | new = false; |
| 3534 | } |
| 3535 | |
| 3536 | CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p", |
| 3537 | zone->uz_name, zone, bucket); |
no test coverage detected