| 3569 | } |
| 3570 | |
| 3571 | void * |
| 3572 | uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) |
| 3573 | { |
| 3574 | #ifdef NUMA |
| 3575 | uma_bucket_t bucket; |
| 3576 | uma_zone_domain_t zdom; |
| 3577 | void *item; |
| 3578 | #endif |
| 3579 | |
| 3580 | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ |
| 3581 | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); |
| 3582 | |
| 3583 | /* This is the fast path allocation */ |
| 3584 | CTR4(KTR_UMA, "uma_zalloc_domain zone %s(%p) domain %d flags %d", |
| 3585 | zone->uz_name, zone, domain, flags); |
| 3586 | |
| 3587 | if (flags & M_WAITOK) { |
| 3588 | WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, |
| 3589 | "uma_zalloc_domain: zone \"%s\"", zone->uz_name); |
| 3590 | } |
| 3591 | KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), |
| 3592 | ("uma_zalloc_domain: called with spinlock or critical section held")); |
| 3593 | KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, |
| 3594 | ("uma_zalloc_domain: called with SMR zone.")); |
| 3595 | #ifdef NUMA |
| 3596 | KASSERT((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0, |
| 3597 | ("uma_zalloc_domain: called with non-FIRSTTOUCH zone.")); |
| 3598 | |
| 3599 | if (vm_ndomains == 1) |
| 3600 | return (uma_zalloc_arg(zone, udata, flags)); |
| 3601 | |
| 3602 | /* |
| 3603 | * Try to allocate from the bucket cache before falling back to the keg. |
| 3604 | * We could try harder and attempt to allocate from per-CPU caches or |
| 3605 | * the per-domain cross-domain buckets, but the complexity is probably |
| 3606 | * not worth it. It is more important that frees of previous |
| 3607 | * cross-domain allocations do not blow up the cache. |
| 3608 | */ |
| 3609 | zdom = zone_domain_lock(zone, domain); |
| 3610 | if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) { |
| 3611 | item = bucket->ub_bucket[bucket->ub_cnt - 1]; |
| 3612 | #ifdef INVARIANTS |
| 3613 | bucket->ub_bucket[bucket->ub_cnt - 1] = NULL; |
| 3614 | #endif |
| 3615 | bucket->ub_cnt--; |
| 3616 | zone_put_bucket(zone, domain, bucket, udata, true); |
| 3617 | item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, |
| 3618 | flags, item); |
| 3619 | if (item != NULL) { |
| 3620 | KASSERT(item_domain(item) == domain, |
| 3621 | ("%s: bucket cache item %p from wrong domain", |
| 3622 | __func__, item)); |
| 3623 | counter_u64_add(zone->uz_allocs, 1); |
| 3624 | } |
| 3625 | return (item); |
| 3626 | } |
| 3627 | ZDOM_UNLOCK(zdom); |
| 3628 | return (zone_alloc_item(zone, udata, domain, flags)); |
no test coverage detected