* Populate a free or cross bucket for the current cpu cache. Free any * existing full bucket either to the zone cache or back to the slab layer. * * Enters and returns in a critical section. false return indicates that * we can not satisfy this free in the cache layer. true indicates that * the caller should retry. */
| 4405 | * the caller should retry. |
| 4406 | */ |
| 4407 | static __noinline bool |
| 4408 | cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, void *item, |
| 4409 | int itemdomain) |
| 4410 | { |
| 4411 | uma_cache_bucket_t cbucket; |
| 4412 | uma_bucket_t newbucket, bucket; |
| 4413 | |
| 4414 | CRITICAL_ASSERT(curthread); |
| 4415 | |
| 4416 | if (zone->uz_bucket_size == 0) |
| 4417 | return false; |
| 4418 | |
| 4419 | cache = &zone->uz_cpu[curcpu]; |
| 4420 | newbucket = NULL; |
| 4421 | |
| 4422 | /* |
| 4423 | * FIRSTTOUCH domains need to free to the correct zdom. When |
| 4424 | * enabled this is the zdom of the item. The bucket is the |
| 4425 | * cross bucket if the current domain and itemdomain do not match. |
| 4426 | */ |
| 4427 | cbucket = &cache->uc_freebucket; |
| 4428 | #ifdef NUMA |
| 4429 | if ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) != 0) { |
| 4430 | if (PCPU_GET(domain) != itemdomain) { |
| 4431 | cbucket = &cache->uc_crossbucket; |
| 4432 | if (cbucket->ucb_cnt != 0) |
| 4433 | counter_u64_add(zone->uz_xdomain, |
| 4434 | cbucket->ucb_cnt); |
| 4435 | } |
| 4436 | } |
| 4437 | #endif |
| 4438 | bucket = cache_bucket_unload(cbucket); |
| 4439 | KASSERT(bucket == NULL || bucket->ub_cnt == bucket->ub_entries, |
| 4440 | ("cache_free: Entered with non-full free bucket.")); |
| 4441 | |
| 4442 | /* We are no longer associated with this CPU. */ |
| 4443 | critical_exit(); |
| 4444 | |
| 4445 | /* |
| 4446 | * Don't let SMR zones operate without a free bucket. Force |
| 4447 | * a synchronize and re-use this one. We will only degrade |
| 4448 | * to a synchronize every bucket_size items rather than every |
| 4449 | * item if we fail to allocate a bucket. |
| 4450 | */ |
| 4451 | if ((zone->uz_flags & UMA_ZONE_SMR) != 0) { |
| 4452 | if (bucket != NULL) |
| 4453 | bucket->ub_seq = smr_advance(zone->uz_smr); |
| 4454 | newbucket = bucket_alloc(zone, udata, M_NOWAIT); |
| 4455 | if (newbucket == NULL && bucket != NULL) { |
| 4456 | bucket_drain(zone, bucket); |
| 4457 | newbucket = bucket; |
| 4458 | bucket = NULL; |
| 4459 | } |
| 4460 | } else if (!bucketdisable) |
| 4461 | newbucket = bucket_alloc(zone, udata, M_NOWAIT); |
| 4462 | |
| 4463 | if (bucket != NULL) |
| 4464 | zone_free_bucket(zone, bucket, udata, itemdomain, true); |
no test coverage detected