| 263 | |
| 264 | |
| 265 | static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) |
| 266 | { |
| 267 | mi_assert_internal(blocks <= MI_BITMAP_FIELD_BITS); |
| 268 | mem_region_t* region; |
| 269 | mi_bitmap_index_t bit_idx; |
| 270 | const int numa_node = (_mi_os_numa_node_count() <= 1 ? -1 : _mi_os_numa_node(tld)); |
| 271 | // try to claim in existing regions |
| 272 | if (!mi_region_try_claim(numa_node, blocks, *large, ®ion, &bit_idx, tld)) { |
| 273 | // otherwise try to allocate a fresh region and claim in there |
| 274 | if (!mi_region_try_alloc_os(blocks, *commit, *large, ®ion, &bit_idx, tld)) { |
| 275 | // out of regions or memory |
| 276 | return NULL; |
| 277 | } |
| 278 | } |
| 279 | |
| 280 | // ------------------------------------------------ |
| 281 | // found a region and claimed `blocks` at `bit_idx`, initialize them now |
| 282 | mi_assert_internal(region != NULL); |
| 283 | mi_assert_internal(_mi_bitmap_is_claimed(®ion->in_use, 1, blocks, bit_idx)); |
| 284 | |
| 285 | mi_region_info_t info; |
| 286 | info.value = mi_atomic_load_acquire(®ion->info); |
| 287 | uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t,®ion->start); |
| 288 | mi_assert_internal(!(info.x.is_large && !*large)); |
| 289 | mi_assert_internal(start != NULL); |
| 290 | |
| 291 | *is_zero = _mi_bitmap_claim(®ion->dirty, 1, blocks, bit_idx, NULL); |
| 292 | *large = info.x.is_large; |
| 293 | *is_pinned = info.x.is_pinned; |
| 294 | *memid = mi_memid_create(region, bit_idx); |
| 295 | void* p = start + (mi_bitmap_index_bit_in_field(bit_idx) * MI_SEGMENT_SIZE); |
| 296 | |
| 297 | // commit |
| 298 | if (*commit) { |
| 299 | // ensure commit |
| 300 | bool any_uncommitted; |
| 301 | _mi_bitmap_claim(®ion->commit, 1, blocks, bit_idx, &any_uncommitted); |
| 302 | if (any_uncommitted) { |
| 303 | mi_assert_internal(!info.x.is_large && !info.x.is_pinned); |
| 304 | bool commit_zero = false; |
| 305 | if (!_mi_mem_commit(p, blocks * MI_SEGMENT_SIZE, &commit_zero, tld)) { |
| 306 | // failed to commit! unclaim and return |
| 307 | mi_bitmap_unclaim(®ion->in_use, 1, blocks, bit_idx); |
| 308 | return NULL; |
| 309 | } |
| 310 | if (commit_zero) *is_zero = true; |
| 311 | } |
| 312 | } |
| 313 | else { |
| 314 | // no need to commit, but check if already fully committed |
| 315 | *commit = _mi_bitmap_is_claimed(®ion->commit, 1, blocks, bit_idx); |
| 316 | } |
| 317 | mi_assert_internal(!*commit || _mi_bitmap_is_claimed(®ion->commit, 1, blocks, bit_idx)); |
| 318 | |
| 319 | // unreset reset blocks |
| 320 | if (_mi_bitmap_is_any_claimed(®ion->reset, 1, blocks, bit_idx)) { |
| 321 | // some blocks are still reset |
| 322 | mi_assert_internal(!info.x.is_large && !info.x.is_pinned); |
no test coverage detected