| 538 | #else |
| 539 | template<AllocationMode canAlloc, typename U> |
| 540 | bool inner_enqueue(U&& element) AE_NO_TSAN |
| 541 | #endif |
| 542 | { |
| 543 | #ifndef NDEBUG |
| 544 | ReentrantGuard guard(this->enqueuing); |
| 545 | #endif |
| 546 | |
| 547 | // High-level pseudocode (assuming we're allowed to alloc a new block): |
| 548 | // If room in tail block, add to tail |
| 549 | // Else check next block |
| 550 | // If next block is not the head block, enqueue on next block |
| 551 | // Else create a new block and enqueue there |
| 552 | // Advance tail to the block we just enqueued to |
| 553 | |
| 554 | Block* tailBlock_ = tailBlock.load(); |
| 555 | size_t blockFront = tailBlock_->localFront; |
| 556 | size_t blockTail = tailBlock_->tail.load(); |
| 557 | |
| 558 | size_t nextBlockTail = (blockTail + 1) & tailBlock_->sizeMask; |
| 559 | if (nextBlockTail != blockFront |
| 560 | || nextBlockTail != (tailBlock_->localFront = tailBlock_->front.load())) { |
| 561 | fence(memory_order_acquire); |
| 562 | // This block has room for at least one more element |
| 563 | char* location = tailBlock_->data + blockTail * sizeof(T); |
| 564 | #if MOODYCAMEL_HAS_EMPLACE |
| 565 | new (location) T(std::forward<Args>(args)...); |
| 566 | #else |
| 567 | new (location) T(std::forward<U>(element)); |
| 568 | #endif |
| 569 | |
| 570 | fence(memory_order_release); |
| 571 | tailBlock_->tail = nextBlockTail; |
| 572 | } else { |
| 573 | fence(memory_order_acquire); |
| 574 | if (tailBlock_->next.load() != frontBlock) { |
| 575 | // Note that the reason we can't advance to the frontBlock and start |
| 576 | // adding new entries there is because if we did, then dequeue would |
| 577 | // stay in that block, eventually reading the new values, instead of |
| 578 | // advancing to the next full block (whose values were enqueued first |
| 579 | // and so should be consumed first). |
| 580 | |
| 581 | fence(memory_order_acquire); // Ensure we get latest writes if we got |
| 582 | // the latest frontBlock |
| 583 | |
| 584 | // tailBlock is full, but there's a free block ahead, use it |
| 585 | Block* tailBlockNext = tailBlock_->next.load(); |
| 586 | size_t nextBlockFront = tailBlockNext->localFront = tailBlockNext->front.load(); |
| 587 | nextBlockTail = tailBlockNext->tail.load(); |
| 588 | fence(memory_order_acquire); |
| 589 | |
| 590 | // This block must be empty since it's not the head block and we |
| 591 | // go through the blocks in a circle |
| 592 | assert(nextBlockFront == nextBlockTail); |
| 593 | tailBlockNext->localFront = nextBlockFront; |
| 594 | |
| 595 | char* location = tailBlockNext->data + nextBlockTail * sizeof(T); |
| 596 | #if MOODYCAMEL_HAS_EMPLACE |
| 597 | new (location) T(std::forward<Args>(args)...); |
no test coverage detected