* multi-producer safe lock-free ring buffer enqueue * */
| 60 | * |
| 61 | */ |
| 62 | static __inline int |
| 63 | buf_ring_enqueue(struct buf_ring *br, void *buf) |
| 64 | { |
| 65 | uint32_t prod_head, prod_next, cons_tail; |
| 66 | #ifdef DEBUG_BUFRING |
| 67 | int i; |
| 68 | |
| 69 | /* |
| 70 | * Note: It is possible to encounter an mbuf that was removed |
| 71 | * via drbr_peek(), and then re-added via drbr_putback() and |
| 72 | * trigger a spurious panic. |
| 73 | */ |
| 74 | for (i = br->br_cons_head; i != br->br_prod_head; |
| 75 | i = ((i + 1) & br->br_cons_mask)) |
| 76 | if(br->br_ring[i] == buf) |
| 77 | panic("buf=%p already enqueue at %d prod=%d cons=%d", |
| 78 | buf, i, br->br_prod_tail, br->br_cons_tail); |
| 79 | #endif |
| 80 | critical_enter(); |
| 81 | do { |
| 82 | prod_head = br->br_prod_head; |
| 83 | prod_next = (prod_head + 1) & br->br_prod_mask; |
| 84 | cons_tail = br->br_cons_tail; |
| 85 | |
| 86 | if (prod_next == cons_tail) { |
| 87 | rmb(); |
| 88 | if (prod_head == br->br_prod_head && |
| 89 | cons_tail == br->br_cons_tail) { |
| 90 | br->br_drops++; |
| 91 | critical_exit(); |
| 92 | return (ENOBUFS); |
| 93 | } |
| 94 | continue; |
| 95 | } |
| 96 | } while (!atomic_cmpset_acq_int(&br->br_prod_head, prod_head, prod_next)); |
| 97 | #ifdef DEBUG_BUFRING |
| 98 | if (br->br_ring[prod_head] != NULL) |
| 99 | panic("dangling value in enqueue"); |
| 100 | #endif |
| 101 | br->br_ring[prod_head] = buf; |
| 102 | |
| 103 | /* |
| 104 | * If there are other enqueues in progress |
| 105 | * that preceded us, we need to wait for them |
| 106 | * to complete |
| 107 | */ |
| 108 | while (br->br_prod_tail != prod_head) |
| 109 | cpu_spinwait(); |
| 110 | atomic_store_rel_int(&br->br_prod_tail, prod_next); |
| 111 | critical_exit(); |
| 112 | return (0); |
| 113 | } |
| 114 | |
| 115 | /* |
| 116 | * multi-consumer safe dequeue |
no test coverage detected