| 93 | |
| 94 | #ifdef MP_RING_NO_64BIT_ATOMICS |
| 95 | static void |
| 96 | drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget) |
| 97 | { |
| 98 | union ring_state ns; |
| 99 | int n, pending, total; |
| 100 | uint16_t cidx = os.cidx; |
| 101 | uint16_t pidx = os.pidx_tail; |
| 102 | |
| 103 | MPASS(os.flags == BUSY); |
| 104 | MPASS(cidx != pidx); |
| 105 | |
| 106 | if (prev == IDLE) |
| 107 | counter_u64_add(r->starts, 1); |
| 108 | pending = 0; |
| 109 | total = 0; |
| 110 | |
| 111 | while (cidx != pidx) { |
| 112 | /* Items from cidx to pidx are available for consumption. */ |
| 113 | n = r->drain(r, cidx, pidx); |
| 114 | if (n == 0) { |
| 115 | os.state = ns.state = r->state; |
| 116 | ns.cidx = cidx; |
| 117 | ns.flags = STALLED; |
| 118 | r->state = ns.state; |
| 119 | if (prev != STALLED) |
| 120 | counter_u64_add(r->stalls, 1); |
| 121 | else if (total > 0) { |
| 122 | counter_u64_add(r->restarts, 1); |
| 123 | counter_u64_add(r->stalls, 1); |
| 124 | } |
| 125 | break; |
| 126 | } |
| 127 | cidx = increment_idx(r, cidx, n); |
| 128 | pending += n; |
| 129 | total += n; |
| 130 | |
| 131 | /* |
| 132 | * We update the cidx only if we've caught up with the pidx, the |
| 133 | * real cidx is getting too far ahead of the one visible to |
| 134 | * everyone else, or we have exceeded our budget. |
| 135 | */ |
| 136 | if (cidx != pidx && pending < 64 && total < budget) |
| 137 | continue; |
| 138 | |
| 139 | os.state = ns.state = r->state; |
| 140 | ns.cidx = cidx; |
| 141 | ns.flags = state_to_flags(ns, total >= budget); |
| 142 | r->state = ns.state; |
| 143 | |
| 144 | if (ns.flags == ABDICATED) |
| 145 | counter_u64_add(r->abdications, 1); |
| 146 | if (ns.flags != BUSY) { |
| 147 | /* Wrong loop exit if we're going to stall. */ |
| 148 | MPASS(ns.flags != STALLED); |
| 149 | if (prev == STALLED) { |
| 150 | MPASS(total > 0); |
| 151 | counter_u64_add(r->restarts, 1); |
| 152 | } |
no test coverage detected