* @internal Get several objects from the mempool; used internally. * @param mp * A pointer to the mempool structure. * @param obj_table * A pointer to a table of void * pointers (objects). * @param n * The number of objects to get, must be strictly positive. * @param cache * A pointer to a mempool cache structure. May be NULL if not needed. * @return * - 0: Success. * - <0:
| 1494 | * - <0: Error; code of driver dequeue function. |
| 1495 | */ |
| 1496 | static __rte_always_inline int |
| 1497 | rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table, |
| 1498 | unsigned int n, struct rte_mempool_cache *cache) |
| 1499 | { |
| 1500 | int ret; |
| 1501 | unsigned int remaining; |
| 1502 | uint32_t index, len; |
| 1503 | void **cache_objs; |
| 1504 | |
| 1505 | /* No cache provided */ |
| 1506 | if (unlikely(cache == NULL)) { |
| 1507 | remaining = n; |
| 1508 | goto driver_dequeue; |
| 1509 | } |
| 1510 | |
| 1511 | /* The cache is a stack, so copy will be in reverse order. */ |
| 1512 | cache_objs = &cache->objs[cache->len]; |
| 1513 | |
| 1514 | if (__extension__(__builtin_constant_p(n)) && n <= cache->len) { |
| 1515 | /* |
| 1516 | * The request size is known at build time, and |
| 1517 | * the entire request can be satisfied from the cache, |
| 1518 | * so let the compiler unroll the fixed length copy loop. |
| 1519 | */ |
| 1520 | cache->len -= n; |
| 1521 | for (index = 0; index < n; index++) |
| 1522 | *obj_table++ = *--cache_objs; |
| 1523 | |
| 1524 | RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1); |
| 1525 | RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n); |
| 1526 | |
| 1527 | return 0; |
| 1528 | } |
| 1529 | |
| 1530 | /* |
| 1531 | * Use the cache as much as we have to return hot objects first. |
| 1532 | * If the request size 'n' is known at build time, the above comparison |
| 1533 | * ensures that n > cache->len here, so omit RTE_MIN(). |
| 1534 | */ |
| 1535 | len = __extension__(__builtin_constant_p(n)) ? cache->len : |
| 1536 | RTE_MIN(n, cache->len); |
| 1537 | cache->len -= len; |
| 1538 | remaining = n - len; |
| 1539 | for (index = 0; index < len; index++) |
| 1540 | *obj_table++ = *--cache_objs; |
| 1541 | |
| 1542 | /* |
| 1543 | * If the request size 'n' is known at build time, the case |
| 1544 | * where the entire request can be satisfied from the cache |
| 1545 | * has already been handled above, so omit handling it here. |
| 1546 | */ |
| 1547 | if (!__extension__(__builtin_constant_p(n)) && remaining == 0) { |
| 1548 | /* The entire request is satisfied from the cache. */ |
| 1549 | |
| 1550 | RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1); |
| 1551 | RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n); |
| 1552 | |
| 1553 | return 0; |
no test coverage detected