* Walk the CQ to find completed receive descriptors. * Any completed descriptor found is refilled. */
| 433 | * Any completed descriptor found is refilled. |
| 434 | */ |
| 435 | static __rte_always_inline void |
| 436 | ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do, |
| 437 | struct ionic_rx_service *rx_svc) |
| 438 | { |
| 439 | struct ionic_cq *cq = &rxq->qcq.cq; |
| 440 | struct ionic_queue *q = &rxq->qcq.q; |
| 441 | struct ionic_rxq_desc *q_desc_base = q->base; |
| 442 | struct ionic_rxq_comp *cq_desc_base = cq->base; |
| 443 | volatile struct ionic_rxq_comp *cq_desc; |
| 444 | uint32_t work_done = 0; |
| 445 | uint64_t then, now, hz, delta; |
| 446 | |
| 447 | cq_desc = &cq_desc_base[cq->tail_idx]; |
| 448 | |
| 449 | while (color_match(cq_desc->pkt_type_color, cq->done_color)) { |
| 450 | cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); |
| 451 | if (cq->tail_idx == 0) |
| 452 | cq->done_color = !cq->done_color; |
| 453 | |
| 454 | /* Prefetch 8 x 8B bufinfo */ |
| 455 | rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8))); |
| 456 | /* Prefetch 4 x 16B comp */ |
| 457 | rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); |
| 458 | /* Prefetch 4 x 16B descriptors */ |
| 459 | if (!(rxq->flags & IONIC_QCQ_F_CMB)) |
| 460 | rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]); |
| 461 | |
| 462 | /* Clean one descriptor */ |
| 463 | ionic_rx_clean_one_sg(rxq, cq_desc, rx_svc); |
| 464 | q->tail_idx = Q_NEXT_TO_SRVC(q, 1); |
| 465 | |
| 466 | /* Fill one descriptor */ |
| 467 | (void)ionic_rx_fill_one_sg(rxq); |
| 468 | |
| 469 | q->head_idx = Q_NEXT_TO_POST(q, 1); |
| 470 | |
| 471 | if (++work_done == work_to_do) |
| 472 | break; |
| 473 | |
| 474 | cq_desc = &cq_desc_base[cq->tail_idx]; |
| 475 | } |
| 476 | |
| 477 | /* Update the queue indices and ring the doorbell */ |
| 478 | if (work_done) { |
| 479 | ionic_q_flush(q); |
| 480 | rxq->last_wdog_cycles = rte_get_timer_cycles(); |
| 481 | rxq->wdog_ms = IONIC_Q_WDOG_MS; |
| 482 | } else { |
| 483 | /* |
| 484 | * Ring the doorbell again if no recvs were posted and the |
| 485 | * recv queue is not empty after the deadline. |
| 486 | * |
| 487 | * Exponentially back off the deadline to avoid excessive |
| 488 | * doorbells when the recv queue is idle. |
| 489 | */ |
| 490 | if (q->head_idx != q->tail_idx) { |
| 491 | then = rxq->last_wdog_cycles; |
| 492 | now = rte_get_timer_cycles(); |
no test coverage detected