* Dispatch a packet for netisr processing; direct dispatch is permitted by * calling context. */
| 1104 | * calling context. |
| 1105 | */ |
| 1106 | int |
| 1107 | netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m) |
| 1108 | { |
| 1109 | #ifdef NETISR_LOCKING |
| 1110 | struct rm_priotracker tracker; |
| 1111 | #endif |
| 1112 | struct netisr_workstream *nwsp; |
| 1113 | struct netisr_proto *npp; |
| 1114 | struct netisr_work *npwp; |
| 1115 | int dosignal, error; |
| 1116 | u_int cpuid, dispatch_policy; |
| 1117 | |
| 1118 | NET_EPOCH_ASSERT(); |
| 1119 | KASSERT(proto < NETISR_MAXPROT, |
| 1120 | ("%s: invalid proto %u", __func__, proto)); |
| 1121 | #ifdef NETISR_LOCKING |
| 1122 | NETISR_RLOCK(&tracker); |
| 1123 | #endif |
| 1124 | npp = &netisr_proto[proto]; |
| 1125 | KASSERT(npp->np_handler != NULL, ("%s: invalid proto %u", __func__, |
| 1126 | proto)); |
| 1127 | |
| 1128 | #ifdef VIMAGE |
| 1129 | if (V_netisr_enable[proto] == 0) { |
| 1130 | m_freem(m); |
| 1131 | return (ENOPROTOOPT); |
| 1132 | } |
| 1133 | #endif |
| 1134 | |
| 1135 | dispatch_policy = netisr_get_dispatch(npp); |
| 1136 | if (dispatch_policy == NETISR_DISPATCH_DEFERRED) |
| 1137 | return (netisr_queue_src(proto, source, m)); |
| 1138 | |
| 1139 | /* |
| 1140 | * If direct dispatch is forced, then unconditionally dispatch |
| 1141 | * without a formal CPU selection. Borrow the current CPU's stats, |
| 1142 | * even if there's no worker on it. In this case we don't update |
| 1143 | * nws_flags because all netisr processing will be source ordered due |
| 1144 | * to always being forced to directly dispatch. |
| 1145 | */ |
| 1146 | if (dispatch_policy == NETISR_DISPATCH_DIRECT) { |
| 1147 | nwsp = DPCPU_PTR(nws); |
| 1148 | npwp = &nwsp->nws_work[proto]; |
| 1149 | npwp->nw_dispatched++; |
| 1150 | npwp->nw_handled++; |
| 1151 | netisr_proto[proto].np_handler(m); |
| 1152 | error = 0; |
| 1153 | goto out_unlock; |
| 1154 | } |
| 1155 | |
| 1156 | KASSERT(dispatch_policy == NETISR_DISPATCH_HYBRID, |
| 1157 | ("%s: unknown dispatch policy (%u)", __func__, dispatch_policy)); |
| 1158 | |
| 1159 | /* |
| 1160 | * Otherwise, we execute in a hybrid mode where we will try to direct |
| 1161 | * dispatch if we're on the right CPU and the netisr worker isn't |
| 1162 | * already running. |
| 1163 | */ |
no test coverage detected