* I/O completion callback. */
| 286 | * I/O completion callback. |
| 287 | */ |
| 288 | static void |
| 289 | sendfile_iodone(void *arg, vm_page_t *pa, int count, int error) |
| 290 | { |
| 291 | struct sf_io *sfio = arg; |
| 292 | struct socket *so; |
| 293 | int i; |
| 294 | |
| 295 | if (error != 0) |
| 296 | sfio->error = error; |
| 297 | |
| 298 | /* |
| 299 | * Restore the valid page pointers. They are already |
| 300 | * unbusied, but still wired. |
| 301 | * |
| 302 | * XXXKIB since pages are only wired, and we do not |
| 303 | * own the object lock, other users might have |
| 304 | * invalidated them in meantime. Similarly, after we |
| 305 | * unbusied the swapped-in pages, they can become |
| 306 | * invalid under us. |
| 307 | */ |
| 308 | MPASS(count == 0 || pa[0] != bogus_page); |
| 309 | for (i = 0; i < count; i++) { |
| 310 | if (pa[i] == bogus_page) { |
| 311 | sfio->pa[(pa[0]->pindex - sfio->pindex0) + i] = |
| 312 | pa[i] = vm_page_relookup(sfio->obj, |
| 313 | pa[0]->pindex + i); |
| 314 | KASSERT(pa[i] != NULL, |
| 315 | ("%s: page %p[%d] disappeared", |
| 316 | __func__, pa, i)); |
| 317 | } else { |
| 318 | vm_page_xunbusy_unchecked(pa[i]); |
| 319 | } |
| 320 | } |
| 321 | |
| 322 | if (!refcount_release(&sfio->nios)) |
| 323 | return; |
| 324 | |
| 325 | #ifdef INVARIANTS |
| 326 | for (i = 1; i < sfio->npages; i++) { |
| 327 | if (sfio->pa[i] == NULL) |
| 328 | break; |
| 329 | KASSERT(vm_page_wired(sfio->pa[i]), |
| 330 | ("sfio %p page %d %p not wired", sfio, i, sfio->pa[i])); |
| 331 | if (i == 0) |
| 332 | continue; |
| 333 | KASSERT(sfio->pa[0]->object == sfio->pa[i]->object, |
| 334 | ("sfio %p page %d %p wrong owner %p %p", sfio, i, |
| 335 | sfio->pa[i], sfio->pa[0]->object, sfio->pa[i]->object)); |
| 336 | KASSERT(sfio->pa[0]->pindex + i == sfio->pa[i]->pindex, |
| 337 | ("sfio %p page %d %p wrong index %jx %jx", sfio, i, |
| 338 | sfio->pa[i], (uintmax_t)sfio->pa[0]->pindex, |
| 339 | (uintmax_t)sfio->pa[i]->pindex)); |
| 340 | } |
| 341 | #endif |
| 342 | |
| 343 | vm_object_pip_wakeup(sfio->obj); |
| 344 | |
| 345 | if (sfio->m == NULL) { |
no test coverage detected