| 1981 | } |
| 1982 | |
| 1983 | static __noinline void |
| 1984 | ktls_encrypt(struct mbuf *top) |
| 1985 | { |
| 1986 | struct ktls_session *tls; |
| 1987 | struct socket *so; |
| 1988 | struct mbuf *m; |
| 1989 | vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; |
| 1990 | struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; |
| 1991 | struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; |
| 1992 | vm_page_t pg; |
| 1993 | int error, i, len, npages, off, total_pages; |
| 1994 | bool is_anon; |
| 1995 | |
| 1996 | so = top->m_epg_so; |
| 1997 | tls = top->m_epg_tls; |
| 1998 | KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top)); |
| 1999 | KASSERT(so != NULL, ("so = NULL, top = %p\n", top)); |
| 2000 | #ifdef INVARIANTS |
| 2001 | top->m_epg_so = NULL; |
| 2002 | #endif |
| 2003 | total_pages = top->m_epg_enc_cnt; |
| 2004 | npages = 0; |
| 2005 | |
| 2006 | /* |
| 2007 | * Encrypt the TLS records in the chain of mbufs starting with |
| 2008 | * 'top'. 'total_pages' gives us a total count of pages and is |
| 2009 | * used to know when we have finished encrypting the TLS |
| 2010 | * records originally queued with 'top'. |
| 2011 | * |
| 2012 | * NB: These mbufs are queued in the socket buffer and |
| 2013 | * 'm_next' is traversing the mbufs in the socket buffer. The |
| 2014 | * socket buffer lock is not held while traversing this chain. |
| 2015 | * Since the mbufs are all marked M_NOTREADY their 'm_next' |
| 2016 | * pointers should be stable. However, the 'm_next' of the |
| 2017 | * last mbuf encrypted is not necessarily NULL. It can point |
| 2018 | * to other mbufs appended while 'top' was on the TLS work |
| 2019 | * queue. |
| 2020 | * |
| 2021 | * Each mbuf holds an entire TLS record. |
| 2022 | */ |
| 2023 | error = 0; |
| 2024 | for (m = top; npages != total_pages; m = m->m_next) { |
| 2025 | KASSERT(m->m_epg_tls == tls, |
| 2026 | ("different TLS sessions in a single mbuf chain: %p vs %p", |
| 2027 | tls, m->m_epg_tls)); |
| 2028 | KASSERT((m->m_flags & (M_EXTPG | M_NOTREADY)) == |
| 2029 | (M_EXTPG | M_NOTREADY), |
| 2030 | ("%p not unready & nomap mbuf (top = %p)\n", m, top)); |
| 2031 | KASSERT(npages + m->m_epg_npgs <= total_pages, |
| 2032 | ("page count mismatch: top %p, total_pages %d, m %p", top, |
| 2033 | total_pages, m)); |
| 2034 | |
| 2035 | /* |
| 2036 | * Generate source and destination ivoecs to pass to |
| 2037 | * the SW encryption backend. For writable mbufs, the |
| 2038 | * destination iovec is a copy of the source and |
| 2039 | * encryption is done in place. For file-backed mbufs |
| 2040 | * (from sendfile), anonymous wired pages are |
no test coverage detected