Method doHandshakeUnwrap

utils/src/main/java/com/cloud/utils/nio/Link.java:482–538

(final SocketChannel socketChannel, final SSLEngine sslEngine,
                                             ByteBuffer peerAppData, ByteBuffer peerNetData, final int appBufferSize)

    }

    private static HandshakeHolder doHandshakeUnwrap(final SocketChannel socketChannel, final SSLEngine sslEngine,
                                             ByteBuffer peerAppData, ByteBuffer peerNetData, final int appBufferSize) throws IOException {
        if (socketChannel == null || sslEngine == null || peerAppData == null || peerNetData == null || appBufferSize < 0) {
            return new HandshakeHolder(peerAppData, peerNetData, false);
        }
        if (socketChannel.read(peerNetData) < 0) {
            if (sslEngine.isInboundDone() && sslEngine.isOutboundDone()) {
                return new HandshakeHolder(peerAppData, peerNetData, false);
            }
            try {
                sslEngine.closeInbound();
            } catch (SSLException e) {
                LOGGER.warn("This SSL engine was forced to close inbound due to end of stream.", e);
            }
            sslEngine.closeOutbound();
            // After closeOutbound the engine will be set to WRAP state,
            // in order to try to send a close message to the client.
            return new HandshakeHolder(peerAppData, peerNetData, true);
        }
        peerNetData.flip();
        SSLEngineResult result = null;
        try {
            result = sslEngine.unwrap(peerNetData, peerAppData);
            peerNetData.compact();
        } catch (final SSLException sslException) {
            LOGGER.error(String.format("SSL error caught during unwrap data: %s, for local address=%s, remote address=%s. The client may have invalid ca-certificates.",
                    sslException.getMessage(), socketChannel.getLocalAddress(), socketChannel.getRemoteAddress()));
            sslEngine.closeOutbound();
            return new HandshakeHolder(peerAppData, peerNetData, false);
        }
        if (result == null) {
            return new HandshakeHolder(peerAppData, peerNetData, false);
        }
        switch (result.getStatus()) {
            case OK:
                break;
            case BUFFER_OVERFLOW:
                // Will occur when peerAppData's capacity is smaller than the data derived from peerNetData's unwrap.
                peerAppData = enlargeBuffer(peerAppData, appBufferSize);
                break;
            case BUFFER_UNDERFLOW:
                // Will occur either when no data was read from the peer or when the peerNetData buffer
                // was too small to hold all peer's data.
                peerNetData = handleBufferUnderflow(sslEngine, peerNetData);
                break;
            case CLOSED:
                if (sslEngine.isOutboundDone()) {
                    return new HandshakeHolder(peerAppData, peerNetData, false);
                } else {
                    sslEngine.closeOutbound();
                }
                break;
            default:
                throw new IllegalStateException("Invalid SSL status: " + result.getStatus());
        }
        return new HandshakeHolder(peerAppData, peerNetData, true);
    }