| 433 | } |
| 434 | |
| 435 | int32_t QKVToContextPluginDynamic::enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc, |
| 436 | void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept |
| 437 | { |
| 438 | PLUGIN_ASSERT(mS == inputDesc->dims.d[SDIM]); |
| 439 | PLUGIN_ASSERT(mB == inputDesc->dims.d[BDIM]); |
| 440 | |
| 441 | try |
| 442 | { |
| 443 | void const* const maskPtr = mHasImask ? inputs[1] : nullptr; |
| 444 | if (fusedDispatcher.get() && fusedDispatcher->isValid(inputDesc->dims.d[SDIM])) |
| 445 | { |
| 446 | fusedDispatcher->run(inputDesc[0], outputDesc[0], inputs[0], maskPtr, outputs[0], workspace, stream); |
| 447 | } |
| 448 | else |
| 449 | { |
| 450 | PLUGIN_VALIDATE(unfusedDispatcher.get(), "The Unfused MHARunner is uninitialized, no MHARunner available!"); |
| 451 | unfusedDispatcher->run(inputDesc[0], outputDesc[0], inputs[0], maskPtr, outputs[0], workspace, stream); |
| 452 | } |
| 453 | } |
| 454 | catch (std::exception const& e) |
| 455 | { |
| 456 | caughtError(e); |
| 457 | return -1; |
| 458 | } |
| 459 | return 0; |
| 460 | } |
| 461 | |
| 462 | QKVToContextPluginDynamicCreator::QKVToContextPluginDynamicCreator() |
| 463 | { |
nothing calls this directly
no test coverage detected