Function timeDeserialize

samples/common/sampleInference.cpp:1181–1270

Returns true if deserialization is slower than expected or fails.

//! Returns true if deserialization is slower than expected or fails.
bool timeDeserialize(InferenceEnvironment& iEnv, SystemOptions const& sys)
{
    constexpr int32_t kNB_ITERS{20};
    std::unique_ptr<IRuntime> rt{createRuntime()};
    std::unique_ptr<ICudaEngine> engine;

    std::unique_ptr<safe::IRuntime> safeRT{sample::createSafeInferRuntime(sample::gLogger.getTRTLogger())};
    std::unique_ptr<safe::ICudaEngine> safeEngine;

    if (iEnv.safe)
    {
        ASSERT(sample::hasSafeRuntime() && safeRT != nullptr);
        safeRT->setErrorRecorder(&gRecorder);
    }

    auto timeDeserializeFn = [&]() -> float {
        bool deserializeOK{false};
        engine.reset(nullptr);
        safeEngine.reset(nullptr);
        auto startClock = std::chrono::high_resolution_clock::now();
        if (iEnv.safe)
        {
            safeEngine.reset(safeRT->deserializeCudaEngine(iEnv.engine.getBlob().data(), iEnv.engine.getBlob().size()));
            deserializeOK = (safeEngine != nullptr);
        }
        else
        {
            for (auto const& pluginPath : sys.dynamicPlugins)
            {
                rt->getPluginRegistry().loadLibrary(pluginPath.c_str());
            }
            engine.reset(
                rt->deserializeCudaEngine(iEnv.engine.getBlob().data(), iEnv.engine.getBlob().size(), nullptr));
            deserializeOK = (engine != nullptr);
        }
        auto endClock = std::chrono::high_resolution_clock::now();
        // return NAN if deserialization failed.
        return deserializeOK ? std::chrono::duration<float, std::milli>(endClock - startClock).count() : NAN;
    };

    // Warmup the caches to make sure that cache thrashing isn't throwing off the results
    {
        sample::gLogInfo << "Begin deserialization warmup..." << std::endl;
        for (int32_t i = 0, e = 2; i < e; ++i)
        {
            timeDeserializeFn();
        }
    }
    sample::gLogInfo << "Begin deserialization engine timing..." << std::endl;
    float const first = timeDeserializeFn();

    // Check if first deserialization succeeded.
    if (std::isnan(first))
    {
        sample::gLogError << "Engine deserialization failed." << std::endl;
        return true;
    }