Function setUpInference

samples/common/sampleInference.cpp:211–437

}

bool setUpInference(InferenceEnvironment& iEnv, InferenceOptions const& inference, SystemOptions const& system)
{
    int32_t device{};
    cudaCheck(cudaGetDevice(&device));

    cudaDeviceProp properties;
    cudaCheck(cudaGetDeviceProperties(&properties, device));
    // Use managed memory on integrated devices when transfers are skipped
    // and when it is explicitly requested on the commandline.
    bool useManagedMemory{(inference.skipTransfers && properties.integrated) || inference.useManaged};
    using FillSafeBindings = FillBindingClosure<nvinfer1::safe::ICudaEngine, nvinfer1::safe::IExecutionContext>;
    if (iEnv.safe)
    {
        ASSERT(sample::hasSafeRuntime());

        auto* safeEngine = iEnv.engine.getSafe();
        SMP_RETVAL_IF_FALSE(safeEngine != nullptr, "Got invalid safeEngine!", false, sample::gLogError);

        // Release serialized blob to save memory space.
        iEnv.engine.releaseBlob();

        for (int32_t s = 0; s < inference.infStreams; ++s)
        {
            auto ec = safeEngine->createExecutionContext();
            if (ec == nullptr)
            {
                sample::gLogError << "Unable to create execution context for stream " << s << "." << std::endl;
                return false;
            }
            iEnv.safeContexts.emplace_back(ec);
            iEnv.bindings.emplace_back(new Bindings(useManagedMemory));
        }
        int32_t const nbBindings = safeEngine->getNbBindings();
        auto const* safeContext = iEnv.safeContexts.front().get();
        // batch is set to 1 because safety only support explicit batch.
        return FillSafeBindings(safeEngine, safeContext, inference.inputs, iEnv.bindings, 1, nbBindings)();
    }

    using FillStdBindings = FillBindingClosure<nvinfer1::ICudaEngine, nvinfer1::IExecutionContext>;

    auto* engine = iEnv.engine.get();
    SMP_RETVAL_IF_FALSE(engine != nullptr, "Got invalid engine!", false, sample::gLogError);

    bool const hasDLA = system.DLACore >= 0;
    if (engine->hasImplicitBatchDimension() && hasDLA && inference.batch != engine->getMaxBatchSize())
    {
        sample::gLogError << "When using DLA with an implicit batch engine, the inference batch size must be the same "
                             "as the engine's maximum batch size. Please specify the batch size by adding: '--batch="
                          << engine->getMaxBatchSize() << "' to your command." << std::endl;
        return false;
    }

    // Release serialized blob to save memory space.
    iEnv.engine.releaseBlob();

    for (int32_t s = 0; s < inference.infStreams; ++s)
    {
        auto ec = engine->createExecutionContext();