Function inferenceExecution

samples/common/sampleInference.cpp:1008–1068

template <class ContextType>
void inferenceExecution(InferenceOptions const& inference, InferenceEnvironment& iEnv, SyncStruct& sync,
    int32_t const threadIdx, int32_t const streamsPerThread, int32_t device, std::vector<InferenceTrace>& trace) noexcept
{
    try
    {
        float warmupMs = inference.warmup;
        float durationMs = -1.F;
        if (inference.duration != -1.F)
        {
            durationMs = inference.duration * 1000.F + warmupMs;
        }

        cudaCheck(cudaSetDevice(device));

        std::vector<std::unique_ptr<Iteration<ContextType>>> iStreams;

        for (int32_t s = 0; s < streamsPerThread; ++s)
        {
            int32_t const streamId{threadIdx * streamsPerThread + s};
            auto* iteration = new Iteration<ContextType>(
                streamId, inference, *iEnv.template getContext<ContextType>(streamId), *iEnv.bindings[streamId]);
            if (inference.skipTransfers)
            {
                iteration->setInputData(true);
            }
            iStreams.emplace_back(iteration);
        }

        for (auto& s : iStreams)
        {
            s->wait(sync.gpuStart);
        }

        std::vector<InferenceTrace> localTrace;
        if (!inferenceLoop(iStreams, sync.cpuStart, sync.gpuStart, inference.iterations, durationMs, warmupMs, localTrace,
                inference.skipTransfers, inference.idle))
        {
            sync.mutex.lock();
            iEnv.error = true;
            sync.mutex.unlock();
        }

        if (inference.skipTransfers)
        {
            for (auto& s : iStreams)
            {
                s->fetchOutputData(true);
            }
        }

        sync.mutex.lock();
        trace.insert(trace.end(), localTrace.begin(), localTrace.end());
        sync.mutex.unlock();
    }
    catch(...)
    {
        sync.mutex.lock();
        iEnv.error = true;