Function measure_python_inference_code

demo/HuggingFace/NNDF/general_utils.py:151–206

Measures the time it takes to run Pythonic inference code. Statement given should be the actual model inference like forward() in torch. Args: stmt (Union[Callable, str]): Callable or string for generating numbers. timing_profile (TimingProfile): The timing profile sett

(
    stmt: Union[Callable, str], timing_profile: TimingProfile
)

def measure_python_inference_code(
    stmt: Union[Callable, str], timing_profile: TimingProfile
) -> None:
    """
    Measures the time it takes to run Pythonic inference code.
    Statement given should be the actual model inference like forward() in torch.

    Args:
        stmt (Union[Callable, str]): Callable or string for generating numbers.
        timing_profile (TimingProfile): The timing profile settings with the following fields.
            warmup (int): Number of iterations to run as warm-up before actual measurement cycles.
            number (int): Number of times to call function per iteration.
            iterations (int): Number of measurement cycles.
            duration (float): Minimal duration for measurement cycles.
            percentile (int or list of ints): key percentile number(s) for measurement.
    """

    def simple_percentile(data, p):
        """
        Temporary replacement for numpy.percentile() because TRT CI/CD pipeline requires additional packages to be added at boot up in this general_utils.py file.
        """
        assert p >= 0 and p <= 100, "Percentile must be between 1 and 99"

        rank = len(data) * p / 100
        if rank.is_integer():
            return sorted(data)[int(rank)]
        else:
            return sorted(data)[int(math.ceil(rank)) - 1]

    warmup = timing_profile.warmup
    number = timing_profile.number
    iterations = timing_profile.iterations
    duration = timing_profile.duration
    percentile = timing_profile.percentile

    G_LOGGER.debug(
        "Measuring inference call with warmup: {} and number: {} and iterations {} and duration {} secs".format(
            warmup, number, iterations, duration
        )
    )
    # Warmup
    warmup_mintime = timeit.repeat(stmt, number=number, repeat=warmup)
    G_LOGGER.debug("Warmup times: {}".format(warmup_mintime))

    # Actual measurement cycles
    results = []
    start_time = datetime.now()
    iter_idx = 0
    while iter_idx < iterations or (datetime.now() - start_time).total_seconds() < duration:
        iter_idx += 1
        results.append(timeit.timeit(stmt, number=number))

    if isinstance(percentile, int):
        return simple_percentile(results, percentile) / number
    else:
        return [simple_percentile(results, p) / number for p in percentile]

class NNFolderWorkspace: