MCPcopy Create free account
hub / github.com/OpenSparseLLMs/MoM / get_batched

Method get_batched

lm-eval-harness/lm_eval/models/utils.py:383–405  ·  view source on GitHub ↗

Generates and yields batches from the reordered array. Parameters: - n (int): The size of each batch. Defaults to 1. - batch_fn (Optional[Callable[[int, Iterable], int]]): A function to determine the size of each batch. Defaults to None. Yields: Ite

(self, n: int = 1, batch_fn: Optional[Callable] = None)

Source from the content-addressed store, hash-verified

381 )
382
383 def get_batched(self, n: int = 1, batch_fn: Optional[Callable] = None) -> Iterator:
384 """
385 Generates and yields batches from the reordered array.
386
387 Parameters:
388 - n (int): The size of each batch. Defaults to 1.
389 - batch_fn (Optional[Callable[[int, Iterable], int]]): A function to determine the size of each batch. Defaults to None.
390
391 Yields:
392 Iterator: An iterator over batches of reordered elements.
393 """
394 if self.grouping:
395 for (
396 key,
397 values,
398 ) in self.arr_with_indices.items(): # type: ignore
399 values = self._reorder(values)
400 batch = self.get_chunks(values, n=n, fn=batch_fn)
401 yield from batch
402 else:
403 values = self._reorder(self.arr_with_indices) # type: ignore
404 batch = self.get_chunks(values, n=n, fn=batch_fn)
405 yield from batch
406
407 def _reorder(self, arr: Union[List, Tuple[Tuple[int, Any], ...]]) -> List:
408 """

Callers 4

generate_untilMethod · 0.95
_loglikelihood_tokensMethod · 0.95
_loglikelihood_tokensMethod · 0.95
generate_untilMethod · 0.95

Calls 2

_reorderMethod · 0.95
get_chunksMethod · 0.95

Tested by

no test coverage detected