Method encode

extra/models/t5.py:46–49

(self, text:str, max_length:int)

    self.spp = SentencePieceProcessor(str(spiece_path))

  def encode(self, text:str, max_length:int) -> List[int]:
    encoded = self.spp.Encode(text)
    if len(encoded) > max_length - 1: encoded = encoded[:max_length - 1]
    return encoded + [1] + [0]*(max_length - len(encoded) - 1)

class T5LayerNorm:
  def __init__(self, hidden_size:int, eps:float=1e-6):