Method encode_segment

accessory/model/tokenizer.py:64–69

(self, s:str)

        return t

    def encode_segment(self, s:str):
        s = s.lstrip(' ')
        if self.need_space_before_segment:
            return self.encode(" " + s, bos=False, eos=False)
        else:
            return self.encode(s, bos=False, eos=False)

    def encode_wo_prefix_space(self, s:str):
        if self.need_space_before_segment: