Function prepare_meta

tools/tokenizer.py:41–71

Prepare metadata for the given bin file. Args: bin_output_path (str): Output bin file path.

(bin_output_path: str)

def prepare_meta(bin_output_path: str):
    """
    Prepare metadata for the given bin file.

    Args:
        bin_output_path (str): Output bin file path.
    """
    meta = []
    cur = 0
    with open(bin_output_path, "rb") as f:
        while True:
            # read lines
            line = f.readline()
            # if line is empty, then break
            if line == b"":
                break
            # obtain the token amount of each line
            length = len(json.loads(line)["tokens"])
            # meta is a list of tuple(cur, length)
            # cur: the start index of each line
            # length: the token amount of each line
            meta.append((cur, length))
            # update the cur to generate the meta information of next line
            cur += len(line)

    # define path of the generated meta file
    meta_fp = bin_output_path + ".meta"
    # save the generated meta information
    with open(meta_fp, "wb") as f:
        meta = np.array(meta, dtype=np.int32)
        np.save(f, meta)


def text2bin(text_input_path: str, bin_output_path: str):