Function main

python/core/ltp_core/datamodules/adapters/segmention.py:90–95

()

def main():
    from transformers import AutoTokenizer

    tokenizer = AutoTokenizer.from_pretrained("hfl/chinese-macbert-base")
    dataset = build_dataset(data_dir="data/seg", task_name="seg", tokenizer=tokenizer, mode="bmes")
    print(dataset)


if __name__ == "__main__":