| 21 | |
| 22 | |
| 23 | def get_arg_parser(): |
| 24 | parser = argparse.ArgumentParser() |
| 25 | parser.add_argument('--input_file', type=str, required=True, help='path to gz wikidata json dump') |
| 26 | parser.add_argument('--out_dir', type=str, required=True, help='path to output directory') |
| 27 | parser.add_argument('--language_id', type=str, default='en', help='language identifier') |
| 28 | parser.add_argument('--processes', type=int, default=90, help="number of concurrent processes to spin off. ") |
| 29 | parser.add_argument('--batch_size', type=int, default=10000) |
| 30 | parser.add_argument('--num_lines_read', type=int, default=-1, |
| 31 | help='Terminate after num_lines_read lines are read. Useful for debugging.') |
| 32 | parser.add_argument('--num_lines_in_dump', type=int, default=-1, help='Number of lines in dump. If -1, we will count the number of lines.') |
| 33 | return parser |
| 34 | |
| 35 | |
| 36 | def main(): |