(cw22_api: ClueWeb22Api, output_dir: str, input_file: str)
| 10 | |
| 11 | |
| 12 | def fetch(cw22_api: ClueWeb22Api, output_dir: str, input_file: str) -> None: |
| 13 | basename_without_ext = os.path.splitext(os.path.basename(input_file))[0] |
| 14 | output_file = os.path.join(output_dir, f"{basename_without_ext}.jsonl") |
| 15 | |
| 16 | with open(input_file, "r") as f: |
| 17 | doc_ids = [line.strip() for line in f] |
| 18 | |
| 19 | with open(output_file, "w") as f: |
| 20 | for doc_id in doc_ids: |
| 21 | doc_id = doc_id.strip() |
| 22 | doc = cw22_api.get_clean_text(doc_id) |
| 23 | if doc is not None: |
| 24 | doc_stripped = doc.strip() |
| 25 | if doc_stripped != "": |
| 26 | f.write(doc_stripped + "\n") |
| 27 | |
| 28 | |
| 29 | def main(): |
nothing calls this directly
no test coverage detected