Initialize LeetCode Dataset. Args: path: Base path to the dataset directory name: Dataset mode (e.g., "all", or "easy", "hard" if folders are split) split: Dataset split ("test", "validation", "train") lang: The target program
(self, path, name, split, lang="python3")
| 9 | @DATASET.register_module(force=True) |
| 10 | class LeetCodeDataset: |
| 11 | def __init__(self, path, name, split, lang="python3"): |
| 12 | """ |
| 13 | Initialize LeetCode Dataset. |
| 14 | |
| 15 | Args: |
| 16 | path: Base path to the dataset directory |
| 17 | name: Dataset mode (e.g., "all", or "easy", "hard" if folders are split) |
| 18 | split: Dataset split ("test", "validation", "train") |
| 19 | lang: The target programming language for the code template (default: "python3") |
| 20 | Supported: "cpp", "java", "python3", "golang", "rust", etc. |
| 21 | """ |
| 22 | self.path = path |
| 23 | self.name = name |
| 24 | self.split = split |
| 25 | self.lang = lang |
| 26 | |
| 27 | # 1. Path normalization |
| 28 | path = assemble_project_path(path) |
| 29 | |
| 30 | # 2. Load metadata file |
| 31 | # Expected path structure: /data/leetcode/test/metadata.jsonl |
| 32 | metadata_file = os.path.join(path, split, "metadata.jsonl") |
| 33 | |
| 34 | if not os.path.exists(metadata_file): |
| 35 | raise FileNotFoundError(f"Metadata file not found: {metadata_file}") |
| 36 | |
| 37 | data_rows = [] |
| 38 | |
| 39 | with open(metadata_file, "r", encoding="utf-8") as f: |
| 40 | for line in f: |
| 41 | try: |
| 42 | row = json.loads(line) |
| 43 | except json.JSONDecodeError: |
| 44 | continue |
| 45 | |
| 46 | # --- LeetCode specific loading logic --- |
| 47 | |
| 48 | # 1. Read problem description (Markdown content) |
| 49 | # row['file'] example: "./question/1.two_sum.md" |
| 50 | # Parse to absolute path, usually relative to metadata.jsonl directory |
| 51 | rel_file_path = row.get("file", "") |
| 52 | question_content = "" |
| 53 | |
| 54 | metadata_dir = os.path.dirname(os.path.abspath(metadata_file)) |
| 55 | |
| 56 | if rel_file_path: |
| 57 | raw_path = os.path.join(metadata_dir, rel_file_path) |
| 58 | abs_file_path = os.path.normpath(raw_path) |
| 59 | if os.path.exists(abs_file_path): |
| 60 | with open(abs_file_path, "r", encoding="utf-8") as qf: |
| 61 | question_content = qf.read() |
| 62 | else: |
| 63 | print(f"[Warning] 路径不存在: {abs_file_path}") |
| 64 | continue |
| 65 | |
| 66 | code_template = row.get("code_template", {}) |
| 67 | |
| 68 |