Find all test.txt files in dataset directories.
(self)
| 61 | raise FileNotFoundError(f"Model file not found: {self.model_path}") |
| 62 | |
| 63 | def find_datasets(self): |
| 64 | """Find all test.txt files in dataset directories.""" |
| 65 | datasets = [] |
| 66 | |
| 67 | if not self.data_dir.exists(): |
| 68 | print(f"❌ Data directory not found: {self.data_dir}") |
| 69 | return datasets |
| 70 | |
| 71 | print(f"\n🔍 Searching for datasets in {self.data_dir}...") |
| 72 | |
| 73 | # Look for test.txt files in subdirectories |
| 74 | for dataset_dir in sorted(self.data_dir.iterdir()): |
| 75 | if dataset_dir.is_dir(): |
| 76 | test_file = dataset_dir / "test.txt" |
| 77 | if test_file.exists(): |
| 78 | size_mb = test_file.stat().st_size / (1024 * 1024) |
| 79 | datasets.append({ |
| 80 | 'name': dataset_dir.name, |
| 81 | 'path': test_file, |
| 82 | 'size': test_file.stat().st_size, |
| 83 | 'size_mb': size_mb |
| 84 | }) |
| 85 | print(f" ✅ {dataset_dir.name:<20} ({size_mb:.2f} MB)") |
| 86 | else: |
| 87 | print(f" ⚠️ {dataset_dir.name:<20} (no test.txt found)") |
| 88 | |
| 89 | return datasets |
| 90 | |
| 91 | def create_quick_dataset(self, dataset_path, num_chars=4096): |
| 92 | """Create a temporary dataset with only the first N characters for quick testing.""" |