Load data from input JSON file.
(input_file: str)
| 19 | parser.add_argument('--check_model', type=str, required=True, help='Path to the local model') |
| 20 | return parser.parse_args() |
| 21 | def load_data(input_file: str) -> List[Dict[str, Any]]: |
| 22 | """Load data from input JSON file.""" |
| 23 | # with open(input_file, 'r', encoding='utf-8') as f: |
| 24 | # data = json.load(f) |
| 25 | dataset = load_dataset('json', data_files=input_file) |
| 26 | # data = dataset['train'] |
| 27 | data = list(dataset['train']) |
| 28 | if DEBUG: |
| 29 | data = data[:5] # 取前5个样本用于调试 |
| 30 | |
| 31 | if not isinstance(data, list): |
| 32 | raise ValueError("Input data should be a list") |
| 33 | |
| 34 | if not data: |
| 35 | raise ValueError("Input data is empty") |
| 36 | |
| 37 | # Check if the first item has the required keys |
| 38 | first_item = data[0] |
| 39 | if not isinstance(first_item, dict): |
| 40 | raise ValueError("Each item in the data should be a dictionary") |
| 41 | |
| 42 | return data |
| 43 | def solve_init(): |
| 44 | args = parse_args() |
| 45 | data = load_data(args.input) |