| 8 | |
| 9 | |
| 10 | class FuncCallDataset(ToolDataset): |
| 11 | def __init__(self, dataset_name, tool_task, filepath): |
| 12 | self.dataset_name = dataset_name |
| 13 | self.tool_task = tool_task |
| 14 | self.filepath = filepath |
| 15 | self.datas = self.load_data() |
| 16 | |
| 17 | def load_data(self, ) -> list: |
| 18 | if self.filepath: |
| 19 | return self.load_data_from_local(self.filepath) |
| 20 | elif self.dataset_name and self.tool_task: |
| 21 | return self.load_data_from_hf(self.tool_task) |
| 22 | return [] |
| 23 | |
| 24 | def load_data_from_local(self, filepath): |
| 25 | def _load_from_file(filename): |
| 26 | if "jsonl" in filename: |
| 27 | return read_jsonl_file(filename) |
| 28 | elif "json" in filename: |
| 29 | return read_json_file(filename) |
| 30 | |
| 31 | datas = [] |
| 32 | if os.path.isdir(filepath): |
| 33 | for filename in os.listdir(filepath): |
| 34 | datas.extend(_load_from_file(os.path.join(filepath, filename))) |
| 35 | else: |
| 36 | datas = _load_from_file(filepath) |
| 37 | |
| 38 | return datas |
| 39 | |
| 40 | def load_data_from_hf(self, tool_task): |
| 41 | pass |