| 48 | yield os.path.join(dirpath, filename) |
| 49 | |
| 50 | def load_jsonl(self, filepaths): |
| 51 | res = [] |
| 52 | for filepath in filepaths: |
| 53 | with open(filepath, "r") as f: |
| 54 | for line in f: |
| 55 | res.append(json.loads(line)) |
| 56 | sents = [] |
| 57 | for r in res: |
| 58 | messages = r["conversation_history"] |
| 59 | reply = r["assistant_reply"] |
| 60 | sents.append(reply) |
| 61 | sents.append(messages[-2]["content"]) |
| 62 | data = pd.DataFrame(sents, columns=["query"]) |
| 63 | data["_id"] = data.index |
| 64 | return data |
| 65 | |
| 66 | def run(self, git_repo: str, repo_name: str) -> Tuple[Tuple[np.ndarray, np.ndarray], dict]: |
| 67 | """ |