Method load_data

ppo/ppo_datahelper.py:300–312

(self, file_path: str)

    def load_data(self, file_path: str):
        with open(file_path, 'r') as f:
            data: List[List[str]] = json.load(f)

        output: List[Tuple[List[str], str]] = []

        for turn in data:
            if not isinstance(turn, list) or len(turn) < 2 or not all(turn):
                continue
            output.append(turn)

        del data
        return output

    def format(self, sample: Tuple[List[str], str]) -> Dict[str, Any]:
        # original text concat special prompt: human prompt and assistant prompt