Writes JSON arrays to a file in a streaming fashion.
| 46 | |
| 47 | |
| 48 | class StreamingJSONWriter: |
| 49 | """Writes JSON arrays to a file in a streaming fashion.""" |
| 50 | def __init__(self, file: TextIO): |
| 51 | self.file = file |
| 52 | self.is_first = True |
| 53 | self.file.write('[\n') |
| 54 | |
| 55 | def write_item(self, item: Dict): |
| 56 | """Write a single item to the JSON array.""" |
| 57 | if not self.is_first: |
| 58 | self.file.write(',\n') |
| 59 | json.dump(item, self.file, indent=2) |
| 60 | self.is_first = False |
| 61 | # Flush after each write to ensure immediate disk writing |
| 62 | self.file.flush() |
| 63 | |
| 64 | def close(self): |
| 65 | """Close the JSON array and the file.""" |
| 66 | self.file.write('\n]') |
| 67 | self.file.flush() |
| 68 | |
| 69 | |
| 70 | def get_base_model_state_dict_from_peft(peft_state_dict, lora_alpha, lora_r): |