MCPcopy Create free account
hub / github.com/Pints-AI/1.5-Pints / _write_chunk

Method _write_chunk

lit_gpt/packed_dataset.py:136–154  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

134 self._filenames = []
135
136 def _write_chunk(self):
137 filename = f'{self._prefix}_{self._counter:010d}.bin'
138 filename = os.path.join(self._outdir, filename)
139
140 with open(filename, 'wb') as f:
141 # File headers to identify the file type and some meta
142 f.write(HDR_MAGIC)
143 f.write(struct.pack('<Q', self._version))
144 f.write(struct.pack('<B', code(self._dtype)))
145 f.write(struct.pack('<Q', self._chunk_size))
146 # Now write all of it in.
147 f.write(self._arr.tobytes(order='C'))
148
149 self._filenames.append(filename)
150 self._counter += 1
151
152 # Have written the file, so we reset the array to start again
153 self._arr.fill(self._pad_token)
154 self._idx = 0
155
156 @property
157 def dtype(self):

Callers 2

add_arrayMethod · 0.95
write_remainderMethod · 0.95

Calls 1

codeFunction · 0.85

Tested by

no test coverage detected