Method split

python/graphscope/client/utils.py:63–91

Traverse `large_attr` of op and split into a list of chunks. Note that this method will modify `large_attr` attribute of op in dag_def. Returns: Sequence[Sequence[bytes]]: splited chunks.

(self, dag_def)

        return chunk_meta

    def split(self, dag_def):
        """Traverse `large_attr` of op and split into a list of chunks.

        Note that this method will modify `large_attr` attribute of op in dag_def.

        Returns:
            Sequence[Sequence[bytes]]: splited chunks.
        """
        chunks_list = []
        for op in dag_def.op:
            large_attr = attr_value_pb2.LargeAttrValue()
            for chunk in op.large_attr.chunk_list.items:
                # construct chunk meta
                large_attr.chunk_meta_list.items.extend(
                    [self._generate_chunk_meta(chunk)]
                )
                # split buffer
                chunks_list.append(
                    (
                        [
                            chunk.buffer[i : i + self.CHUNK_SIZE]
                            for i in range(0, len(chunk.buffer), self.CHUNK_SIZE)
                        ],
                        op.key,
                    )
                )
            # replace chunk with chunk_meta
            op.large_attr.CopyFrom(large_attr)
        return chunks_list

    def generate_runstep_requests(self, session_id, dag_def):
        runstep_requests = []