Method reprocess

rft/trainer/zmq.py:376–381

处理任务并生成处理后的数据块

(self)

        socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, 10)
    
    def reprocess(self):
        """处理任务并生成处理后的数据块"""
        while True:
            collected = [self.valid_tasks.get().data for _ in range(self.chunk_size)]
            chunk_data = _process_inputs(collected, self.processor, self.max_prompt_length)
            self.ready_queue.put(chunk_data)
    
    def provider(self):
        """为工作进程提供数据"""