| 90 | } |
| 91 | |
| 92 | async fn run<P: Processor>(input: FileId, mut processor: P) -> Result<()> { |
| 93 | let streamer = SeaStreamer::connect(input.to_streamer_uri()?, Default::default()).await?; |
| 94 | let info_stream = StreamKey::new(INFO_STREAM)?; |
| 95 | let file_stream = StreamKey::new(FILE_STREAM)?; |
| 96 | let event_stream = StreamKey::new(EVENT_STREAM)?; |
| 97 | let breakpoint_stream = StreamKey::new(BREAKPOINT_STREAM)?; |
| 98 | let alloc_stream = StreamKey::new(ALLOCATION_STREAM)?; |
| 99 | |
| 100 | let mut options = SeaConsumerOptions::new(ConsumerMode::RealTime); |
| 101 | options.set_auto_stream_reset(SeaStreamReset::Earliest); |
| 102 | |
| 103 | let stream_keys = [ |
| 104 | info_stream, |
| 105 | file_stream, |
| 106 | event_stream, |
| 107 | breakpoint_stream, |
| 108 | alloc_stream, |
| 109 | ]; |
| 110 | let consumer = streamer.create_consumer(&stream_keys, options).await?; |
| 111 | |
| 112 | // From sea-streamer/examples/buffered |
| 113 | let (sender, receiver) = bounded(1024); |
| 114 | |
| 115 | let handle = spawn_task::<_, Result<(), StreamErr<BackendErr>>>(async move { |
| 116 | loop { |
| 117 | let message = consumer.next().await?; |
| 118 | let err = |_| StreamErr::Backend(BackendErr::File(FileErr::TaskDead("channel closed"))); |
| 119 | // If the queue is full, we'll wait |
| 120 | sender |
| 121 | .send_async(unpack(message)) |
| 122 | .await |
| 123 | .context("Fail to buffer message") |
| 124 | .map_err(err)?; |
| 125 | } |
| 126 | }); |
| 127 | |
| 128 | while !receiver.is_empty() || !receiver.is_disconnected() { |
| 129 | // Take all messages currently buffered in the queue, but do not wait |
| 130 | let messages = receiver.drain(); |
| 131 | if messages.len() > 1 { |
| 132 | processor.batch(messages).await?; |
| 133 | } else if messages.len() == 1 { |
| 134 | // if there is only 1 item; wait for more |
| 135 | sleep(std::time::Duration::from_millis(1)).await; |
| 136 | processor.batch(messages.chain(receiver.drain())).await?; |
| 137 | } else { |
| 138 | // no messages; sleep |
| 139 | sleep(std::time::Duration::from_millis(10)).await; |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | processor.end().await?; |
| 144 | if let Err(e) = handle.await? { |
| 145 | ok_if_stream_ended(e)?; |
| 146 | } |
| 147 | |
| 148 | processor.finish() |
| 149 | } |