Add data with optimized Redis storage and TS packet alignment
(self, chunk)
| 63 | self.chunk_available = gevent.event.Event() |
| 64 | |
| 65 | def add_chunk(self, chunk): |
| 66 | """Add data with optimized Redis storage and TS packet alignment""" |
| 67 | if not chunk or self.stopping: |
| 68 | return False |
| 69 | |
| 70 | try: |
| 71 | # Accumulate partial packets between chunks |
| 72 | if not hasattr(self, '_partial_packet'): |
| 73 | self._partial_packet = bytearray() |
| 74 | |
| 75 | # Lock the full operation to prevent race with reset_buffer_position |
| 76 | writes_done = 0 |
| 77 | with self.lock: |
| 78 | # Combine with any previous partial packet |
| 79 | combined_data = bytearray(self._partial_packet) + bytearray(chunk) |
| 80 | |
| 81 | # Calculate complete packets |
| 82 | complete_packets_size = (len(combined_data) // self.TS_PACKET_SIZE) * self.TS_PACKET_SIZE |
| 83 | |
| 84 | if complete_packets_size == 0: |
| 85 | # Not enough data for a complete packet |
| 86 | self._partial_packet = combined_data |
| 87 | return True |
| 88 | |
| 89 | # Split into complete packets and remainder |
| 90 | complete_packets = combined_data[:complete_packets_size] |
| 91 | self._partial_packet = combined_data[complete_packets_size:] |
| 92 | |
| 93 | # Add completed packets to write buffer |
| 94 | self._write_buffer.extend(complete_packets) |
| 95 | |
| 96 | # Only write to Redis when we have enough data for an optimized chunk |
| 97 | while len(self._write_buffer) >= self.target_chunk_size: |
| 98 | # Extract a full chunk |
| 99 | chunk_data = self._write_buffer[:self.target_chunk_size] |
| 100 | self._write_buffer = self._write_buffer[self.target_chunk_size:] |
| 101 | |
| 102 | # Write optimized chunk to Redis. We need the new index from |
| 103 | # incr() to build the chunk key, so issue that first; the |
| 104 | # remaining writes are pipelined into one round trip. |
| 105 | if self.redis_client: |
| 106 | chunk_index = self.redis_client.incr(self.buffer_index_key) |
| 107 | chunk_key = f"{self.buffer_prefix}{chunk_index}" |
| 108 | |
| 109 | pipe = self.redis_client.pipeline(transaction=False) |
| 110 | pipe.setex(chunk_key, self.chunk_ttl, bytes(chunk_data)) |
| 111 | |
| 112 | if self.chunk_timestamps_key: |
| 113 | now = time.time() |
| 114 | pipe.zadd(self.chunk_timestamps_key, {str(chunk_index): now}) |
| 115 | pipe.zremrangebyscore(self.chunk_timestamps_key, '-inf', now - self.chunk_ttl) |
| 116 | pipe.expire(self.chunk_timestamps_key, self.chunk_ttl) |
| 117 | |
| 118 | pipe.execute() |
| 119 | |
| 120 | # Update local tracking |
| 121 | self.index = chunk_index |
| 122 | writes_done += 1 |