| 482 | |
| 483 | |
| 484 | class BigFile(object): |
| 485 | def __init__(self, site, inner_path, prebuffer=0): |
| 486 | self.site = site |
| 487 | self.inner_path = inner_path |
| 488 | file_path = site.storage.getPath(inner_path) |
| 489 | file_info = self.site.content_manager.getFileInfo(inner_path) |
| 490 | self.piece_size = file_info["piece_size"] |
| 491 | self.sha512 = file_info["sha512"] |
| 492 | self.size = file_info["size"] |
| 493 | self.prebuffer = prebuffer |
| 494 | self.read_bytes = 0 |
| 495 | |
| 496 | self.piecefield = self.site.storage.piecefields[self.sha512] |
| 497 | self.f = open(file_path, "rb+") |
| 498 | self.read_lock = gevent.lock.Semaphore() |
| 499 | |
| 500 | def read(self, buff=64 * 1024): |
| 501 | with self.read_lock: |
| 502 | pos = self.f.tell() |
| 503 | read_until = min(self.size, pos + buff) |
| 504 | requests = [] |
| 505 | # Request all required blocks |
| 506 | while 1: |
| 507 | piece_i = int(pos / self.piece_size) |
| 508 | if piece_i * self.piece_size >= read_until: |
| 509 | break |
| 510 | pos_from = piece_i * self.piece_size |
| 511 | pos_to = pos_from + self.piece_size |
| 512 | if not self.piecefield[piece_i]: |
| 513 | requests.append(self.site.needFile("%s|%s-%s" % (self.inner_path, pos_from, pos_to), blocking=False, update=True, priority=10)) |
| 514 | pos += self.piece_size |
| 515 | |
| 516 | if not all(requests): |
| 517 | return None |
| 518 | |
| 519 | # Request prebuffer |
| 520 | if self.prebuffer: |
| 521 | prebuffer_until = min(self.size, read_until + self.prebuffer) |
| 522 | priority = 3 |
| 523 | while 1: |
| 524 | piece_i = int(pos / self.piece_size) |
| 525 | if piece_i * self.piece_size >= prebuffer_until: |
| 526 | break |
| 527 | pos_from = piece_i * self.piece_size |
| 528 | pos_to = pos_from + self.piece_size |
| 529 | if not self.piecefield[piece_i]: |
| 530 | self.site.needFile("%s|%s-%s" % (self.inner_path, pos_from, pos_to), blocking=False, update=True, priority=max(0, priority)) |
| 531 | priority -= 1 |
| 532 | pos += self.piece_size |
| 533 | |
| 534 | gevent.joinall(requests) |
| 535 | self.read_bytes += buff |
| 536 | |
| 537 | # Increase buffer for long reads |
| 538 | if self.read_bytes > 7 * 1024 * 1024 and self.prebuffer < 5 * 1024 * 1024: |
| 539 | self.site.log.debug("%s: Increasing bigfile buffer size to 5MB..." % self.inner_path) |
| 540 | self.prebuffer = 5 * 1024 * 1024 |
| 541 | |