Params ------ fin : binary file with `read(buf_size : int)` method fout : binary file with `write` (and optionally `flush`) methods. callback : function(float), e.g.: `tqdm.update` callback_len : If (default: True) do `callback(len(buffer))`. Otherwise, do `callbac
(fin, fout, delim=b'\\n', buf_size=256,
callback=lambda float: None, callback_len=True)
| 53 | |
| 54 | |
| 55 | def posix_pipe(fin, fout, delim=b'\\n', buf_size=256, |
| 56 | callback=lambda float: None, callback_len=True): |
| 57 | """ |
| 58 | Params |
| 59 | ------ |
| 60 | fin : binary file with `read(buf_size : int)` method |
| 61 | fout : binary file with `write` (and optionally `flush`) methods. |
| 62 | callback : function(float), e.g.: `tqdm.update` |
| 63 | callback_len : If (default: True) do `callback(len(buffer))`. |
| 64 | Otherwise, do `callback(data) for data in buffer.split(delim)`. |
| 65 | """ |
| 66 | fp_write = fout.write |
| 67 | |
| 68 | if not delim: |
| 69 | while True: |
| 70 | tmp = fin.read(buf_size) |
| 71 | |
| 72 | # flush at EOF |
| 73 | if not tmp: |
| 74 | getattr(fout, 'flush', lambda: None)() |
| 75 | return |
| 76 | |
| 77 | fp_write(tmp) |
| 78 | callback(len(tmp)) |
| 79 | # return |
| 80 | |
| 81 | buf = b'' |
| 82 | len_delim = len(delim) |
| 83 | # n = 0 |
| 84 | while True: |
| 85 | tmp = fin.read(buf_size) |
| 86 | |
| 87 | # flush at EOF |
| 88 | if not tmp: |
| 89 | if buf: |
| 90 | fp_write(buf) |
| 91 | if callback_len: |
| 92 | # n += 1 + buf.count(delim) |
| 93 | callback(1 + buf.count(delim)) |
| 94 | else: |
| 95 | for i in buf.split(delim): |
| 96 | callback(i) |
| 97 | getattr(fout, 'flush', lambda: None)() |
| 98 | return # n |
| 99 | |
| 100 | while True: |
| 101 | i = tmp.find(delim) |
| 102 | if i < 0: |
| 103 | buf += tmp |
| 104 | break |
| 105 | fp_write(buf + tmp[:i + len(delim)]) |
| 106 | # n += 1 |
| 107 | callback(1 if callback_len else (buf + tmp[:i])) |
| 108 | buf = b'' |
| 109 | tmp = tmp[i + len_delim:] |
| 110 | |
| 111 | |
| 112 | # ((opt, type), ... ) |