text -> 0-order probability statistics as a dictionary Text must not contain the NUL (0x00) character because that's used to indicate the end of data.
(text)
| 18 | R = Rational.rational |
| 19 | |
| 20 | def train(text): |
| 21 | """text -> 0-order probability statistics as a dictionary |
| 22 | |
| 23 | Text must not contain the NUL (0x00) character because that's |
| 24 | used to indicate the end of data. |
| 25 | """ |
| 26 | assert "\x00" not in text |
| 27 | counts = {} |
| 28 | for c in text: |
| 29 | counts[c]=counts.get(c,0)+1 |
| 30 | counts["\x00"] = 1 |
| 31 | tot_letters = sum(counts.values()) |
| 32 | |
| 33 | tot = 0 |
| 34 | d = {} |
| 35 | prev = R(0) |
| 36 | for c, count in counts.items(): |
| 37 | next = R(tot + count, tot_letters) |
| 38 | d[c] = (prev, next) |
| 39 | prev = next |
| 40 | tot = tot + count |
| 41 | assert tot == tot_letters |
| 42 | |
| 43 | return d |
| 44 | |
| 45 | |
| 46 | def encode(text, probs): |
no test coverage detected