| 255 | ###################################################################### |
| 256 | # Let us generate some realistic fake data from Zipf's law. |
| 257 | def zipf_sentence_lengths(alpha: float, batch_size: int) -> torch.Tensor: |
| 258 | # generate fake corpus by unigram Zipf distribution |
| 259 | # from wikitext-2 corpus, we get rank "." = 3, "!" = 386, "?" = 858 |
| 260 | sentence_lengths = np.empty(batch_size, dtype=int) |
| 261 | for ibatch in range(batch_size): |
| 262 | sentence_lengths[ibatch] = 1 |
| 263 | word = np.random.zipf(alpha) |
| 264 | while word != 3 and word != 386 and word != 858: |
| 265 | sentence_lengths[ibatch] += 1 |
| 266 | word = np.random.zipf(alpha) |
| 267 | return torch.tensor(sentence_lengths) |
| 268 | |
| 269 | ###################################################################### |
| 270 | # Create nested tensor batch inputs |