Take corpus, split it into sentences, and extract word frequencies. Write frequencies to `filepath` as a tsv. Only write the first MAX_SENTENCEPIECE_SENTENCES most common words to the file.
(dataset, filepath, filetype='tsv')
| 615 | |
| 616 | |
| 617 | def get_corpus_freq(dataset, filepath, filetype='tsv'): |
| 618 | """ |
| 619 | Take corpus, split it into sentences, and extract word frequencies. |
| 620 | Write frequencies to `filepath` as a tsv. Only write the first |
| 621 | MAX_SENTENCEPIECE_SENTENCES most common words to the file. |
| 622 | """ |
| 623 | nltk.download('punkt', download_dir="./nltk") |
| 624 | if filetype == 'tsv': |
| 625 | delimiter = '\t' |
| 626 | else: |
| 627 | delimiter = ',' |
| 628 | |
| 629 | print("compute corpus frequency\n", flush=True) |
| 630 | |
| 631 | total_sentence_count = 0 |
| 632 | maxlen = 0 |
| 633 | freqs = {} |
| 634 | for entry in dataset: |
| 635 | if isinstance(entry, dict): |
| 636 | entry = entry['text'] |
| 637 | lines = entry.strip().split('\n') |
| 638 | for line in lines: |
| 639 | sentences = nltk_tokenize.sent_tokenize(line) |
| 640 | total_sentence_count += len(sentences) |
| 641 | for sentence in sentences: |
| 642 | maxlen = max(len(line), maxlen) |
| 643 | for word in sentence.split(): |
| 644 | if word not in freqs: |
| 645 | freqs[word] = 0 |
| 646 | freqs[word] += 1 |
| 647 | |
| 648 | print("length of freqs before truncating " + str(len(freqs)), flush=True) |
| 649 | print("file path for freq " + str(filepath), flush=True) |
| 650 | |
| 651 | freqs_sorted = {} |
| 652 | counter = 0 |
| 653 | for word, count in sorted(freqs.items(), key=lambda x: x[1], reverse=True): |
| 654 | if counter >= MAX_SENTENCEPIECE_SENTENCES: |
| 655 | break |
| 656 | counter += 1 |
| 657 | freqs_sorted[word] = count |
| 658 | |
| 659 | print("length of freqs after trancating " + str(len(freqs_sorted)), flush=True) |
| 660 | |
| 661 | with open(filepath, 'w') as f: |
| 662 | writer = csv.writer(f, delimiter=delimiter) |
| 663 | for k, v in freqs_sorted.items(): |
| 664 | writer.writerow([str(k), str(v)]) |
| 665 | |
| 666 | return total_sentence_count, maxlen |
| 667 | |
| 668 | |
| 669 | class SentencePieceTokenizer(TextTokenizer): |