| 404 | MIN_COUNT = 3 # Minimum word count threshold for trimming |
| 405 | |
| 406 | def trimRareWords(voc, pairs, MIN_COUNT): |
| 407 | # Trim words used under the MIN_COUNT from the voc |
| 408 | voc.trim(MIN_COUNT) |
| 409 | # Filter out pairs with trimmed words |
| 410 | keep_pairs = [] |
| 411 | for pair in pairs: |
| 412 | input_sentence = pair[0] |
| 413 | output_sentence = pair[1] |
| 414 | keep_input = True |
| 415 | keep_output = True |
| 416 | # Check input sentence |
| 417 | for word in input_sentence.split(' '): |
| 418 | if word not in voc.word2index: |
| 419 | keep_input = False |
| 420 | break |
| 421 | # Check output sentence |
| 422 | for word in output_sentence.split(' '): |
| 423 | if word not in voc.word2index: |
| 424 | keep_output = False |
| 425 | break |
| 426 | |
| 427 | # Only keep pairs that do not contain trimmed word(s) in their input or output sentence |
| 428 | if keep_input and keep_output: |
| 429 | keep_pairs.append(pair) |
| 430 | |
| 431 | print("Trimmed from {} pairs to {}, {:.4f} of total".format(len(pairs), len(keep_pairs), len(keep_pairs) / len(pairs))) |
| 432 | return keep_pairs |
| 433 | |
| 434 | |
| 435 | # Trim voc and pairs |