| 104 | |
| 105 | |
| 106 | class Janitor: |
| 107 | |
| 108 | # FIXME delete_chars: Should anything else go here? Special chars? |
| 109 | def __init__( |
| 110 | self, |
| 111 | ngram_n=13, |
| 112 | window_to_remove=200, |
| 113 | too_dirty_cutoff=10, |
| 114 | minimum_slice_length=200, |
| 115 | delete_chars=string.punctuation, |
| 116 | ): |
| 117 | self.ngram_n = ngram_n |
| 118 | self.window_to_remove = window_to_remove |
| 119 | self.too_dirty_cutoff = too_dirty_cutoff |
| 120 | self.minimum_slice_length = minimum_slice_length |
| 121 | self.delete_chars = delete_chars |
| 122 | |
| 123 | self.dirt_ngrams = set() |
| 124 | |
| 125 | # If in python, we'll translate uppercase to lowercase and delete naughty characters. |
| 126 | # This is fast by python standards |
| 127 | # https://stackoverflow.com/questions/638893/what-is-the-most-efficient-way-in-python-to-convert-a-string-to-all-lowercase-st |
| 128 | self.translation_table = str.maketrans( |
| 129 | string.ascii_lowercase + string.ascii_uppercase, # These characters |
| 130 | string.ascii_lowercase * 2, # Become these characters |
| 131 | self.delete_chars, # These are deleted |
| 132 | ) |
| 133 | |
| 134 | ############## |
| 135 | # I/O for saving contamination ngrams |
| 136 | ############## |
| 137 | |
| 138 | def save_contamination_ngrams(self, filename): |
| 139 | with open(filename, "wb") as fp: |
| 140 | pickle.dump(filename, fp) |
| 141 | |
| 142 | def load_contamination_ngrams(self, filename): |
| 143 | with open(filename, "rb") as fp: |
| 144 | self.dirt_ngrams = pickle.load(fp) |
| 145 | |
| 146 | ############## |
| 147 | # Call these :) |
| 148 | ############## |
| 149 | |
| 150 | def register_contaminant(self, dirt_string): |
| 151 | """Register a string as contamination to be removed, e.g. a test set |
| 152 | This breaks the dirt_string into ngrams to store for future cleaning""" |
| 153 | if JANITOR_CPP: |
| 154 | return self.register_contaminant_cpp(dirt_string) |
| 155 | else: |
| 156 | print("WARNING: Janitor running in python mode") |
| 157 | return self.register_contaminant_python(dirt_string) |
| 158 | |
| 159 | def clean(self, dirty_string): |
| 160 | """Clean a string (e.g. a training set) by removing all ngrams previously |
| 161 | registered as contaminants. Returns a list of clean chunks, or empty if |
| 162 | the string was too dirty""" |
| 163 | if JANITOR_CPP: |