Tokenization/string cleaning for all datasets except for SST. Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
(string)
| 23 | |
| 24 | |
| 25 | def clean_str(string): |
| 26 | """ |
| 27 | Tokenization/string cleaning for all datasets except for SST. |
| 28 | Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py |
| 29 | """ |
| 30 | string = string.strip().strip('"') |
| 31 | string = re.sub(r"[^A-Za-z0-9(),!?\.\'\`]", " ", string) |
| 32 | string = re.sub(r"\'s", " \'s", string) |
| 33 | string = re.sub(r"\'ve", " \'ve", string) |
| 34 | string = re.sub(r"n\'t", " n\'t", string) |
| 35 | string = re.sub(r"\'re", " \'re", string) |
| 36 | string = re.sub(r"\'d", " \'d", string) |
| 37 | string = re.sub(r"\'ll", " \'ll", string) |
| 38 | string = re.sub(r",", " , ", string) |
| 39 | string = re.sub(r"\.", " \. ", string) |
| 40 | string = re.sub(r"\"", " , ", string) |
| 41 | string = re.sub(r"!", " ! ", string) |
| 42 | string = re.sub(r"\(", " \( ", string) |
| 43 | string = re.sub(r"\)", " \) ", string) |
| 44 | string = re.sub(r"\?", " \? ", string) |
| 45 | string = re.sub(r"\s{2,}", " ", string) |
| 46 | return string.strip().lower() |
| 47 | |
| 48 | |
| 49 | def convert_multi_slots_to_single_slots(slots): |