| 8 | |
| 9 | |
| 10 | class StringProcess(object): |
| 11 | def __init__(self): |
| 12 | self.other_char = re.compile(r"[^A-Za-z0-9(),!?\'\`]", flags=0) |
| 13 | self.num = re.compile(r"[+-]?\d+\.?\d*", flags=0) |
| 14 | # self.url = re.compile(r"[a-z]*[:.]+\S+|\n|\s+", flags=0) |
| 15 | self.url = re.compile( |
| 16 | r"(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]", flags=0) |
| 17 | self.stop_words = None |
| 18 | self.nlp = None |
| 19 | |
| 20 | def clean_str(self, string): |
| 21 | string = re.sub(self.other_char, " ", string) |
| 22 | string = re.sub(r"\'s", " \'s", string) |
| 23 | string = re.sub(r"\'ve", " \'ve", string) |
| 24 | string = re.sub(r"n\'t", " n\'t", string) |
| 25 | string = re.sub(r"\'re", " \'re", string) |
| 26 | string = re.sub(r"\'d", " \'d", string) |
| 27 | string = re.sub(r"\'ll", " \'ll", string) |
| 28 | string = re.sub(r",", " , ", string) |
| 29 | string = re.sub(r"!", " ! ", string) |
| 30 | string = re.sub(r"\(", " \( ", string) |
| 31 | string = re.sub(r"\)", " \) ", string) |
| 32 | string = re.sub(r"\?", " \? ", string) |
| 33 | string = re.sub(r"\s{2,}", " ", string) |
| 34 | |
| 35 | return string.strip().lower() |
| 36 | |
| 37 | def norm_str(self, string): |
| 38 | string = re.sub(self.other_char, " ", string) |
| 39 | |
| 40 | if self.nlp is None: |
| 41 | from spacy.lang.en import English |
| 42 | self.nlp = English() |
| 43 | |
| 44 | new_doc = list() |
| 45 | doc = self.nlp(string) |
| 46 | for token in doc: |
| 47 | if token.is_space or token.is_punct: |
| 48 | continue |
| 49 | if token.is_digit: |
| 50 | token = "[num]" |
| 51 | else: |
| 52 | token = token.text |
| 53 | |
| 54 | new_doc.append(token) |
| 55 | |
| 56 | return " ".join(new_doc).lower() |
| 57 | |
| 58 | def lean_str_sst(self, string): |
| 59 | """ |
| 60 | Tokenization/string cleaning for the SST yelp_dataset |
| 61 | Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py |
| 62 | """ |
| 63 | string = re.sub(self.other_char, " ", string) |
| 64 | string = re.sub(r"\s{2,}", " ", string) |
| 65 | return string.strip().lower() |
| 66 | |
| 67 | def remove_stopword(self, string): |