| 24 | |
| 25 | |
| 26 | class Bow(ModelBase): |
| 27 | |
| 28 | def __init__( self, |
| 29 | dic_path=const.BOW_DIC_PATH, |
| 30 | bow_index_path=const.BOW_INDEX_PARH, |
| 31 | stop_word=StopWords ): |
| 32 | ''' |
| 33 | ''' |
| 34 | |
| 35 | self.dic_path = dic_path |
| 36 | self.bow_index_path = bow_index_path |
| 37 | for per_path in [self.dic_path, self.bow_index_path]: |
| 38 | per_path = '/'.join(per_path.split('/')[:-1]) |
| 39 | if os.path.exists(per_path) == False: |
| 40 | os.makedirs(per_path) |
| 41 | self.stop_word = stop_word() |
| 42 | self.vectorizer = CountVectorizer(stop_words = None, max_df=cfg.emb.MAX_DF, min_df=cfg.emb.MIN_DF, max_features=cfg.emb.MAX_FEATURES, token_pattern='(?u)\\b\\w\\w*\\b') |
| 43 | |
| 44 | # init |
| 45 | def init(self, words_list=None, update=True): |
| 46 | if (~os.path.exists(self.dic_path) or ~os.path.exists(self.bow_model_path) or ~os.path.exists(self.bow_index_path) or update) and (words_list!=None): |
| 47 | word_list = self._seg_word(words_list) |
| 48 | |
| 49 | if os.path.exists(self.dic_path) and update==False: |
| 50 | with open(self.dic_path, 'rb') as f: |
| 51 | self.vectorizer = pickle.load(f) |
| 52 | else: |
| 53 | try: |
| 54 | logging.info('[Bow] start build tfidf model.') |
| 55 | if words_list==None: |
| 56 | logging.error( '[Bow] words_list is None' ) |
| 57 | self._gen_model(word_list) |
| 58 | logging.info('[Bow] build tfidf model success.') |
| 59 | except Exception as e: |
| 60 | logging.error( '[Bow] build tfidf model error，error info: {} '.format(e) ) |
| 61 | if words_list!=None: |
| 62 | self.words_list_pre = [] |
| 63 | for per_word in words_list: |
| 64 | self.words_list_pre.append( self._normalize( self._predict(per_word) )[0] ) |
| 65 | self.words_list_pre = np.array(self.words_list_pre) |
| 66 | return self |
| 67 | ''' |
| 68 | # seg word |
| 69 | def _seg_word(self, words_list, jieba_flag=True, del_stopword=False): |
| 70 | if jieba_flag: |
| 71 | word_list = [[self.stop_word.del_stopwords(words) if del_stopword else word for word in jieba.cut(words)] for words in words_list] |
| 72 | else: |
| 73 | word_list = [[self.stop_word.del_stopwords(words) if del_stopword else word for word in words] for words in words_list] |
| 74 | print( 'word_list>>>', word_list ) |
| 75 | return [ ' '.join(word) for word in word_list ] |
| 76 | ''' |
| 77 | # seg word |
| 78 | def _seg_word(self, words_list, jieba_flag=cfg.emb.JIEBA_FLAG, del_stopword=cfg.emb.DEL_STOPWORD): |
| 79 | word_list = [] |
| 80 | if jieba_flag: |
| 81 | for words in words_list: |
| 82 | if del_stopword: |
| 83 | if words!='' and type(words) == str: |
no outgoing calls
no test coverage detected