Parameters ---------- model_type : string gensim_lsi,gensim_lda,gensim_rp,sklearn_nmf vec_size : int vector size of model annoy_trees : int number of trees to create for Annoy approx nearest neighbour work_folder : str folder for tmp files s
| 96 | |
| 97 | |
| 98 | class DocumentSimilarity(Recommender): |
| 99 | |
| 100 | """ |
| 101 | |
| 102 | Parameters |
| 103 | ---------- |
| 104 | |
| 105 | model_type : string |
| 106 | gensim_lsi,gensim_lda,gensim_rp,sklearn_nmf |
| 107 | vec_size : int |
| 108 | vector size of model |
| 109 | annoy_trees : int |
| 110 | number of trees to create for Annoy approx nearest neighbour |
| 111 | work_folder : str |
| 112 | folder for tmp files |
| 113 | sklearn_tfidf_args : dict, optional |
| 114 | args to pass to sklearn TfidfVectorizer |
| 115 | sklear_nmf_args : dict, optional |
| 116 | args to pass to sklearn NMF model |
| 117 | """ |
| 118 | def __init__(self,model_type='gensim_lsi',vec_size=100,annoy_trees=100,work_folder="/tmp",sklearn_tfidf_args={'stop_words':"english"},sklearn_nmf_args={"random_state":1,"alpha":.1,"l1_ratio":.5}): |
| 119 | if not (model_type == 'gensim_lsi' or model_type == 'gensim_lda' or model_type == 'gensim_rp' or model_type == 'sklearn_nmf'): |
| 120 | raise ValueError("Unknown model type") |
| 121 | self.model_type=model_type |
| 122 | self.vec_size=vec_size |
| 123 | self.annoy_trees=annoy_trees |
| 124 | self.gensim_output_prefix = "gensim_index" |
| 125 | self.annoy_output_prefix = "annoy_index" |
| 126 | self.meta_output_prefix = "meta" |
| 127 | self.sklearn_tfidf_args = sklearn_tfidf_args |
| 128 | self.sklearn_nmf_args = sklearn_nmf_args |
| 129 | self.work_folder=work_folder |
| 130 | |
| 131 | def __getstate__(self): |
| 132 | """ |
| 133 | Remove things that should not be pickled as they are handled in save/load |
| 134 | """ |
| 135 | result = self.__dict__.copy() |
| 136 | del result['index'] |
| 137 | del result['index_annoy'] |
| 138 | del result['seq2meta'] |
| 139 | del result['id2meta'] |
| 140 | return result |
| 141 | |
| 142 | def __setstate__(self, dict): |
| 143 | self.__dict__ = dict |
| 144 | |
| 145 | |
| 146 | |
| 147 | def create_gensim_model(self,corpus): |
| 148 | """ |
| 149 | Create a gensim model |
| 150 | |
| 151 | Parameters |
| 152 | ---------- |
| 153 | |
| 154 | corpus : an object that satisfies a gensim TextCorpus |
| 155 |
no outgoing calls