A default JSON corpus based on gensim TextCorpus. It assumes a file or list of JSON as input. The methods provided by gensim TextCorpus are needed for the GenSim training. Any corpus provided to DocumentSimilarity should provide the methods given in this class.
| 27 | current_milli_time = lambda: int(round(time.time() * 1000)) |
| 28 | |
| 29 | class DefaultJsonCorpus(object): |
| 30 | """ |
| 31 | A default JSON corpus based on gensim TextCorpus. It assumes a file or list of JSON as input. |
| 32 | The methods provided by gensim TextCorpus are needed for the GenSim training. |
| 33 | Any corpus provided to DocumentSimilarity should provide the methods given in this class. |
| 34 | """ |
| 35 | def __init__(self, input=None,create_dictionary=True): |
| 36 | super(DefaultJsonCorpus, self).__init__() |
| 37 | self.input = input |
| 38 | self.dictionary = Dictionary() |
| 39 | self.metadata = False |
| 40 | if create_dictionary: |
| 41 | self.dictionary.add_documents(self.get_texts()) |
| 42 | |
| 43 | |
| 44 | def __iter__(self): |
| 45 | for text in self.get_texts(): |
| 46 | yield self.dictionary.doc2bow(text, allow_update=False) |
| 47 | |
| 48 | def getstream(self): |
| 49 | return utils.file_or_filename(self.input) |
| 50 | |
| 51 | def __len__(self): |
| 52 | if not hasattr(self, 'length'): |
| 53 | # cache the corpus length |
| 54 | self.length = sum(1 for _ in self.get_texts()) |
| 55 | return self.length |
| 56 | |
| 57 | def get_json(self): |
| 58 | if isinstance(self.input,list): |
| 59 | for j in self.input: |
| 60 | yield j |
| 61 | else: |
| 62 | with self.getstream() as lines: |
| 63 | for line in lines: |
| 64 | line = line.rstrip() |
| 65 | j = json.loads(line) |
| 66 | yield j |
| 67 | |
| 68 | def get_texts(self,raw=False): |
| 69 | """ |
| 70 | yield raw text or tokenized text |
| 71 | """ |
| 72 | for j in self.get_json(): |
| 73 | text = j["text"] |
| 74 | if raw: |
| 75 | yield text |
| 76 | else: |
| 77 | yield utils.tokenize(text, deacc=True, lowercase=True) |
| 78 | |
| 79 | def get_meta(self): |
| 80 | """ |
| 81 | return a json object with meta data for the documents. It must return: |
| 82 | id - id for this document |
| 83 | optional title and tags. Tags will be used as base truth used to score document similarity results. |
| 84 | """ |
| 85 | doc_id = 0 |
| 86 | for j in self.get_json(): |
no outgoing calls