(input_table, tab_id, LABEL_DICT, TYPE_DICT, CLASS_DICT, TYPE_SCORE_DICT, RELATION_DICT)
| 132 | |
| 133 | #compute column semantics |
| 134 | def computeColumnSemantics(input_table, tab_id, LABEL_DICT, TYPE_DICT, CLASS_DICT, TYPE_SCORE_DICT, RELATION_DICT): |
| 135 | col_id = 0 |
| 136 | not_found_in_yago = [] |
| 137 | column_bag_of_words = [] |
| 138 | column_dictionary = {} |
| 139 | for (columnName, columnData) in input_table.iteritems(): |
| 140 | if genFunc.getColumnType(input_table[columnName].tolist()) == 1: #check column Type |
| 141 | input_table[columnName] = input_table[columnName].map(str) |
| 142 | #get unique values in the column and preprocess them. |
| 143 | value_list = genFunc.preprocessListValues(input_table[columnName].unique()) |
| 144 | all_found_types = {} |
| 145 | total_kb_hits = 0 |
| 146 | for value in value_list: |
| 147 | current_entities = set() |
| 148 | current_types = set() |
| 149 | current_entities = RELATION_DICT.get(str(col_id) + "_"+ value, "None") |
| 150 | if current_entities != "None": |
| 151 | total_kb_hits += 1 |
| 152 | for entity in current_entities: |
| 153 | if entity in TYPE_DICT: |
| 154 | temp_type = TYPE_DICT[entity] |
| 155 | for entity_type in temp_type: |
| 156 | current_types.add(entity_type) |
| 157 | for each_type in current_types: |
| 158 | if each_type in all_found_types: |
| 159 | all_found_types[each_type] +=1 |
| 160 | else: |
| 161 | all_found_types[each_type] = 1 |
| 162 | else: |
| 163 | current_entities = LABEL_DICT.get(value, "None") |
| 164 | if current_entities != "None": #found in KB |
| 165 | total_kb_hits += 1 |
| 166 | for entity in current_entities: |
| 167 | if entity in TYPE_DICT: |
| 168 | temp_type = TYPE_DICT[entity] |
| 169 | for entity_type in temp_type: |
| 170 | current_types.add(entity_type) |
| 171 | for each_type in current_types: |
| 172 | if each_type in all_found_types: |
| 173 | all_found_types[each_type] +=1 |
| 174 | else: |
| 175 | all_found_types[each_type] = 1 |
| 176 | else: |
| 177 | not_found_in_yago.append(value) |
| 178 | #print(all_found_types) |
| 179 | #find the top-level type with highest count. |
| 180 | all_top_types = [v for v in sorted(all_found_types.items(), key=lambda kv: (-kv[1], kv[0])) if v[0] in CLASS_DICT] |
| 181 | if all_top_types: |
| 182 | selected_top_type = all_top_types[0][0] |
| 183 | children_of_top_types = CLASS_DICT[selected_top_type] |
| 184 | #add children of top types to the bag of word\ |
| 185 | for each in all_found_types: |
| 186 | if each in children_of_top_types and each in type_score_dict: |
| 187 | penalized_score = type_score_dict[each][0] |
| 188 | #penalization score |
| 189 | current_type_score = all_found_types[each] * penalized_score |
| 190 | column_bag_of_words.append((each+"-c", str(col_id),current_type_score/ total_kb_hits)) |
| 191 | if int(col_id) in column_dictionary: |
no test coverage detected