(file_tuple, data_class=Event, target_class=ETRTText2Role,
output_folder='data/text2target/framenet',
ignore_nonevent=False, zh=False,
mark_tree=False, type_format='subtype')
| 11 | english_stopwords = set(stopwords.words('english') + ["'s", "'re", "%"]) |
| 12 | |
| 13 | def convert_file_tuple(file_tuple, data_class=Event, target_class=ETRTText2Role, |
| 14 | output_folder='data/text2target/framenet', |
| 15 | ignore_nonevent=False, zh=False, |
| 16 | mark_tree=False, type_format='subtype'): |
| 17 | counter = defaultdict(Counter) |
| 18 | data_counter = defaultdict(Counter) |
| 19 | |
| 20 | event_schema_set = set() |
| 21 | |
| 22 | span_output_folder = output_folder |
| 23 | |
| 24 | if not os.path.exists(span_output_folder): |
| 25 | os.makedirs(span_output_folder) |
| 26 | |
| 27 | for in_filename, output_filename in file_tuple(output_folder): |
| 28 | span_event_output = open(output_filename + '.json', 'w') |
| 29 | |
| 30 | for line in read_file(in_filename): |
| 31 | document = data_class(json.loads(line.strip())) # 每行都是一个json文件格式 |
| 32 | for sentence in document.generate_sentence(type_format=type_format): |
| 33 | |
| 34 | if ignore_nonevent and len(sentence['events']) == 0: |
| 35 | continue |
| 36 | |
| 37 | # 处理schema数据信息, 并进行统计 |
| 38 | for event in sentence['events']: |
| 39 | event_schema_set = event_schema_set | get_schema(event) # set((ET, RT)) 合并后再遍历根据et 整理 dict |
| 40 | sep = '' if zh else ' ' |
| 41 | predicate = sep.join([sentence['tokens'][index] |
| 42 | for index in event['tokens']]) # 触发词的文本信息 |
| 43 | counter['pred'].update([predicate]) |
| 44 | counter['type'].update([event['type']]) # 事件类型 |
| 45 | data_counter[in_filename].update(['event']) |
| 46 | for argument in event['arguments']: |
| 47 | data_counter[in_filename].update(['argument']) |
| 48 | counter['role'].update([argument[0]]) |
| 49 | |
| 50 | data_counter[in_filename].update(['sentence']) |
| 51 | |
| 52 | # 训练集与验证集、测试集区分处理的类别 |
| 53 | if (target_class == ETText2Tri or target_class == ETRTText2Role) and "train" not in in_filename: |
| 54 | # 处理生成 span target |
| 55 | span_source_list, span_target_list = target_class.annotate_span( |
| 56 | tokens=sentence['tokens'], |
| 57 | predicate_arguments=sentence['events'], |
| 58 | zh=zh, |
| 59 | mark_tree=mark_tree, |
| 60 | isTest=True |
| 61 | ) |
| 62 | # print("================================") |
| 63 | else: |
| 64 | # 处理生成 span target |
| 65 | span_source_list, span_target_list = target_class.annotate_span( |
| 66 | tokens=sentence['tokens'], |
| 67 | predicate_arguments=sentence['events'], |
| 68 | zh=zh, |
| 69 | mark_tree=mark_tree |
| 70 | ) |
no test coverage detected