Load data from path to generate corpus
(cls, path, fields)
| 109 | |
| 110 | @classmethod |
| 111 | def load(cls, path, fields): |
| 112 | """Load data from path to generate corpus""" |
| 113 | start, sentences = 0, [] |
| 114 | fields = [fd if fd is not None else Field(str(i)) for i, fd in enumerate(fields)] |
| 115 | with open(path, 'r', encoding='utf-8') as f: |
| 116 | lines = [ |
| 117 | line.strip() for line in f.readlines() |
| 118 | if not line.startswith('#') and (len(line) == 1 or line.split()[0].isdigit()) |
| 119 | ] |
| 120 | for i, line in enumerate(lines): |
| 121 | if not line: |
| 122 | values = list(zip(*[j.split('\t') for j in lines[start:i]])) |
| 123 | if values: |
| 124 | sentences.append(Sentence(fields, values)) |
| 125 | start = i + 1 |
| 126 | |
| 127 | return cls(fields, sentences) |
| 128 | |
| 129 | @classmethod |
| 130 | def load_lac_results(cls, inputs, fields): |