| 49 | |
| 50 | |
| 51 | def get_stories(f): |
| 52 | # data will return a list of triples |
| 53 | # each triple contains: |
| 54 | # 1. a story |
| 55 | # 2. a question about the story |
| 56 | # 3. the answer to the question |
| 57 | data = [] |
| 58 | |
| 59 | # use this list to keep track of the story so far |
| 60 | story = [] |
| 61 | |
| 62 | # print a random story, helpful to see the data |
| 63 | printed = False |
| 64 | for line in f: |
| 65 | line = line.decode('utf-8').strip() |
| 66 | |
| 67 | # split the line number from the rest of the line |
| 68 | nid, line = line.split(' ', 1) |
| 69 | |
| 70 | # see if we should begin a new story |
| 71 | if int(nid) == 1: |
| 72 | story = [] |
| 73 | |
| 74 | # this line contains a question and answer if it has a tab |
| 75 | # question<TAB>answer |
| 76 | # it also tells us which line in the story is relevant to the answer |
| 77 | # Note: we actually ignore this fact, since the model will learn |
| 78 | # which lines are important |
| 79 | # Note: the max line number is not the number of lines of the story |
| 80 | # since lines with questions do not contain any story |
| 81 | # one story may contain MULTIPLE questions |
| 82 | if '\t' in line: |
| 83 | q, a, supporting = line.split('\t') |
| 84 | q = tokenize(q) |
| 85 | |
| 86 | # numbering each line is very useful |
| 87 | # it's the equivalent of adding a unique token to the front |
| 88 | # of each sentence |
| 89 | story_so_far = [[str(i)] + s for i, s in enumerate(story) if s] |
| 90 | |
| 91 | # uncomment if you want to see what a story looks like |
| 92 | # if not printed and np.random.rand() < 0.5: |
| 93 | # print("story_so_far:", story_so_far) |
| 94 | # printed = True |
| 95 | data.append((story_so_far, q, a)) |
| 96 | story.append('') |
| 97 | else: |
| 98 | # just add the line to the current story |
| 99 | story.append(tokenize(line)) |
| 100 | return data |
| 101 | |
| 102 | |
| 103 | # recursively flatten a list |