Read data from source and target files and put into buckets. Args: source_path: path to the files with token-ids for the source language. target_path: path to the file with token-ids for the target language; it must be aligned with the source file: n-th line contains the desired
(source_path, target_path, max_size=None)
| 62 | |
| 63 | |
| 64 | def read_data(source_path, target_path, max_size=None): |
| 65 | """Read data from source and target files and put into buckets. |
| 66 | |
| 67 | Args: |
| 68 | source_path: path to the files with token-ids for the source language. |
| 69 | target_path: path to the file with token-ids for the target language; |
| 70 | it must be aligned with the source file: n-th line contains the desired |
| 71 | output for n-th line from the source_path. |
| 72 | max_size: maximum number of lines to read, all other will be ignored; |
| 73 | if 0 or None, data files will be read completely (no limit). |
| 74 | |
| 75 | Returns: |
| 76 | data_set: a list of length len(_buckets); data_set[n] contains a list of |
| 77 | (source, target) pairs read from the provided data files that fit |
| 78 | into the n-th bucket, i.e., such that len(source) < _buckets[n][0] and |
| 79 | len(target) < _buckets[n][1]; source and target are lists of token-ids. |
| 80 | """ |
| 81 | data_set = [[] for _ in _buckets] |
| 82 | with tf.gfile.GFile(source_path, mode="r") as source_file: |
| 83 | with tf.gfile.GFile(target_path, mode="r") as target_file: |
| 84 | source, target = source_file.readline(), target_file.readline() |
| 85 | counter = 0 |
| 86 | while source and target and (not max_size or counter < max_size): |
| 87 | counter += 1 |
| 88 | if counter % 10 == 0: |
| 89 | print(" reading data line %d" % counter) |
| 90 | sys.stdout.flush() |
| 91 | source_ids = [int(x) for x in source.split()] |
| 92 | target_ids = [int(x) for x in target.split()] |
| 93 | target_ids.append(data_utils.EOS_ID) |
| 94 | for bucket_id, (source_size, target_size) in enumerate(_buckets): |
| 95 | if len(source_ids) < source_size and len(target_ids) < target_size: |
| 96 | data_set[bucket_id].append([source_ids, target_ids]) |
| 97 | break |
| 98 | source, target = source_file.readline(), target_file.readline() |
| 99 | return data_set |
| 100 | |
| 101 | |
| 102 | def create_model(session, forward_only): |