Given a list of keys that are in a single 'directory' on S3, sort out the details and log what we need before going on to the next "directory". error_log is a csvwriter. Log errors to it with error_log.write_row([key, message]) If dump_dir is specified, JSON which is retrie
(keys, data_csv, error_log, dump_dir=None, force_reload=False)
| 72 | |
| 73 | |
| 74 | def process_keys(keys, data_csv, error_log, dump_dir=None, force_reload=False): |
| 75 | """Given a list of keys that are in a single 'directory' on S3, sort out the details and log what we need |
| 76 | before going on to the next "directory". |
| 77 | |
| 78 | error_log is a csvwriter. Log errors to it with error_log.write_row([key, message]) |
| 79 | |
| 80 | If dump_dir is specified, JSON which is retrieved will be stored there. If dump_dir is specified and force_reload is False, |
| 81 | this script will read a cached file from the dump_dir instead of from S3. |
| 82 | """ |
| 83 | key_parts = keys[0].name.split('/') |
| 84 | if len(key_parts) < 4: |
| 85 | error_log.writerow([keys[0].name, 'key name has too few paths']) |
| 86 | return |
| 87 | user, title_slug = keys[0].name.split('/')[1:3] |
| 88 | |
| 89 | data = None |
| 90 | |
| 91 | if dump_dir and not force_reload: |
| 92 | title_dir = title_slug[:255] # Mac OS can't take file or directory names longer than 255 |
| 93 | mirror_file = os.path.join(dump_dir, user, title_dir, 'data.json') |
| 94 | if os.path.exists(mirror_file): |
| 95 | data = json.load(open(mirror_file,'rb')) |
| 96 | |
| 97 | if data is None: |
| 98 | data = prepare_data_from_keys(keys, error_log) |
| 99 | |
| 100 | if data is None: |
| 101 | return # any anomalies should have been logged already |
| 102 | |
| 103 | if dump_dir: |
| 104 | try: |
| 105 | os.makedirs(os.path.dirname(mirror_file)) |
| 106 | except: |
| 107 | pass #directories already existed |
| 108 | json.dump(data, open(mirror_file, 'wb')) |
| 109 | |
| 110 | |
| 111 | d = { |
| 112 | 'user': user, |
| 113 | 'title_slug': title_slug, |
| 114 | 'status': 'draft', |
| 115 | 'kind': 'map' |
| 116 | } |
| 117 | |
| 118 | d['uploaded_image_count'] = data['image_count'] |
| 119 | d['status'] = data['status'] |
| 120 | |
| 121 | |
| 122 | d['base_layer'] = data['storymap'].get('map_type','default') |
| 123 | if d['base_layer'] == 'zoomify': |
| 124 | d['kind'] = 'gigapixel' |
| 125 | |
| 126 | d['url'] = data['url'] |
| 127 | |
| 128 | d['slide_count'] = len(data['storymap']['slides']) |
| 129 | for i,slide in enumerate(data['storymap']['slides']): |
| 130 | d['slide_index'] = i |
| 131 | if slide is None: |
no test coverage detected