Class, which stores fetched data on storage device as JSON files.
| 61 | |
| 62 | |
| 63 | class PyfaJsonWriter(BaseWriter): |
| 64 | """ |
| 65 | Class, which stores fetched data on storage device |
| 66 | as JSON files. |
| 67 | """ |
| 68 | |
| 69 | def __init__(self, folder, indent=None, group=None): |
| 70 | self.base_folder = folder |
| 71 | self.indent = indent |
| 72 | self.group = group |
| 73 | |
| 74 | @staticmethod |
| 75 | def __grouper(iterable, n, fillvalue=None): |
| 76 | args = [iter(iterable)] * n |
| 77 | return izip_longest(fillvalue=fillvalue, *args) |
| 78 | |
| 79 | def write(self, miner_name, container_name, container_data): |
| 80 | # Create folder structure to path, if not created yet |
| 81 | folder = os.path.join(self.base_folder, self.__secure_name(miner_name)) |
| 82 | if not os.path.exists(folder): |
| 83 | os.makedirs(folder, mode=0o755) |
| 84 | |
| 85 | if type(container_data) == dict: |
| 86 | container_data = OrderedDict(natsort.natsorted(container_data.items())) |
| 87 | |
| 88 | if self.group is None: |
| 89 | filepath = os.path.join(folder, u'{}.json'.format(self.__secure_name(container_name))) |
| 90 | self.__write_file(container_data, filepath) |
| 91 | else: |
| 92 | for i, group in enumerate(PyfaJsonWriter.__grouper(container_data, self.group)): |
| 93 | filepath = os.path.join(folder, u'{}.{}.json'.format(self.__secure_name(container_name), i)) |
| 94 | if type(container_data) in (dict, OrderedDict): |
| 95 | data = dict((k, container_data[k]) for k in group if k is not None) |
| 96 | else: |
| 97 | data = [k for k in group if k is not None] |
| 98 | self.__write_file(data, filepath) |
| 99 | |
| 100 | def __write_file(self, data, filepath): |
| 101 | data_str = json.dumps( |
| 102 | data, |
| 103 | ensure_ascii=False, |
| 104 | cls=CustomEncoder, |
| 105 | indent=self.indent, |
| 106 | # We're handling sorting in customized encoder |
| 107 | sort_keys=False) |
| 108 | data_bytes = data_str.encode('utf8') |
| 109 | with open(filepath, 'wb') as f: |
| 110 | f.write(data_bytes) |
| 111 | |
| 112 | def __secure_name(self, name): |
| 113 | """ |
| 114 | As we're writing to disk, we should get rid of all |
| 115 | filesystem-specific symbols. |
| 116 | """ |
| 117 | # Prefer safe way - replace any characters besides |
| 118 | # alphanumeric and few special characters with |
| 119 | # underscore |
| 120 | writer_safe_name = re.sub(r'[^\w\-.,() ]', '_', name, flags=re.UNICODE) |