| 112 | |
| 113 | |
| 114 | def get_crontab(): |
| 115 | spider_crontab_error_list = [] |
| 116 | path = os.path.join(cur_dir, 'shannon_spider', 'spiders') |
| 117 | spider_file_list = os.listdir(path) |
| 118 | with open(os.path.join(cur_dir, 'dist/spider.crontab'), 'wb') as fw: |
| 119 | for spider_file in spider_file_list: |
| 120 | with open(os.path.join(path, spider_file), 'r', encoding='utf-8') as fr: |
| 121 | file_content = fr.read() |
| 122 | spider_name = re.findall(r'name\s?=\s?.(.*).\s', file_content) |
| 123 | crontab = re.findall(r'crontab\s?=\s?.(.*).\s', file_content) |
| 124 | if spider_name != [] and crontab != []: |
| 125 | crontab_info = crontab[0].split(' ') |
| 126 | if len(crontab_info) == 5: |
| 127 | info = spider_name[0] + ':' + ','.join(crontab_info) + '|' |
| 128 | fw.write(bytes(info, encoding='utf-8')) |
| 129 | else: |
| 130 | spider_crontab_error_list.append(spider_name[0]) |
| 131 | if len(spider_crontab_error_list) > 0: |
| 132 | for spider_error_name in spider_crontab_error_list: |
| 133 | print('*' * 10 + '爬虫{}:crontab字段填写有误!'.format(spider_error_name) + '*' * 10) |
| 134 | sys.exit() |
| 135 | |
| 136 | |
| 137 | def main(): |