Function main

tools/download_prices.py:48–99

()

        self.downloader.download(self.stocks)

def main():
    args = parse_args()

    cfg = Config.fromfile(args.config)

    if args.cfg_options is None:
        args.cfg_options = dict()
    if args.root is not None:
        args.cfg_options["root"] = args.root
    if args.workdir is not None:
        args.cfg_options["workdir"] = args.workdir
    if args.tag is not None:
        args.cfg_options["tag"] = args.tag
    if args.batch_size is not None:
        args.cfg_options["batch_size"] = args.batch_size
    cfg.merge_from_dict(args.cfg_options)

    update_data_root(cfg, root=args.root)

    exp_path = os.path.join(cfg.root, cfg.workdir, cfg.tag)
    if args.if_remove is None:
        args.if_remove = bool(input(f"| Arguments PRESS 'y' to REMOVE: {exp_path}? ") == 'y')
    if args.if_remove:
        import shutil
        shutil.rmtree(exp_path, ignore_errors=True)
        print(f"| Arguments Remove work_dir: {exp_path}")
    else:
        print(f"| Arguments Keep work_dir: {exp_path}")
    os.makedirs(exp_path, exist_ok=True)

    downloader = DOWNLOADER.build(cfg.downloader)

    print(f"| Check Downloading {cfg.tag}...")
    stocks = downloader.check_download()
    print(f"| Check Downloading {cfg.tag} Done! Failed: {len(stocks)} / {len(downloader.stocks)}")

    batch_size = cfg.batch_size if cfg.batch_size < len(stocks) else 5
    batch_size = min(len(stocks), batch_size)

    processes = []
    remaining_stocks = downloader.check_download()

    while remaining_stocks:
        batch = remaining_stocks[:batch_size]
        remaining_stocks = remaining_stocks[batch_size:]

        process = StockDownloaderProcess(batch, downloader)
        processes.append(process)
        process.start()

    for process in processes:
        process.join()

if __name__ == '__main__':
    main()