()
| 78 | |
| 79 | |
| 80 | def main(): |
| 81 | parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) |
| 82 | parser.add_argument("--models_dir", help="models dir", required=True) |
| 83 | parser.add_argument( |
| 84 | "--load_and_run_file", help="path for load_and_run", required=True |
| 85 | ) |
| 86 | args = parser.parse_args() |
| 87 | assert os.path.isdir( |
| 88 | args.models_dir |
| 89 | ), "invalid args for models_dir, need a dir for models" |
| 90 | assert os.path.isfile(args.load_and_run_file), "invalid args for load_and_run_file" |
| 91 | for m in test_cpu_models: |
| 92 | assert os.path.isfile( |
| 93 | os.path.join(args.models_dir, m) |
| 94 | ), "invalid args for models_dir, need put model: {} to args.models_dir".format( |
| 95 | test_cpu_models |
| 96 | ) |
| 97 | |
| 98 | # init device |
| 99 | ssh = SshConnector() |
| 100 | ssh.setup(device["login_name"], device["ip"], device["port"]) |
| 101 | # create test dir |
| 102 | workspace = "cpu_evaluation_workspace" |
| 103 | ssh.cmd(["mkdir -p {}".format(workspace)]) |
| 104 | # copy load_and_run_file |
| 105 | ssh.copy([args.load_and_run_file], workspace) |
| 106 | # call test |
| 107 | result = [] |
| 108 | for m in test_cpu_models: |
| 109 | m_path = os.path.join(args.models_dir, m) |
| 110 | # copy model file |
| 111 | ssh.copy([m_path], workspace) |
| 112 | # run single thread |
| 113 | sub_b = ["-cpu", "-multithread {}".format(device["thread_number"])] |
| 114 | for b in sub_b: |
| 115 | cmd = [] |
| 116 | cmd0 = "cd {} && rm -rf fastrun.cache".format(workspace) |
| 117 | cmd1 = "cd {} && ./load_and_run {} --fast-run --fast_run_algo_policy fastrun.cache --iter 1 --warmup-iter 1 --no-sanity-check --weight-preprocess".format( |
| 118 | workspace, m, b |
| 119 | ) |
| 120 | cmd2 = "cd {} && ./load_and_run {} {} --fast_run_algo_policy fastrun.cache --iter 20 --warmup-iter 5 --no-sanity-check --weight-preprocess --record-comp-seq".format( |
| 121 | workspace, m, b |
| 122 | ) |
| 123 | cmd.append(cmd0) |
| 124 | cmd.append(cmd1) |
| 125 | cmd.append(cmd2) |
| 126 | raw_log = ssh.cmd(cmd) |
| 127 | # logging.debug(raw_log) |
| 128 | ret = get_finally_bench_resulut_from_log(raw_log) |
| 129 | logging.debug("model: {} with backend: {} result is: {}".format(m, b, ret)) |
| 130 | result.append(ret) |
| 131 | |
| 132 | total_time = 0.0 |
| 133 | for r in result: |
| 134 | total_time += r |
| 135 | logging.debug("total time is: {}".format(total_time)) |
| 136 | score = 100000.0 / total_time * 1000 |
| 137 | logging.debug("device: {} score is: {}".format(device["name"], score)) |
no test coverage detected