Args: predictors ([PredictorBase])
(predictors, nr_eval, get_player_fn, verbose=False)
| 46 | |
| 47 | |
| 48 | def eval_with_funcs(predictors, nr_eval, get_player_fn, verbose=False): |
| 49 | """ |
| 50 | Args: |
| 51 | predictors ([PredictorBase]) |
| 52 | """ |
| 53 | class Worker(StoppableThread, ShareSessionThread): |
| 54 | def __init__(self, func, queue): |
| 55 | super(Worker, self).__init__() |
| 56 | self._func = func |
| 57 | self.q = queue |
| 58 | |
| 59 | def func(self, *args, **kwargs): |
| 60 | if self.stopped(): |
| 61 | raise RuntimeError("stopped!") |
| 62 | return self._func(*args, **kwargs) |
| 63 | |
| 64 | def run(self): |
| 65 | with self.default_sess(): |
| 66 | player = get_player_fn(train=False) |
| 67 | while not self.stopped(): |
| 68 | try: |
| 69 | score = play_one_episode(player, self.func) |
| 70 | except RuntimeError: |
| 71 | return |
| 72 | self.queue_put_stoppable(self.q, score) |
| 73 | |
| 74 | q = queue.Queue() |
| 75 | threads = [Worker(f, q) for f in predictors] |
| 76 | |
| 77 | for k in threads: |
| 78 | k.start() |
| 79 | time.sleep(0.1) # avoid simulator bugs |
| 80 | stat = StatCounter() |
| 81 | |
| 82 | def fetch(): |
| 83 | r = q.get() |
| 84 | stat.feed(r) |
| 85 | if verbose: |
| 86 | logger.info("Score: {}".format(r)) |
| 87 | |
| 88 | for _ in get_tqdm(range(nr_eval)): |
| 89 | fetch() |
| 90 | # waiting is necessary, otherwise the estimated mean score is biased |
| 91 | logger.info("Waiting for all the workers to finish the last run...") |
| 92 | for k in threads: |
| 93 | k.stop() |
| 94 | for k in threads: |
| 95 | k.join() |
| 96 | while q.qsize(): |
| 97 | fetch() |
| 98 | |
| 99 | if stat.count > 0: |
| 100 | return (stat.average, stat.max) |
| 101 | return (0, 0) |
| 102 | |
| 103 | |
| 104 | def eval_model_multithread(pred, nr_eval, get_player_fn): |