| 1027 | self.log_handler.setLevel(logging.INFO) |
| 1028 | |
| 1029 | def run_one(self, host_id=None, reserve_timeout=None, erase_created_workdir=False): |
| 1030 | if host_id == None: |
| 1031 | host_id = ("%s:%i" % (socket.gethostname(), os.getpid()),) |
| 1032 | job = None |
| 1033 | start_time = time.time() |
| 1034 | mj = self.mj |
| 1035 | while job is None: |
| 1036 | if reserve_timeout and (time.time() - start_time) > reserve_timeout: |
| 1037 | raise ReserveTimeout() |
| 1038 | job = mj.reserve(host_id, exp_key=self.exp_key) |
| 1039 | if not job: |
| 1040 | interval = 1 + numpy.random.rand() * (float(self.poll_interval) - 1.0) |
| 1041 | logger.info("no job found, sleeping for %.1fs" % interval) |
| 1042 | time.sleep(interval) |
| 1043 | |
| 1044 | logger.debug("job found: %s" % str(job)) |
| 1045 | |
| 1046 | # -- don't let the cmd mess up our trial object |
| 1047 | spec = spec_from_misc(job["misc"]) |
| 1048 | |
| 1049 | ctrl = MongoCtrl( |
| 1050 | trials=MongoTrials(mj, exp_key=job["exp_key"], refresh=False), |
| 1051 | read_only=False, |
| 1052 | current_trial=job, |
| 1053 | ) |
| 1054 | if self.workdir is None: |
| 1055 | workdir = job["misc"].get("workdir", os.getcwd()) |
| 1056 | if workdir is None: |
| 1057 | workdir = "" |
| 1058 | workdir = os.path.join(workdir, str(job["_id"])) |
| 1059 | else: |
| 1060 | workdir = self.workdir |
| 1061 | workdir = os.path.abspath(os.path.expanduser(workdir)) |
| 1062 | try: |
| 1063 | root_logger = logging.getLogger() |
| 1064 | if self.logfilename: |
| 1065 | self.make_log_handler() |
| 1066 | root_logger.addHandler(self.log_handler) |
| 1067 | |
| 1068 | cmd = job["misc"]["cmd"] |
| 1069 | cmd_protocol = cmd[0] |
| 1070 | try: |
| 1071 | if cmd_protocol == "cpickled fn": |
| 1072 | worker_fn = pickler.loads(cmd[1]) |
| 1073 | elif cmd_protocol == "call evaluate": |
| 1074 | bandit = pickler.loads(cmd[1]) |
| 1075 | worker_fn = bandit.evaluate |
| 1076 | elif cmd_protocol == "token_load": |
| 1077 | cmd_toks = cmd[1].split(".") |
| 1078 | cmd_module = ".".join(cmd_toks[:-1]) |
| 1079 | worker_fn = exec_import(cmd_module, cmd[1]) |
| 1080 | elif cmd_protocol == "bandit_json evaluate": |
| 1081 | bandit = json_call(cmd[1]) |
| 1082 | worker_fn = bandit.evaluate |
| 1083 | elif cmd_protocol == "driver_attachment": |
| 1084 | # name = 'driver_attachment_%s' % job['exp_key'] |
| 1085 | blob = ctrl.trials.attachments[cmd[1]] |
| 1086 | bandit_name, bandit_args, bandit_kwargs = pickler.loads(blob) |