| 267 | |
| 268 | |
| 269 | def untrusted_check( |
| 270 | dataset: str, |
| 271 | code: str, |
| 272 | inputs: List[Any], |
| 273 | entry_point: str, |
| 274 | expected, |
| 275 | atol, |
| 276 | ref_time: List[float], |
| 277 | fast_check: bool = False, |
| 278 | min_time_limit: float = 0.1, |
| 279 | gt_time_limit_factor: float = 2.0, |
| 280 | ) -> Tuple[str, np.ndarray]: |
| 281 | |
| 282 | time_limits = [max(min_time_limit, gt_time_limit_factor * t) for t in ref_time] |
| 283 | timeout = sum(time_limits) + 1 |
| 284 | if not fast_check: |
| 285 | timeout += 1 # extra time for data collection |
| 286 | |
| 287 | # shared memory objects |
| 288 | progress = Value("i", 0) |
| 289 | stat = Value("i", _UNKNOWN) |
| 290 | details = Array("b", [False for _ in range(len(inputs))]) |
| 291 | feedback_size = 500 |
| 292 | feedback = Array('c', b'\0' * feedback_size) |
| 293 | |
| 294 | p = multiprocessing.Process( |
| 295 | target=unsafe_execute, |
| 296 | args=( |
| 297 | dataset, |
| 298 | entry_point, |
| 299 | code, |
| 300 | inputs, |
| 301 | expected, |
| 302 | time_limits, |
| 303 | atol, |
| 304 | fast_check, |
| 305 | stat, |
| 306 | details, |
| 307 | progress, |
| 308 | feedback, |
| 309 | feedback_size, |
| 310 | ), |
| 311 | ) |
| 312 | p.start() |
| 313 | p.join(timeout=timeout + 1) |
| 314 | if p.is_alive(): |
| 315 | p.terminate() |
| 316 | time.sleep(0.1) |
| 317 | if p.is_alive(): |
| 318 | p.kill() |
| 319 | time.sleep(0.1) |
| 320 | |
| 321 | stat = _mapping[stat.value] |
| 322 | details = details[: progress.value] |
| 323 | feedback = feedback.value.decode("utf-8").strip() |
| 324 | if entry_point not in code: |
| 325 | feedback = f"Please rename your function to {entry_point} as there is no function named {entry_point}." |
| 326 | |