| 112 | |
| 113 | |
| 114 | def run_process(directory, label, argv, environment): |
| 115 | record = {"argv": argv, "deadline_seconds": DEADLINE, "returncode": None, |
| 116 | "failure": None, "quiescent": False, |
| 117 | "stdout": label + ".stdout.log", "stderr": label + ".stderr.log"} |
| 118 | start = time.monotonic() |
| 119 | proc = None |
| 120 | threads, started_pipes, states = [], [], {} |
| 121 | stop = threading.Event() |
| 122 | try: |
| 123 | proc = subprocess.Popen(argv, env=environment, stdin=subprocess.DEVNULL, |
| 124 | stdout=subprocess.PIPE, stderr=subprocess.PIPE, |
| 125 | bufsize=0, start_new_session=True) |
| 126 | record["pid"] = proc.pid |
| 127 | for kind, pipe in (("stdout", proc.stdout), ("stderr", proc.stderr)): |
| 128 | state = {"seen": 0, "saved": 0, "overflow": False, "error": None} |
| 129 | thread = threading.Thread(target=drain, |
| 130 | args=(pipe, directory / record[kind], state, stop), daemon=True) |
| 131 | thread.start() |
| 132 | threads.append(thread) |
| 133 | started_pipes.append(pipe) |
| 134 | states[kind] = state |
| 135 | while proc.poll() is None: |
| 136 | left = DEADLINE - (time.monotonic() - start) |
| 137 | if stop.is_set() or left <= 0: |
| 138 | record["failure"] = "capture limit/error" if stop.is_set() else "timeout" |
| 139 | break |
| 140 | try: |
| 141 | proc.wait(timeout=min(left, 0.1)) |
| 142 | except subprocess.TimeoutExpired: |
| 143 | pass |
| 144 | except (OSError, RuntimeError, KeyboardInterrupt) as exc: |
| 145 | record["failure"] = type(exc).__name__ + ": " + str(exc) |
| 146 | finally: |
| 147 | if proc is not None: |
| 148 | try: |
| 149 | record["quiescent"], record["cleanup"] = quiesce(proc) |
| 150 | if record["cleanup"] and record["failure"] is None: |
| 151 | record["failure"] = "unexpected processes remained after leader exit" |
| 152 | except (OSError, KeyboardInterrupt) as exc: |
| 153 | record["cleanup_error"] = str(exc) |
| 154 | if not record["quiescent"]: |
| 155 | record["failure"] = record["failure"] or "owned process group did not quiesce" |
| 156 | for pipe in (proc.stdout, proc.stderr): |
| 157 | if pipe not in started_pipes: |
| 158 | pipe.close() |
| 159 | until = time.monotonic() + GRACE |
| 160 | for thread in threads: |
| 161 | thread.join(timeout=max(0, until - time.monotonic())) |
| 162 | if any(thread.is_alive() for thread in threads): |
| 163 | record["failure"] = record["failure"] or "pipe readers did not quiesce" |
| 164 | record["returncode"] = proc.poll() |
| 165 | if any(state["overflow"] or state["error"] for state in states.values()): |
| 166 | record["failure"] = record["failure"] or "bounded capture failed" |
| 167 | record["streams"] = states |
| 168 | record["elapsed_seconds"] = round(time.monotonic() - start, 3) |
| 169 | PROCESS_RECORDS.append(record) |
| 170 | save(directory / (label + ".process.json"), record) |
| 171 | return record |