Starts tracing on a single or multiple TPU hosts and saves the result in the given logdir. If no trace was collected, retries tracing for num_tracing_attempts.
| 191 | // given logdir. If no trace was collected, retries tracing for |
| 192 | // num_tracing_attempts. |
| 193 | Status StartTracing(const tensorflow::string& service_addr, |
| 194 | const tensorflow::string& logdir, |
| 195 | const tensorflow::string& workers_list, |
| 196 | bool include_dataset_ops, int duration_ms, |
| 197 | int num_tracing_attempts) { |
| 198 | // Use the current timestamp as the run name. |
| 199 | tensorflow::string session_id = GetCurrentTimeStampAsString(); |
| 200 | constexpr char kProfilePluginDirectory[] = "plugins/profile/"; |
| 201 | tensorflow::string repository_root = |
| 202 | io::JoinPath(logdir, kProfilePluginDirectory); |
| 203 | std::vector<tensorflow::string> hostnames = |
| 204 | tensorflow::str_util::Split(workers_list, ","); |
| 205 | |
| 206 | TF_RETURN_IF_ERROR(MaybeCreateEmptyEventFile(logdir)); |
| 207 | |
| 208 | Status status = Status::OK(); |
| 209 | int remaining_attempts = num_tracing_attempts; |
| 210 | tensorflow::ProfileOptions opts; |
| 211 | opts.set_include_dataset_ops(include_dataset_ops); |
| 212 | while (true) { |
| 213 | std::cout << "Starting to profile TPU traces for " << duration_ms << " ms. " |
| 214 | << "Remaining attempt(s): " << --remaining_attempts << std::endl; |
| 215 | if (hostnames.empty()) { |
| 216 | status = Profile(service_addr, logdir, duration_ms, repository_root, |
| 217 | session_id, opts); |
| 218 | } else { |
| 219 | tensorflow::string tpu_master = service_addr; |
| 220 | status = NewSession(tpu_master, hostnames, duration_ms, repository_root, |
| 221 | session_id, opts); |
| 222 | } |
| 223 | if (remaining_attempts <= 0 || status.ok() || !ShouldRetryTracing(status)) |
| 224 | break; |
| 225 | std::cout << "No trace event is collected. Automatically retrying." |
| 226 | << std::endl |
| 227 | << std::endl; |
| 228 | } |
| 229 | |
| 230 | if (ShouldRetryTracing(status)) { |
| 231 | std::cout << "No trace event is collected after " << num_tracing_attempts |
| 232 | << " attempt(s). " |
| 233 | << "Perhaps, you want to try again (with more attempts?)." |
| 234 | << std::endl |
| 235 | << "Tip: increase number of attempts with --num_tracing_attempts." |
| 236 | << std::endl; |
| 237 | } |
| 238 | return status; |
| 239 | } |
| 240 | |
| 241 | MonitorRequest PopulateMonitorRequest(int duration_ms, int monitoring_level, |
| 242 | bool timestamp) { |
no test coverage detected