| 1349 | } |
| 1350 | } |
| 1351 | void HogwildWorker::TrainFiles() { |
| 1352 | platform::SetNumThreads(1); |
| 1353 | platform::Timer timeline; |
| 1354 | timeline.Start(); |
| 1355 | |
| 1356 | int total_batch_num = 0; |
| 1357 | // how to accumulate fetched values here |
| 1358 | device_reader_->Start(); |
| 1359 | int cur_batch = 0; |
| 1360 | int batch_cnt = 0; |
| 1361 | if (thread_id_ == 0) { |
| 1362 | quit_flag_.store(false); |
| 1363 | // quit_flag_2 = false; |
| 1364 | } |
| 1365 | g_barrier.wait(); |
| 1366 | |
| 1367 | std::unique_ptr<GarbageCollector> gc = nullptr; |
| 1368 | int64_t max_memory_size = GetEagerDeletionThreshold(); |
| 1369 | if (max_memory_size >= 0) { |
| 1370 | gc = CreateGarbageCollector(place_, max_memory_size); |
| 1371 | } |
| 1372 | bool infer_out_of_ins = false; |
| 1373 | while (true) { |
| 1374 | cur_batch = device_reader_->Next(); |
| 1375 | if (cur_batch <= 0 && !infer_out_of_ins) { |
| 1376 | break; |
| 1377 | } |
| 1378 | if (infer_out_of_ins) { |
| 1379 | for (auto &op : ops_) { |
| 1380 | if (op->Type() == "c_broadcast") { |
| 1381 | op->Run(*thread_scope_, place_); |
| 1382 | } |
| 1383 | if (gc) { |
| 1384 | DeleteUnusedTensors(*thread_scope_, op.get(), unused_vars_, gc.get()); |
| 1385 | } |
| 1386 | } |
| 1387 | } else { |
| 1388 | for (auto &op : ops_) { |
| 1389 | if (FLAGS_gpugraph_enable_print_op_debug) { |
| 1390 | VLOG(0) << "thread id=" << thread_id_ << ", " |
| 1391 | << op->DebugStringEx(thread_scope_); |
| 1392 | } |
| 1393 | op->Run(*thread_scope_, place_); |
| 1394 | if (gc) { |
| 1395 | DeleteUnusedTensors(*thread_scope_, op.get(), unused_vars_, gc.get()); |
| 1396 | } |
| 1397 | } |
| 1398 | } |
| 1399 | |
| 1400 | if (need_dump_field_) { |
| 1401 | DumpField(*thread_scope_, dump_mode_, dump_interval_); |
| 1402 | } |
| 1403 | if (need_dump_param_ && (sharding_mode_ || thread_id_ == 0)) { |
| 1404 | DumpParam(*thread_scope_, batch_cnt); |
| 1405 | } |
| 1406 | |
| 1407 | // for (auto var_name: thread_scope_->LocalVarNames()) { |
| 1408 | // // for (std::string& var_name : check_nan_var_names_) { |
nothing calls this directly
no test coverage detected