| 887 | } |
| 888 | |
| 889 | void ModelUpdater::WorkLoop() { |
| 890 | int try_count = 0; |
| 891 | while(!is_stop_) { |
| 892 | Version version; |
| 893 | auto status = model_store_->GetLatestVersion(version); |
| 894 | LOG(INFO) << "[Processor] ModelUpdater::WorkLoop get latest version: " |
| 895 | << version.DebugString(); |
| 896 | if (!version.IsValid()) { |
| 897 | try_count++; |
| 898 | LOG(ERROR) << "[Processor] Found a invalid model, " |
| 899 | << "please check other error message, " |
| 900 | << "we will try 60 seconds later. status: " << status.error_message() |
| 901 | << "version debug string: " << version.DebugString(); |
| 902 | if (try_count >= MAX_TRY_COUNT) { |
| 903 | LOG(FATAL) << "Try to get the latest model failed " << try_count << " times, " |
| 904 | << "please check the model directory or network."; |
| 905 | } |
| 906 | } else { |
| 907 | try_count = 0; |
| 908 | if (!status.ok()) { |
| 909 | LOG(WARNING) << "[Processor] Not found full model or incremental model directory. " |
| 910 | << "Please ignore this warning if you confirm it. " |
| 911 | << "And we will try 60 seconds later. Warning message: " |
| 912 | << status.error_message(); |
| 913 | } |
| 914 | |
| 915 | // New model directory is generated or the version step is greater than the pre. |
| 916 | Version pre_version = GetVersion(); |
| 917 | bool new_full_ckpt_generated = version.IsValid() && |
| 918 | (pre_version.full_ckpt_name != version.full_ckpt_name); |
| 919 | if (new_full_ckpt_generated || pre_version < version) { |
| 920 | LOG(INFO) << "Start to load new version model: " << version.DebugString(); |
| 921 | auto status = ModelUpdate(version, model_config_, |
| 922 | new_full_ckpt_generated); |
| 923 | if (!status.ok()) { |
| 924 | LOG(ERROR) << "Load new version model failed: " << status.error_message() |
| 925 | << ", version info: " << version.DebugString(); |
| 926 | } else { |
| 927 | LOG(INFO) << "Load new version model successful: " << version.DebugString(); |
| 928 | } |
| 929 | } |
| 930 | } |
| 931 | |
| 932 | sleep(_60_Seconds); |
| 933 | } |
| 934 | } |
| 935 | |
| 936 | } // processor |
| 937 | } // tensorflow |
nothing calls this directly
no test coverage detected