| 59 | } |
| 60 | |
| 61 | int LockOrWait(int64_t latest_version, |
| 62 | IFeatureStoreMgr* sparse_storage, |
| 63 | Version* real_version, |
| 64 | ModelConfig* config, bool* locked) { |
| 65 | *locked = false; |
| 66 | |
| 67 | // Only one instance can get the distribute lock, |
| 68 | // other instances should wait here, |
| 69 | // and check the model version in the storage. |
| 70 | // Wait here when the lock timeout is not reached. |
| 71 | // If lock timeout, other instances should |
| 72 | // try to get the lock again. |
| 73 | do { |
| 74 | auto lock_value = GetRandomNum(); |
| 75 | Status status = sparse_storage->GetStorageLock( |
| 76 | lock_value, config->lock_timeout, locked); |
| 77 | if (status.ok() && locked) { |
| 78 | return lock_value; |
| 79 | } |
| 80 | *locked = false; |
| 81 | |
| 82 | // WAIT and CHECK version in the storage |
| 83 | int64_t curr_full_version = -1; |
| 84 | int64_t model_version = -1; |
| 85 | int time_cost = 0; |
| 86 | // try util lock timeout |
| 87 | while (time_cost < config->lock_timeout) { |
| 88 | // check model_version from storage like redis. |
| 89 | status = sparse_storage->GetModelVersion( |
| 90 | &curr_full_version, &model_version); |
| 91 | if (!status.ok()) { |
| 92 | LOG(WARNING) << "Check latest version error, will try again."; |
| 93 | } else { |
| 94 | LOG(INFO) << "Waiting for the latest model to be updated. version is " |
| 95 | << latest_version << ", current redis model version is " |
| 96 | << model_version; |
| 97 | if (model_version >= latest_version) { |
| 98 | MaybeUpdateRealVersion(real_version, curr_full_version, |
| 99 | model_version); |
| 100 | return 0; |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | sleep(_30_Seconds); |
| 105 | time_cost += _30_Seconds; |
| 106 | } |
| 107 | |
| 108 | LOG(WARNING) << "Lock timeout, try to grab the lock again."; |
| 109 | // continue: lock timeout, try to get lock again |
| 110 | |
| 111 | } while(true); |
| 112 | |
| 113 | return 0; |
| 114 | } |
| 115 | |
| 116 | void CreateCustomThreadPool(CustomThreadPoolImpl** tp, mutex& mu, |
| 117 | int num, const std::string& name) { |
no test coverage detected