| 5163 | |
| 5164 | #[instrument] |
| 5165 | pub async fn load_remote_system_parameters( |
| 5166 | storage: &mut Box<dyn OpenableDurableCatalogState>, |
| 5167 | system_parameter_sync_config: Option<SystemParameterSyncConfig>, |
| 5168 | system_parameter_sync_timeout: Duration, |
| 5169 | ) -> Result<Option<BTreeMap<String, String>>, AdapterError> { |
| 5170 | if let Some(system_parameter_sync_config) = system_parameter_sync_config { |
| 5171 | tracing::info!("parameter sync on boot: start sync"); |
| 5172 | |
| 5173 | // We intentionally block initial startup, potentially forever, |
| 5174 | // on initializing LaunchDarkly. This may seem scary, but the |
| 5175 | // alternative is even scarier. Over time, we expect that the |
| 5176 | // compiled-in default values for the system parameters will |
| 5177 | // drift substantially from the defaults configured in |
| 5178 | // LaunchDarkly, to the point that starting an environment |
| 5179 | // without loading the latest values from LaunchDarkly will |
| 5180 | // result in running an untested configuration. |
| 5181 | // |
| 5182 | // Note this only applies during initial startup. Restarting |
| 5183 | // after we've synced once only blocks for a maximum of |
| 5184 | // `FRONTEND_SYNC_TIMEOUT` on LaunchDarkly, as it seems |
| 5185 | // reasonable to assume that the last-synced configuration was |
| 5186 | // valid enough. |
| 5187 | // |
| 5188 | // This philosophy appears to provide a good balance between not |
| 5189 | // running untested configurations in production while also not |
| 5190 | // making LaunchDarkly a "tier 1" dependency for existing |
| 5191 | // environments. |
| 5192 | // |
| 5193 | // If this proves to be an issue, we could seek to address the |
| 5194 | // configuration drift in a different way--for example, by |
| 5195 | // writing a script that runs in CI nightly and checks for |
| 5196 | // deviation between the compiled Rust code and LaunchDarkly. |
| 5197 | // |
| 5198 | // If it is absolutely necessary to bring up a new environment |
| 5199 | // while LaunchDarkly is down, the following manual mitigation |
| 5200 | // can be performed: |
| 5201 | // |
| 5202 | // 1. Edit the environmentd startup parameters to omit the |
| 5203 | // LaunchDarkly configuration. |
| 5204 | // 2. Boot environmentd. |
| 5205 | // 3. Use the catalog-debug tool to run `edit config "{\"key\":\"system_config_synced\"}" "{\"value\": 1}"`. |
| 5206 | // 4. Adjust any other parameters as necessary to avoid |
| 5207 | // running a nonstandard configuration in production. |
| 5208 | // 5. Edit the environmentd startup parameters to restore the |
| 5209 | // LaunchDarkly configuration, for when LaunchDarkly comes |
| 5210 | // back online. |
| 5211 | // 6. Reboot environmentd. |
| 5212 | let mut params = SynchronizedParameters::new(SystemVars::default()); |
| 5213 | let frontend_sync = async { |
| 5214 | let frontend = SystemParameterFrontend::from(&system_parameter_sync_config).await?; |
| 5215 | frontend.pull(&mut params); |
| 5216 | let ops = params |
| 5217 | .modified() |
| 5218 | .into_iter() |
| 5219 | .map(|param| { |
| 5220 | let name = param.name; |
| 5221 | let value = param.value; |
| 5222 | tracing::info!(name, value, initial = true, "sync parameter"); |