(ctx context.Context, session *chaos_mesh.FaultSession)
| 140 | } |
| 141 | |
| 142 | func waitForInjectionCompleted(ctx context.Context, session *chaos_mesh.FaultSession) error { |
| 143 | // First, wait 10 seconds to allow chaos-mesh to inject into the cluster. |
| 144 | // If injection isn't complete after 10 seconds, something is wrong and we should terminate. |
| 145 | timeoutAt := time.Now().Add(time.Second * 10) |
| 146 | |
| 147 | targetingGracePeriod := time.Now().Add(time.Second * 5) |
| 148 | |
| 149 | for { |
| 150 | if time.Now().After(timeoutAt) { |
| 151 | errmsg := "chaos-mesh is still in a 'starting' state after 10 seconds. Check kubernetes events to see what's wrong." |
| 152 | return stacktrace.NewError(errmsg) |
| 153 | } |
| 154 | |
| 155 | status, err := session.GetStatus(ctx) |
| 156 | if err != nil { |
| 157 | time.Sleep(250 * time.Millisecond) |
| 158 | continue |
| 159 | } |
| 160 | |
| 161 | switch status { |
| 162 | case chaos_mesh.InProgress: |
| 163 | log.Info("Fault injected successfully") |
| 164 | return nil |
| 165 | case chaos_mesh.Stopping: |
| 166 | log.Warn("Fault changed to 'stopping' state immediately after injection. May indicate something is wrong.") |
| 167 | return nil |
| 168 | case chaos_mesh.Starting: |
| 169 | if !session.TargetSelectionCompleted { |
| 170 | if time.Now().After(targetingGracePeriod) { |
| 171 | errmsg := "chaos-mesh was unable to identify any pods for injection based on the configured criteria" |
| 172 | return stacktrace.NewError(errmsg) |
| 173 | } |
| 174 | } |
| 175 | case chaos_mesh.Error: |
| 176 | errmsg := "there was an unspecified error returned by chaos-mesh. inspect the fault resource" |
| 177 | return stacktrace.NewError(errmsg) |
| 178 | case chaos_mesh.Completed: |
| 179 | // occurs for faults that perform an action immediately then terminate. (killing pods, etc) |
| 180 | log.Info("Fault injected successfully") |
| 181 | return nil |
| 182 | default: |
| 183 | return stacktrace.NewError("unknown chaos session state %s", status) |
| 184 | } |
| 185 | time.Sleep(250 * time.Millisecond) |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | func waitForFaultRecovery(ctx context.Context, session *chaos_mesh.FaultSession) error { |
| 190 | for { |
no test coverage detected