TestJobManagerCleansUpAfterJobPanic verifies that when a submitted job panics, the worker still releases the in-flight name and decrements its active-worker counter. Without these cleanups, a single panic would silently strand all future renewals for that name (and, after enough panics, every name)
(t *testing.T)
| 17 | // panics, every name) until process restart. See certmagic issue for |
| 18 | // caddyserver/caddy#7366. |
| 19 | func TestJobManagerCleansUpAfterJobPanic(t *testing.T) { |
| 20 | // Suppress the worker's "panic: certificate worker: ..." message so it |
| 21 | // doesn't pollute test output. We're intentionally triggering a panic. |
| 22 | stdlog.SetOutput(io.Discard) |
| 23 | t.Cleanup(func() { stdlog.SetOutput(io.Discard) }) |
| 24 | |
| 25 | jm := &jobManager{maxConcurrentJobs: 10} |
| 26 | logger := zap.NewNop() |
| 27 | |
| 28 | jm.Submit(logger, "renewal_X", func() error { |
| 29 | panic("simulated panic from acme library") |
| 30 | }) |
| 31 | |
| 32 | // Cleanup happens in deferred handlers inside worker(), so we cannot |
| 33 | // synchronize on it from inside the job itself. Poll until state settles. |
| 34 | if !waitUntil(time.Second, func() bool { |
| 35 | jm.mu.Lock() |
| 36 | defer jm.mu.Unlock() |
| 37 | _, nameStillTracked := jm.names["renewal_X"] |
| 38 | return !nameStillTracked && jm.activeWorkers == 0 |
| 39 | }) { |
| 40 | jm.mu.Lock() |
| 41 | _, nameStillTracked := jm.names["renewal_X"] |
| 42 | active := jm.activeWorkers |
| 43 | jm.mu.Unlock() |
| 44 | t.Fatalf("worker did not clean up after panic: name still tracked=%v, activeWorkers=%d (want false, 0)", |
| 45 | nameStillTracked, active) |
| 46 | } |
| 47 | |
| 48 | // A subsequent submission with the same name must actually run. |
| 49 | // If the names leak regressed, this Submit would be silently dropped. |
| 50 | var ran int32 |
| 51 | jm.Submit(logger, "renewal_X", func() error { |
| 52 | atomic.StoreInt32(&ran, 1) |
| 53 | return nil |
| 54 | }) |
| 55 | if !waitUntil(time.Second, func() bool { |
| 56 | return atomic.LoadInt32(&ran) == 1 |
| 57 | }) { |
| 58 | t.Fatal("second Submit with the same name was silently dropped after panic") |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | func waitUntil(timeout time.Duration, cond func() bool) bool { |
| 63 | deadline := time.Now().Add(timeout) |