(projects: string[])
| 55 | } |
| 56 | |
| 57 | async function inspectRedis(projects: string[]): Promise<{ |
| 58 | total: number; |
| 59 | totalEligible: number; |
| 60 | totalStuck: number; |
| 61 | totalOrphans: number; |
| 62 | stats: ProjectStat[]; |
| 63 | }> { |
| 64 | const redis = getRedisCache(); |
| 65 | const now = Date.now(); |
| 66 | const eligibleCutoff = now - DEADMAN_MS; |
| 67 | const stuckCutoff = now - STUCK_CUTOFF_MS; |
| 68 | |
| 69 | const stats: ProjectStat[] = []; |
| 70 | let total = 0; |
| 71 | let totalEligible = 0; |
| 72 | let totalStuck = 0; |
| 73 | let totalOrphans = 0; |
| 74 | |
| 75 | for (const project of projects) { |
| 76 | const wallclockKey = `session:wallclock:${project}`; |
| 77 | |
| 78 | const [active, eligible, stuck, oldestEntry] = await Promise.all([ |
| 79 | redis.zcard(wallclockKey), |
| 80 | redis.zcount(wallclockKey, '-inf', eligibleCutoff), |
| 81 | redis.zcount(wallclockKey, '-inf', stuckCutoff), |
| 82 | redis.zrange(wallclockKey, 0, 0, 'WITHSCORES'), |
| 83 | ]); |
| 84 | |
| 85 | const oldestAgeMin = |
| 86 | oldestEntry.length === 2 ? (now - Number(oldestEntry[1])) / 60_000 : 0; |
| 87 | |
| 88 | // Sample N oldest entries and verify their blob exists. |
| 89 | const sampleSize = Math.min(SAMPLE_SIZE_PER_PROJECT, active); |
| 90 | let orphans = 0; |
| 91 | if (sampleSize > 0) { |
| 92 | const sample = await redis.zrange(wallclockKey, 0, sampleSize - 1); |
| 93 | const multi = redis.multi(); |
| 94 | for (const did of sample) { |
| 95 | multi.exists(`session:${project}:${did}`); |
| 96 | } |
| 97 | const results = await multi.exec(); |
| 98 | for (const entry of results ?? []) { |
| 99 | if (Number(entry?.[1] ?? 0) === 0) orphans++; |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | stats.push({ project, active, eligible, stuck, orphans, oldestAgeMin }); |
| 104 | total += active; |
| 105 | totalEligible += eligible; |
| 106 | totalStuck += stuck; |
| 107 | totalOrphans += orphans; |
| 108 | } |
| 109 | |
| 110 | return { total, totalEligible, totalStuck, totalOrphans, stats }; |
| 111 | } |
| 112 | |
| 113 | async function inspectQueue() { |
| 114 | const [waiting, delayed, active, failed, completed] = await Promise.all([ |
no test coverage detected