diff --git a/apps/api/src/judge/run.ts b/apps/api/src/judge/run.ts index 2897ef1..62a974e 100644 --- a/apps/api/src/judge/run.ts +++ b/apps/api/src/judge/run.ts @@ -325,6 +325,31 @@ async function markSystemError(submissionId: string, userId: number, error: unkn } } +/** + * 判题任务在 `judgeSubmission` 之外失败时的兜底。 + * + * 正常路径上的异常都被 judgeSubmission 自己的 try/catch 接住、落成 SYSTEM_ERROR + * 并且推给前端,所以能走到队列 `failed` 事件的只剩两种:取提交那一步就炸了, + * 以及**worker 进程中途死掉** —— 机房断电、容器 OOM 被杀、部署时重启。后一种 + * BullMQ 会先按 stalled 重入队一次,再没人接就彻底放手;判题队列又没配 attempts, + * 失败即终局。没有这个兜底,那条提交就永远停在「等待评分」,学生看着转圈, + * 教师统计里它还占着一个「判题中」的名额。生产库里 3 条卡死的 PENDING + * (2022-11 / 2026-03 / 2026-04,都是旧栈时代留下的)就是这么来的。 + * + * `markSystemError` 只动 PENDING / JUDGING 两个状态,所以判完了的、被重判改过的 + * 都不会被它覆盖。唯一能撞上的是「重判刚把状态置回 PENDING,同一刻上一个被遗弃的 + * 任务才失败」——结果是这次重判被吃掉、显示成系统错误,比静默卡死看得见。 + */ +export async function failAbandonedSubmission(submissionId: string, error: unknown) { + const [row] = await db + .select({ userId: schema.submission.userId }) + .from(schema.submission) + .where(eq(schema.submission.id, submissionId)) + .limit(1) + if (!row) return + await markSystemError(submissionId, row.userId, error) +} + export async function judgeSubmission(job: JudgeJobData) { const [row] = await db .select({ diff --git a/apps/api/src/worker.ts b/apps/api/src/worker.ts index 5592b0c..c9ebe33 100644 --- a/apps/api/src/worker.ts +++ b/apps/api/src/worker.ts @@ -2,7 +2,7 @@ import { Worker } from "bullmq" import { config } from "./config" import { judgeQueueName, type JudgeJobData } from "./judge/job" -import { judgeSubmission } from "./judge/run" +import { failAbandonedSubmission, judgeSubmission } from "./judge/run" import { flowchartQueueName, type FlowchartJobData } from "./flowchart/job" import { evaluateFlowchart } from "./flowchart/run" import { createBlockingRedis } from "./redis" @@ -29,8 +29,17 @@ const flowchartWorker = new Worker( worker.on("ready", () => { console.log(`Judge worker ready (concurrency=${config.judgeConcurrency})`) }) -worker.on("failed", (job, error) => { +worker.on("failed", async (job, error) => { console.error(`Judge job ${job?.id ?? "unknown"} failed`, error) + // 队列没配 attempts,失败即终局;worker 被杀掉那种 BullMQ 走完 stalled 重试也会 + // 落到这里。不在这里写一个终态,提交就永远停在「等待评分」,没有任何人会再管它。 + const submissionId = job?.data.submissionId + if (!submissionId) return + try { + await failAbandonedSubmission(submissionId, error) + } catch (markError) { + console.error(`Failed to mark submission ${submissionId} as system error`, markError) + } }) worker.on("error", (error) => { console.error("Judge worker error", error)