From fafeebd28111482da24cf7405592f1305a1de4fc Mon Sep 17 00:00:00 2001 From: yuetsh <517252939@qq.com> Date: Wed, 2 Sep 2026 22:58:01 -0600 Subject: [PATCH] =?UTF-8?q?feat(=E8=BF=90=E7=BB=B4):=20=E5=8A=A0=20oj2-api?= =?UTF-8?q?=20recount=EF=BC=8C=E6=8A=8A=E5=8F=8D=E8=8C=83=E5=BC=8F?= =?UTF-8?q?=E8=AE=A1=E6=95=B0=E5=88=97=E5=AF=B9=E5=9B=9E=20submission=20?= =?UTF-8?q?=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit problem.submission_number / accepted_number / statistic_info 和 user_profile 的 submission_number / accepted_number / acm_problems_status 由 judge/run.ts 的 persistResult 在判题时手工加减，上线至今没人事后核对过。 已知的漂移来源是重判：routes/submission.ts 的 rejudge 把 result 打回 PENDING 就重新入队，不回退任何计数，persistResult 随后再加一次。实测重判一条提交， problem 的 submission_number、statistic_info 和 user_profile 的 submission_number 全部虚增，而提交一条没多。 默认只读预演，--apply 才写，落库后用同一份 computePlan 复核，还剩差异就非零退出。 口径逐条照抄 persistResult：题目侧连比赛提交一起算、用户侧只算非比赛； accepted_number 是去重到题的首次通过；acm_problems_status 通过过就恒为 ACCEPTED， 没通过过取最后一次结果。acm_problems_status 里 problems / contest_problems 之外 的顶层键原样保留 —— 来历不明的数据不该被重算顺手抹掉。 不管的：acm_contest_rank（罚时与每题尝试次数口径复杂，单独一件事）、 achievement.unlock_count（0010 之后随成就级联，漂不了）、题单进度与奖章 （走 backfill-problemsets）。 --apply 要挑没人做题的时候跑：差异在事务外算、写的是绝对值，算完到写完之间判完 的那一笔加法会被覆盖；复核会把它报成「仍有 N 处差异」并以 1 退出，不会静默。 验证：dev 库先备份计数列，测完原样还原（差异数 0）。验过收敛（--apply 后再跑报 一致）、真实重判造成的漂移精确报出 3 处且无误报、人为删掉某学生已 AC 的格子能按 「曾经 AC → ACCEPTED」恢复、注入的未知顶层键完好保留。 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AeJoYc2t2d7cThVqMBYrBF --- apps/api/src/main.ts | 8 +- apps/api/src/scripts/recount.ts | 286 ++++++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 apps/api/src/scripts/recount.ts diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index 40eddca..a7e2643 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -13,6 +13,7 @@ * oj2-api sql-child # SQL 判题子进程，由服务自己 spawn，不该手动调 * oj2-api migrate # 执行待办的数据库迁移，部署时由 docker/deploy.sh 调 * oj2-api backfill-problemsets # 把题单进度与奖章订正到与规则一致，默认只读预演 + * oj2-api recount # 把题目/用户的计数列重算回与 submission 一致，默认只读预演 * * 用动态 import 而非顶层 import：这几个模块都有导入即执行的副作用 * （Bun.serve、连 Redis 开消费者），静态导入会让 sql-child 也把整个服务拉起来。 @@ -43,6 +44,11 @@ switch (command) { allowRevoke: args.includes("--allow-revoke"), })) } + // 同上，一次性的数据订正。反范式计数列被重判等操作带偏之后拿它对账。 + case "recount": { + const { recount } = await import("./scripts/recount") + process.exit(await recount({ apply: process.argv.slice(3).includes("--apply") })) + } case "sql-child": { const { runSqlChild } = await import("./judge/sql/child") await runSqlChild() @@ -63,6 +69,6 @@ switch (command) { } } default: - console.error(`未知子命令：${command}\n可用：serve | worker | migrate | backfill-problemsets | healthcheck | sql-child`) + console.error(`未知子命令：${command}\n可用：serve | worker | migrate | backfill-problemsets | recount | healthcheck | sql-child`) process.exit(2) } diff --git a/apps/api/src/scripts/recount.ts b/apps/api/src/scripts/recount.ts new file mode 100644 index 0000000..afaa211 --- /dev/null +++ b/apps/api/src/scripts/recount.ts @@ -0,0 +1,286 @@ +import { eq, sql } from "drizzle-orm" + +import { db, schema } from "../db" +import { JudgeStatus, isAccepted } from "../judge/status" +import { objectValue } from "../routes/helpers" + +/** + * 把反范式的计数列重算回与 submission 表一致。 + * + * 这几个列不是缓存、是真值的副本：判题落库时由 `judge/run.ts` 的 persistResult 手工 + * 加减，谁都没在事后核对过。已知的漂移来源是**重判**——`routes/submission.ts` 的 + * rejudge 把 result 打回 PENDING 就重新入队，**不回退任何计数**，于是 persistResult + * 再加一次：重判一条题目的 submission_number 就永久多一。删提交、直接改库同理。 + * + * 管的六个列： + * problem.submission_number / accepted_number / statistic_info + * user_profile.submission_number / accepted_number / acm_problems_status + * + * **不管**的：acm_contest_rank（比赛榜有自己的一套罚时累计，重算要连带 submission_info + * 里每题的尝试次数，口径复杂，单独一件事）、achievement.unlock_count（0010 之后 + * user_achievement 随成就级联，漂不了）、题单进度与奖章（走 backfill-problemsets）。 + * + * 默认只读，把差异打出来；确认无误再加 --apply 落库。跑法对齐 migrate： + * + * docker compose -f docker/compose.debian.yml run --rm oj-api oj2-api recount + * docker compose -f docker/compose.debian.yml run --rm oj-api oj2-api recount --apply + * + * ⚠️ **--apply 要挑没人做题的时候跑。** 差异是在事务外算的，写的是绝对值：算完到写完 + * 之间要是有一条判完了，它那一笔加法会被覆盖掉。落库后的复核会把这种情况报成「仍有 + * N 处差异」并以 1 退出，不会静默 —— 见到了重跑一次即可，但别在上课高峰按。 + */ + +/** 判完的提交才计数。PENDING / JUDGING 是在途状态，persistResult 还没给它们记过账 */ +const UNJUDGED = [JudgeStatus.PENDING, JudgeStatus.JUDGING] + +type ProblemExpected = { + submissionNumber: number + acceptedNumber: number + statisticInfo: Record +} + +/** + * 题目侧的期望值。**比赛提交也算**——persistResult 更新 problem 这一段没有区分 + * contestId，只有 user_profile 那一段才分。 + */ +async function expectedProblems() { + const rows = await db.execute<{ problem_id: number; result: number; n: number }>(sql` + select problem_id, result, count(*)::int as n + from submission + where result not in (${UNJUDGED[0]}, ${UNJUDGED[1]}) + group by problem_id, result + `) + const expected = new Map() + for (const row of rows) { + const current = expected.get(row.problem_id) ?? { + submissionNumber: 0, + acceptedNumber: 0, + statisticInfo: {}, + } + current.submissionNumber += row.n + if (isAccepted(row.result)) current.acceptedNumber += row.n + current.statisticInfo[String(row.result)] = row.n + expected.set(row.problem_id, current) + } + return expected +} + +type ProfileExpected = { + submissionNumber: number + acceptedNumber: number + status: Record> +} + +/** + * 用户侧的期望值。三条口径都照抄 persistResult： + * + * - submission_number：只数**非比赛**的判完提交。 + * - accepted_number：只数非比赛、**去重到题**的首次通过（`acceptedNow && !wasAccepted` + * 等价于「这道题此前没通过过」，累计下来就是 AC 的不同题目数）。 + * - acm_problems_status：`{ problems / contest_problems: { 题号: { status, _id } } }`。 + * 通过过就恒为 ACCEPTED（persistResult 里 `wasAccepted` 之后不再改写）； + * 从没通过过则取**最后一次**判完的结果。 + * + * ⚠️ 「最后一次」这里按 create_time 排，而 persistResult 是按**判完的先后**写的。 + * 两者在重判乱序时可能不同 —— 一条早提交的被重判、比晚提交的更晚判完，真值是那条早的， + * 本工具会算成那条晚的。这种情况只影响「从没 AC 过的题」显示成哪种失败，不影响任何计数， + * 所以按 create_time 算，不额外记判完时间。 + */ +async function expectedProfiles() { + const totals = await db.execute<{ user_id: number; submissions: number; accepted: number }>(sql` + select user_id, + count(*)::int as submissions, + count(distinct problem_id) filter (where result in (${JudgeStatus.ACCEPTED}, ${JudgeStatus.AST_CHECK_FAILED}))::int as accepted + from submission + where result not in (${UNJUDGED[0]}, ${UNJUDGED[1]}) and contest_id is null + group by user_id + `) + const perProblem = await db.execute<{ + user_id: number + is_public: boolean + problem_id: number + display_id: string + ever_accepted: boolean + last_result: number + }>(sql` + select s.user_id, + (s.contest_id is null) as is_public, + s.problem_id, + p._id as display_id, + bool_or(s.result in (${JudgeStatus.ACCEPTED}, ${JudgeStatus.AST_CHECK_FAILED})) as ever_accepted, + (array_agg(s.result order by s.create_time desc, s.id desc))[1] as last_result + from submission s + join problem p on p.id = s.problem_id + where s.result not in (${UNJUDGED[0]}, ${UNJUDGED[1]}) + group by s.user_id, (s.contest_id is null), s.problem_id, p._id + `) + + const expected = new Map() + const blank = (): ProfileExpected => ({ submissionNumber: 0, acceptedNumber: 0, status: {} }) + for (const row of totals) { + const current = expected.get(row.user_id) ?? blank() + current.submissionNumber = row.submissions + current.acceptedNumber = row.accepted + expected.set(row.user_id, current) + } + for (const row of perProblem) { + const current = expected.get(row.user_id) ?? blank() + const bucket = row.is_public ? "problems" : "contest_problems" + current.status[bucket] ??= {} + current.status[bucket]![String(row.problem_id)] = { + status: row.ever_accepted ? JudgeStatus.ACCEPTED : row.last_result, + _id: row.display_id, + } + expected.set(row.user_id, current) + } + return expected +} + +/** 稳定序列化，用来比对 jsonb —— 键序不同不该被当成差异 */ +function stable(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stable).join(",")}]` + if (value && typeof value === "object") { + const entries = Object.entries(value as Record).sort(([a], [b]) => (a < b ? -1 : 1)) + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stable(v)}`).join(",")}}` + } + return JSON.stringify(value) ?? "null" +} + +type Diff = { label: string; field: string; before: unknown; after: unknown } +type Plan = { + diffs: Diff[] + problemFixes: { id: number; value: ProblemExpected }[] + profileFixes: { id: number; value: ProfileExpected & { merged: Record } }[] +} + +/** 只算差异，不写库。预演和落库后的复核共用它 —— 两边口径必须是同一份代码 */ +async function computePlan(): Promise { + const [problems, profiles, expectedProblem, expectedProfile] = await Promise.all([ + db.select({ + id: schema.problem.id, + displayId: schema.problem.displayId, + submissionNumber: schema.problem.submissionNumber, + acceptedNumber: schema.problem.acceptedNumber, + statisticInfo: schema.problem.statisticInfo, + }).from(schema.problem), + db.select({ + id: schema.userProfile.id, + userId: schema.userProfile.userId, + submissionNumber: schema.userProfile.submissionNumber, + acceptedNumber: schema.userProfile.acceptedNumber, + acmProblemsStatus: schema.userProfile.acmProblemsStatus, + }).from(schema.userProfile), + expectedProblems(), + expectedProfiles(), + ]) + + const plan: Plan = { diffs: [], problemFixes: [], profileFixes: [] } + + for (const problem of problems) { + const want = expectedProblem.get(problem.id) ?? { + submissionNumber: 0, + acceptedNumber: 0, + statisticInfo: {}, + } + const label = `题目 ${problem.displayId}(id=${problem.id})` + const rows: Diff[] = [] + if (problem.submissionNumber !== want.submissionNumber) { + rows.push({ label, field: "submission_number", before: problem.submissionNumber, after: want.submissionNumber }) + } + if (problem.acceptedNumber !== want.acceptedNumber) { + rows.push({ label, field: "accepted_number", before: problem.acceptedNumber, after: want.acceptedNumber }) + } + if (stable(objectValue(problem.statisticInfo)) !== stable(want.statisticInfo)) { + rows.push({ label, field: "statistic_info", before: problem.statisticInfo, after: want.statisticInfo }) + } + if (rows.length) { + plan.diffs.push(...rows) + plan.problemFixes.push({ id: problem.id, value: want }) + } + } + + for (const profile of profiles) { + const want = expectedProfile.get(profile.userId) ?? { + submissionNumber: 0, + acceptedNumber: 0, + status: {}, + } + // acm_problems_status 里除了 problems / contest_problems 之外的键原样保留 —— + // persistResult 只写这两个桶，别的键是从哪来的没人说得清，重算不该顺手抹掉。 + const existing = objectValue(profile.acmProblemsStatus) + const merged: Record = { ...existing } + delete merged.problems + delete merged.contest_problems + for (const [bucket, value] of Object.entries(want.status)) merged[bucket] = value + + const label = `用户 ${profile.userId}` + const rows: Diff[] = [] + if (profile.submissionNumber !== want.submissionNumber) { + rows.push({ label, field: "submission_number", before: profile.submissionNumber, after: want.submissionNumber }) + } + if (profile.acceptedNumber !== want.acceptedNumber) { + rows.push({ label, field: "accepted_number", before: profile.acceptedNumber, after: want.acceptedNumber }) + } + if (stable(existing) !== stable(merged)) { + const keys = new Set([...Object.keys(objectValue(existing.problems)), ...Object.keys(want.status.problems ?? {})]) + rows.push({ label, field: "acm_problems_status", before: `${Object.keys(objectValue(existing.problems)).length} 题`, after: `${keys.size} 题（含比赛桶重建）` }) + } + if (rows.length) { + plan.diffs.push(...rows) + plan.profileFixes.push({ id: profile.id, value: { ...want, merged } }) + } + } + return plan +} + +function report(plan: Plan) { + console.log(`发现 ${plan.diffs.length} 处不一致（题目 ${plan.problemFixes.length} 道 / 用户 ${plan.profileFixes.length} 人）：`) + for (const diff of plan.diffs.slice(0, 40)) { + console.log(` ${diff.label} ${diff.field}: ${JSON.stringify(diff.before)} → ${JSON.stringify(diff.after)}`) + } + if (plan.diffs.length > 40) console.log(` ……另有 ${plan.diffs.length - 40} 处`) +} + +/** 退出码：0 = 一致或预演正常，1 = 落库后复核仍有差异 */ +export async function recount(options: { apply: boolean }) { + const plan = await computePlan() + if (plan.diffs.length === 0) { + console.log("计数列与 submission 表一致，没有要订正的。") + return 0 + } + report(plan) + + if (!options.apply) { + console.log("\n以上为预演，没有写库。确认无误后加 --apply 落库。") + return 0 + } + + await db.transaction(async (tx) => { + for (const fix of plan.problemFixes) { + await tx.update(schema.problem).set({ + submissionNumber: fix.value.submissionNumber, + acceptedNumber: fix.value.acceptedNumber, + statisticInfo: fix.value.statisticInfo, + }).where(eq(schema.problem.id, fix.id)) + } + for (const fix of plan.profileFixes) { + await tx.update(schema.userProfile).set({ + submissionNumber: fix.value.submissionNumber, + acceptedNumber: fix.value.acceptedNumber, + acmProblemsStatus: fix.value.merged, + }).where(eq(schema.userProfile.id, fix.id)) + } + }) + console.log(`\n已订正题目 ${plan.problemFixes.length} 道、用户 ${plan.profileFixes.length} 人，复核中……`) + + // 复核跑的是同一份 computePlan。这里还剩差异说明口径本身有问题（不是数据脏）， + // 必须让部署脚本看见非零退出码，而不是打一行字了事。 + const after = await computePlan() + if (after.diffs.length === 0) { + console.log("复核通过：计数列与 submission 表一致") + return 0 + } + console.error(`复核未通过，仍有 ${after.diffs.length} 处差异：`) + report(after) + return 1 +}