按外部分析报告(reports/analysis)挑出来做的几条: - CI 在编完前端之后跑 `bunx vp run verify`,有一项不过就不部署(必须排在编前端之后: 前端类型检查要的 auto-imports.d.ts / components.d.ts 是 vite build 时才生成的) - 连不上判题机(连接被拒、解析不到、被重置)时,正式判题等 2/5/10 秒重试,不再当场记成 系统错误;超时、判题机回错、页面上的运行和自测不重试 - worker 每 10 秒刷一次「还活着」,compose 加 `oj2-api healthcheck --worker`,deploy.sh 起栈后 也等 worker 健康(docker 不会因 unhealthy 自动重启,只是看得见、部署时拦得住) - 三套 compose 的容器日志限 10MB × 3 - `/public/%` 这类畸形编码回 404 而不是 500;找不到的接口回统一的 JSON 错误 - 删掉没人用的「停用判题机」接口,契约不再给 task_number / is_disabled(列留在库里) - 删掉腐化的 apps/web/tests/submitButtonState.test.ts;契约包 zod 范围对齐到 ^4.6.6 - 修几处和实际对不上的注释与文档(表数、编译并发、AST 语言数、deploy.sh 流程、校验处数) ⚠️ 改了 compose 的日志配置,这次部署会把 postgres / redis / 判题机也重建一遍,各停几秒 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b344bd1dd6
commit
b046af4ba0
20 files changed
+187
-104
No files matched your search
@@ -3,7 +3,7 @@
|
||||
// 2026-08-26:旧 Django 后端下线,7 张框架表(auth_group、auth_group_permissions、
|
||||
// auth_permission、django_content_type、django_dramatiq_task、django_migrations、
|
||||
// django_session)已由 0002_drop_django_leftovers 删除,drizzle.config.ts 的
|
||||
// tablesFilter 随之移除。库里现在就是这 27 张业务表。
|
||||
// tablesFilter 随之移除。当时库里剩 27 张业务表(之后陆续加了新表,以本文件为准)。
|
||||
//
|
||||
// 2026-09-02:又清了一批只有 Django 时代写过、OJ2 一次都没读过的列 ——
|
||||
// 0008 删比赛 IP 白名单与 submission.ip(判题机的 judge_server.ip 保留,那是运维数据),
|
||||
|
||||
+11
-1
@@ -44,6 +44,10 @@ app.route('/api', submissionRoutes)
|
||||
app.route('/api', trialRunRoutes)
|
||||
app.route('/api/admin', adminRoutes)
|
||||
|
||||
// 没有这条的话,打错的接口地址回的是 Hono 默认的纯文本「404 Not Found」,前端 utils/api.ts 读不出
|
||||
// error.code / message,提示就成了笼统的「请求失败」。和其它错误同一个形状
|
||||
app.notFound((c) => c.json({ error: { code: 'not-found', message: 'Not found' } }, 404))
|
||||
|
||||
app.onError((error, c) => {
|
||||
console.error(error)
|
||||
return c.json({ error: { code: 'internal-error', message: 'Internal server error' } }, 500)
|
||||
@@ -63,7 +67,13 @@ const DEFAULT_AVATAR_SVG =
|
||||
* 这样开发(Vite 代理)和生产是同一条代码路径,少一处只在服务器上才出错的差异。
|
||||
*/
|
||||
async function serveUpload(pathname: string, prefix: string, directory: string) {
|
||||
const decoded = decodeURIComponent(pathname)
|
||||
// 畸形的百分号编码(`/public/upload/%`)decodeURIComponent 会抛 URIError,不接住就是 500
|
||||
let decoded: string
|
||||
try {
|
||||
decoded = decodeURIComponent(pathname)
|
||||
} catch {
|
||||
return new Response('Not found', { status: 404 })
|
||||
}
|
||||
const filename = basename(decoded)
|
||||
if (!filename || filename !== decoded.slice(prefix.length + 1)) {
|
||||
return new Response('Not found', { status: 404 })
|
||||
|
||||
+23
-11
@@ -16,7 +16,7 @@ import { parseProblemTemplate, shiftCompileLines } from './template'
|
||||
import { withJudgeSlot } from './slots'
|
||||
import { runSqlCase } from './sql'
|
||||
import { nativeRuntimeError, parsePythonTraceback, type RuntimeErrorInfo } from './runtime-diagnosis'
|
||||
import { compileError, judge, type JudgeCase, type JudgeResponse } from './sandbox'
|
||||
import { compileError, judge, JudgeUnreachableError, type JudgeCase, type JudgeResponse } from './sandbox'
|
||||
import { readInfo } from '../services/test-case'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { resolve as resolvePath } from 'node:path'
|
||||
@@ -79,6 +79,13 @@ function judgeTimeoutMs(timeLimit: number) {
|
||||
return Math.max(120_000, timeLimit * 60)
|
||||
}
|
||||
|
||||
/**
|
||||
* 连不上判题机时隔多久再试,一共再试这么多次。判题机容器重启(部署时重建、OOM 被拉起来)
|
||||
* 要几秒,合计十几秒盖得住;原来是当场记成系统错误,那几秒里交的全班都得自己重交。
|
||||
* 只重试「连不上」:超时(已经等了两分钟)、判题机回了错都不重试。等的时候不占名额
|
||||
*/
|
||||
const UNREACHABLE_RETRY_DELAYS_MS = [2_000, 5_000, 10_000]
|
||||
|
||||
async function requestJudge(
|
||||
language: string,
|
||||
code: string,
|
||||
@@ -88,17 +95,22 @@ async function requestJudge(
|
||||
output = false,
|
||||
) {
|
||||
const timeoutMs = judgeTimeoutMs(timeLimit)
|
||||
try {
|
||||
// 超时从拿到名额才开始算:排队等名额不算判题机没回应
|
||||
return await withJudgeSlot('judge', { holdMs: timeoutMs + 30_000 }, () =>
|
||||
judge(language, code, timeLimit, memoryLimit, testCase, { output, signal: AbortSignal.timeout(timeoutMs) }),
|
||||
)
|
||||
} catch (error) {
|
||||
// 这句会写进 err_info,学生能看到,别把 "The operation timed out." 原样给他
|
||||
if (error instanceof DOMException && error.name === 'TimeoutError') {
|
||||
throw new Error(`判题机 ${timeoutMs / 1000} 秒没有回应`)
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
// 超时从拿到名额才开始算:排队等名额不算判题机没回应
|
||||
return await withJudgeSlot('judge', { holdMs: timeoutMs + 30_000 }, () =>
|
||||
judge(language, code, timeLimit, memoryLimit, testCase, { output, signal: AbortSignal.timeout(timeoutMs) }),
|
||||
)
|
||||
} catch (error) {
|
||||
// 这句会写进 err_info,学生能看到,别把 "The operation timed out." 原样给他
|
||||
if (error instanceof DOMException && error.name === 'TimeoutError') {
|
||||
throw new Error(`判题机 ${timeoutMs / 1000} 秒没有回应`)
|
||||
}
|
||||
const delay = UNREACHABLE_RETRY_DELAYS_MS[attempt]
|
||||
if (!(error instanceof JudgeUnreachableError) || delay === undefined) throw error
|
||||
console.warn(`[judge] ${error.message},${delay / 1000} 秒后重试(第 ${attempt + 1} 次)`)
|
||||
await Bun.sleep(delay)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -106,8 +106,30 @@ function headers() {
|
||||
return { authorization: `Bearer ${config.judgeServerToken}` }
|
||||
}
|
||||
|
||||
/**
|
||||
* 判题机根本连不上:连接被拒、主机名解析不到、连接被重置 —— 判题机容器正在重启、OOM 被杀
|
||||
* 的那几秒就是这样。和「判题机回了错」「超时没回」分开,正式判题碰到它会等几秒再试
|
||||
* (judge/run.ts 的 requestJudge),而不是当场把学生的提交记成系统错误
|
||||
*/
|
||||
export class JudgeUnreachableError extends Error {
|
||||
constructor(cause: unknown) {
|
||||
super(`连不上判题机(${cause instanceof Error ? cause.message : String(cause)})`)
|
||||
this.name = 'JudgeUnreachableError'
|
||||
}
|
||||
}
|
||||
|
||||
/** fetch 判题机。网络层失败(Bun 里是带 code 的 TypeError)换成 JudgeUnreachableError;超时、中止原样抛 */
|
||||
async function judgeFetch(path: string, init: RequestInit & { signal?: AbortSignal }) {
|
||||
try {
|
||||
return await fetch(new URL(path, config.judgeServerUrl), init)
|
||||
} catch (error) {
|
||||
if (!init.signal?.aborted && error instanceof TypeError) throw new JudgeUnreachableError(error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function runCommand(command: Record<string, unknown>, signal?: AbortSignal) {
|
||||
const response = await fetch(new URL('/run', config.judgeServerUrl), {
|
||||
const response = await judgeFetch('/run', {
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: { ...headers(), 'content-type': 'application/json' },
|
||||
@@ -120,7 +142,7 @@ async function runCommand(command: Record<string, unknown>, signal?: AbortSignal
|
||||
}
|
||||
|
||||
async function readFile(fileId: string, signal?: AbortSignal) {
|
||||
const response = await fetch(new URL(`/file/${fileId}`, config.judgeServerUrl), { signal, headers: headers() })
|
||||
const response = await judgeFetch(`/file/${fileId}`, { signal, headers: headers() })
|
||||
if (!response.ok) throw new Error(`判题机取文件返回 HTTP ${response.status}`)
|
||||
return new Uint8Array(await response.arrayBuffer())
|
||||
}
|
||||
|
||||
+12
-1
@@ -9,7 +9,7 @@
|
||||
* oj2-api # 等同 serve
|
||||
* oj2-api serve # HTTP + WebSocket
|
||||
* oj2-api worker # BullMQ 判题消费者
|
||||
* oj2-api healthcheck # 探活,给 Dockerfile 的 HEALTHCHECK 用
|
||||
* oj2-api healthcheck # 探活,给 compose 的 healthcheck 用;--worker 探判题 worker
|
||||
* oj2-api sql-child # SQL 判题子进程,由服务自己 spawn,不该手动调
|
||||
* oj2-api migrate # 执行待办的数据库迁移,部署时由 docker/deploy.sh 调
|
||||
* oj2-api recount # 把题目/用户的计数列、成就的已解锁数重算回与明细一致,默认只读预演
|
||||
@@ -54,7 +54,18 @@ switch (command) {
|
||||
// 运行镜像是 debian-slim,没有 curl/wget,探活让二进制自己做。
|
||||
// 只打 /health,不碰库 —— 库挂了该由库自己的 healthcheck 报,
|
||||
// 不该让 api 容器跟着被判成不健康、进而被重启。
|
||||
//
|
||||
// `--worker`:worker 没有 HTTP,看它每 10 秒刷一次的时间戳(worker.ts),60 秒没刷就算挂了
|
||||
case 'healthcheck': {
|
||||
if (process.argv.includes('--worker')) {
|
||||
const { readFileSync } = await import('node:fs')
|
||||
try {
|
||||
const at = Number(readFileSync('/tmp/oj2-worker-alive', 'utf8'))
|
||||
process.exit(Date.now() - at < 60_000 ? 0 : 1)
|
||||
} catch {
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
const { config } = await import('./config')
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${config.port}/health`, {
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import {
|
||||
updateJudgeServerRequestSchema,
|
||||
updateWebsiteConfigRequestSchema,
|
||||
type DashboardInfo,
|
||||
type JudgeServer,
|
||||
@@ -28,7 +27,7 @@ import { TEACHER_ROLES } from '../helpers'
|
||||
import { onlineUserIds } from '../../auth/presence'
|
||||
import { JudgeStatus } from '../../judge/status'
|
||||
import { suggestActiveClass } from '../../services/class-activity'
|
||||
import { classPrefixCondition, queryInteger } from '../helpers'
|
||||
import { classPrefixCondition } from '../helpers'
|
||||
|
||||
export const adminConfRoutes = new Hono<AppEnv>()
|
||||
|
||||
@@ -98,8 +97,10 @@ adminConfRoutes.get('/judge-servers', requireSuperAdmin, async (c) => {
|
||||
return success(c, {
|
||||
// 后台要显示 token 才能拿去配判题机。这个接口是超管专属的
|
||||
token: config.judgeServerToken,
|
||||
// task_number、is_disabled 两列是旧 JudgeServer 时代的:判题不经调度、也没有「停用」这回事,
|
||||
// 列留在库里(删列是破坏性迁移,不值得),不往外给
|
||||
servers: rows.map(
|
||||
(row) =>
|
||||
({ taskNumber: _taskNumber, isDisabled: _isDisabled, ...row }) =>
|
||||
({
|
||||
...row,
|
||||
status: isAlive(row.lastHeartbeat) ? 'normal' : 'abnormal',
|
||||
@@ -108,21 +109,6 @@ adminConfRoutes.get('/judge-servers', requireSuperAdmin, async (c) => {
|
||||
} satisfies JudgeServerList)
|
||||
})
|
||||
|
||||
adminConfRoutes.put('/judge-servers/:id', requireSuperAdmin, async (c) => {
|
||||
const parsed = await parseBody(c, updateJudgeServerRequestSchema, 'isDisabled is required')
|
||||
if (!parsed.success) return parsed.response
|
||||
const updated = await db
|
||||
.update(schema.judgeServer)
|
||||
.set({ isDisabled: parsed.data.isDisabled })
|
||||
.where(eq(schema.judgeServer.id, queryInteger(c.req.param('id'), 0, { min: 1 })))
|
||||
.returning({ id: schema.judgeServer.id })
|
||||
if (updated.length === 0) return failure(c, 404, 'judge-server-not-found', 'Judge server does not exist')
|
||||
// 旧后端在这里会 process_pending_task() 把积压的待判任务重新分发。
|
||||
// 新架构不需要:任务在 BullMQ 里排着,worker 恢复就自己接着消费,不存在「没有新提交
|
||||
// 就一直 waiting」那种情况 —— 那是旧的自研分发器才有的问题。
|
||||
return success(c, null)
|
||||
})
|
||||
|
||||
adminConfRoutes.delete('/judge-servers/:hostname', requireSuperAdmin, async (c) => {
|
||||
const deleted = await db
|
||||
.delete(schema.judgeServer)
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
* 不是合法 id 而返回 404 或者一堆看不懂的结果。阶段 4 真实发生过一次:
|
||||
* 两个教师用的分析端点被 `/problems/:id` 吃掉,评审时才发现。
|
||||
*
|
||||
* 加路由时顺手跑一下,比事后靠人眼在 200 多条路由里看出顺序问题可靠。
|
||||
* 加路由时顺手跑一下,比事后靠人眼在两百来条路由里看出顺序问题可靠。
|
||||
*
|
||||
* 局限:靠正则读源码,只认 `xxxRoutes.get('字面量', …)` 这种写法(单双引号都认),
|
||||
* 以及 `xxxRoutes.route('字面量', 子路由)` 的嵌套挂载(按挂载位置展开)。
|
||||
|
||||
+27
-2
@@ -1,4 +1,5 @@
|
||||
import { Worker } from 'bullmq'
|
||||
import { writeFileSync } from 'node:fs'
|
||||
|
||||
import { config } from './config'
|
||||
import { startJudgeHeartbeat } from './judge/heartbeat'
|
||||
@@ -8,8 +9,11 @@ import { flowchartQueueName, type FlowchartJobData } from './flowchart/job'
|
||||
import { evaluateFlowchart } from './flowchart/run'
|
||||
import { createBlockingRedis } from './redis'
|
||||
|
||||
const judgeConnection = createBlockingRedis()
|
||||
const flowchartConnection = createBlockingRedis()
|
||||
|
||||
const worker = new Worker<JudgeJobData>(judgeQueueName, async (job) => judgeSubmission(job.data), {
|
||||
connection: createBlockingRedis(),
|
||||
connection: judgeConnection,
|
||||
concurrency: config.judgeConcurrency,
|
||||
})
|
||||
|
||||
@@ -21,7 +25,7 @@ const flowchartWorker = new Worker<FlowchartJobData>(
|
||||
evaluateFlowchart(job.data, {
|
||||
isFinalAttempt: job.attemptsMade + 1 >= (job.opts.attempts ?? 1),
|
||||
}),
|
||||
{ connection: createBlockingRedis(), concurrency: 2 },
|
||||
{ connection: flowchartConnection, concurrency: 2 },
|
||||
)
|
||||
|
||||
worker.on('ready', () => {
|
||||
@@ -48,7 +52,28 @@ flowchartWorker.on('error', (error) => console.error('Flowchart worker error', e
|
||||
|
||||
const stopHeartbeat = startJudgeHeartbeat()
|
||||
|
||||
/**
|
||||
* 给 `oj2-api healthcheck --worker` 看的「还活着」:两个消费者都在跑、连 Redis 的连接都正常,
|
||||
* 才每 10 秒刷一次时间戳。事件循环卡死、和 Redis 断了、消费者停了,时间戳都会停下来,
|
||||
* 容器就被判成 unhealthy —— 原来 worker 没有健康检查,挂死了 docker 照样显示 running,
|
||||
* 判题队列静默积压
|
||||
*/
|
||||
// main.ts 的 healthcheck 写死了同一个路径:那边不能 import 这个文件(一 import 就把 worker 起来了)
|
||||
const WORKER_ALIVE_FILE = '/tmp/oj2-worker-alive'
|
||||
const markAlive = () => {
|
||||
if (!worker.isRunning() || !flowchartWorker.isRunning()) return
|
||||
if (judgeConnection.status !== 'ready' || flowchartConnection.status !== 'ready') return
|
||||
try {
|
||||
writeFileSync(WORKER_ALIVE_FILE, String(Date.now()))
|
||||
} catch (error) {
|
||||
console.error('Failed to write worker alive file', error)
|
||||
}
|
||||
}
|
||||
const aliveTimer = setInterval(markAlive, 10_000)
|
||||
worker.on('ready', markAlive)
|
||||
|
||||
async function shutdown() {
|
||||
clearInterval(aliveTimer)
|
||||
stopHeartbeat()
|
||||
await worker.close()
|
||||
await flowchartWorker.close()
|
||||
|
||||
Reference in new issue
Block a user