perf: 提交列表等查询改走覆盖索引,避免读取 html/css/js 大行
SQLite 把 html/css/js 存在行内,而 user_id、task_id、flag 等列排在它们后面, 读这些列也要先读完整段代码;线上库也没有统计信息,规划器常挑错索引。 - 新增覆盖索引 submission_submission_slim - 提交列表、随机打分先在索引上取 rowid,再只读这一页 / 这一行的完整内容 - 列表的 flag / zone 筛选用一元 + 避开单列索引 - 打开数据库时执行 PRAGMA optimize,之后每小时一次 线上备份实测:提交列表 844 ms → 6 ms,打分后重算分数 11.5 ms → 0.2 ms。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4b8109caaa
commit
e6a3da14b0
3 files changed
+33
-13
No files matched your search
@@ -25,6 +25,9 @@ CREATE INDEX IF NOT EXISTS "submission_submission_task_id_91e9556b" ON "submissi
|
|||||||
CREATE INDEX IF NOT EXISTS "submission_submission_user_id_71741691" ON "submission_submission" ("user_id");
|
CREATE INDEX IF NOT EXISTS "submission_submission_user_id_71741691" ON "submission_submission" ("user_id");
|
||||||
CREATE INDEX IF NOT EXISTS "submission_submission_flag_d96bc4aa" ON "submission_submission" ("flag");
|
CREATE INDEX IF NOT EXISTS "submission_submission_flag_d96bc4aa" ON "submission_submission" ("flag");
|
||||||
CREATE INDEX IF NOT EXISTS "submission_submission_zone_a1bbf0a2" ON "submission_submission" ("zone");
|
CREATE INDEX IF NOT EXISTS "submission_submission_zone_a1bbf0a2" ON "submission_submission" ("zone");
|
||||||
|
-- 非 Django 原有:除 html/css/js 外的常用列。user_id 等列在表里排在 html/css/js 后面,
|
||||||
|
-- 直接读表要先读完整段代码,所以列表、打分、统计等扫描都靠这个覆盖索引
|
||||||
|
CREATE INDEX IF NOT EXISTS "submission_submission_slim" ON "submission_submission" ("user_id", "task_id", "created", "score", "flag", "zone", "raw_score", "id");
|
||||||
CREATE TABLE IF NOT EXISTS "submission_rating" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "score" integer NOT NULL, "created" datetime NOT NULL, "submission_id" char(32) NOT NULL REFERENCES "submission_submission" ("id") DEFERRABLE INITIALLY DEFERRED, "user_id" bigint NOT NULL REFERENCES "account_user" ("id") DEFERRABLE INITIALLY DEFERRED);
|
CREATE TABLE IF NOT EXISTS "submission_rating" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "score" integer NOT NULL, "created" datetime NOT NULL, "submission_id" char(32) NOT NULL REFERENCES "submission_submission" ("id") DEFERRABLE INITIALLY DEFERRED, "user_id" bigint NOT NULL REFERENCES "account_user" ("id") DEFERRABLE INITIALLY DEFERRED);
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS "submission_rating_user_id_submission_id_29be4c39_uniq" ON "submission_rating" ("user_id", "submission_id");
|
CREATE UNIQUE INDEX IF NOT EXISTS "submission_rating_user_id_submission_id_29be4c39_uniq" ON "submission_rating" ("user_id", "submission_id");
|
||||||
CREATE INDEX IF NOT EXISTS "submission_rating_submission_id_a20dcece" ON "submission_rating" ("submission_id");
|
CREATE INDEX IF NOT EXISTS "submission_rating_submission_id_a20dcece" ON "submission_rating" ("submission_id");
|
||||||
@@ -57,6 +60,8 @@ export function openDb(path: string): Database {
|
|||||||
db.run("PRAGMA busy_timeout = 5000")
|
db.run("PRAGMA busy_timeout = 5000")
|
||||||
db.run("PRAGMA foreign_keys = ON")
|
db.run("PRAGMA foreign_keys = ON")
|
||||||
db.run(SCHEMA)
|
db.run(SCHEMA)
|
||||||
|
// SQLite 不会自动 ANALYZE,没有统计信息时规划器常挑错索引;0x10002 = 打开时只分析必要的表
|
||||||
|
db.run("PRAGMA optimize=0x10002")
|
||||||
return db
|
return db
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { app } from "./app"
|
import { app } from "./app"
|
||||||
import { config } from "./config"
|
import { config } from "./config"
|
||||||
|
import { db } from "./db"
|
||||||
import { upgrade, websocket, type WsData } from "./ws"
|
import { upgrade, websocket, type WsData } from "./ws"
|
||||||
|
|
||||||
const server = Bun.serve<WsData>({
|
const server = Bun.serve<WsData>({
|
||||||
@@ -16,4 +17,7 @@ const server = Bun.serve<WsData>({
|
|||||||
websocket,
|
websocket,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// 长期运行的连接按 SQLite 文档建议定期刷新统计信息
|
||||||
|
setInterval(() => db.run("PRAGMA optimize"), 60 * 60 * 1000)
|
||||||
|
|
||||||
console.log(`webpreview server listening on http://localhost:${server.port}`)
|
console.log(`webpreview server listening on http://localhost:${server.port}`)
|
||||||
@@ -234,13 +234,14 @@ router.get("/", (c) => {
|
|||||||
where.push("s.user_id = $user_id")
|
where.push("s.user_id = $user_id")
|
||||||
params.user_id = f.user_id
|
params.user_id = f.user_id
|
||||||
}
|
}
|
||||||
if (f.flag === "any") where.push("s.flag IS NOT NULL")
|
// flag / zone 前的一元 + 让 SQLite 不用这两列的单列索引(那样要回表读大行),改走 slim 覆盖索引
|
||||||
|
if (f.flag === "any") where.push("+s.flag IS NOT NULL")
|
||||||
else if (f.flag) {
|
else if (f.flag) {
|
||||||
where.push("s.flag = $flag")
|
where.push("+s.flag = $flag")
|
||||||
params.flag = f.flag
|
params.flag = f.flag
|
||||||
}
|
}
|
||||||
if (f.zone) {
|
if (f.zone) {
|
||||||
where.push("s.zone = $zone")
|
where.push("+s.zone = $zone")
|
||||||
params.zone = f.zone
|
params.zone = f.zone
|
||||||
}
|
}
|
||||||
if (f.score_lt_threshold !== undefined) {
|
if (f.score_lt_threshold !== undefined) {
|
||||||
@@ -258,7 +259,8 @@ router.get("/", (c) => {
|
|||||||
}
|
}
|
||||||
if (f.grouped) {
|
if (f.grouped) {
|
||||||
// 分组模式:每个 (user, task) 只保留最新一条
|
// 分组模式:每个 (user, task) 只保留最新一条
|
||||||
where.push(`s.id = (SELECT s3.id FROM submission_submission s3
|
// 用 rowid 而不是 id,这样子查询能完全走 submission_submission_slim 覆盖索引
|
||||||
|
where.push(`s.rowid = (SELECT s3.rowid FROM submission_submission s3
|
||||||
WHERE s3.user_id = s.user_id AND s3.task_id = s.task_id ORDER BY s3.created DESC LIMIT 1)`)
|
WHERE s3.user_id = s.user_id AND s3.task_id = s.task_id ORDER BY s3.created DESC LIMIT 1)`)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -271,12 +273,16 @@ router.get("/", (c) => {
|
|||||||
const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : ""
|
const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : ""
|
||||||
const { limit, offset } = pageWindow(f)
|
const { limit, offset } = pageWindow(f)
|
||||||
|
|
||||||
|
// 先只在索引上排序分页拿到这一页的 rowid,再读这几行的完整内容;
|
||||||
|
// 否则排序时要把每条提交的 html/css/js 大行都读一遍
|
||||||
const items = all(
|
const items = all(
|
||||||
`${SUBMISSION_SELECT},
|
`${SUBMISSION_SELECT},
|
||||||
(SELECT COUNT(*) FROM submission_submission s2
|
(SELECT COUNT(*) FROM submission_submission s2
|
||||||
WHERE s2.user_id = s.user_id AND s2.task_id = s.task_id) AS submit_count
|
WHERE s2.user_id = s.user_id AND s2.task_id = s.task_id) AS submit_count
|
||||||
${SUBMISSION_FROM} ${whereSql}
|
${SUBMISSION_FROM}
|
||||||
ORDER BY ${orderBy} LIMIT $limit OFFSET $offset`,
|
WHERE s.rowid IN (SELECT s.rowid ${SUBMISSION_FROM} ${whereSql}
|
||||||
|
ORDER BY ${orderBy} LIMIT $limit OFFSET $offset)
|
||||||
|
ORDER BY ${orderBy}`,
|
||||||
{ ...params, me: user.id, limit, offset },
|
{ ...params, me: user.id, limit, offset },
|
||||||
)
|
)
|
||||||
const count = get<{ n: number }>(`SELECT COUNT(*) AS n ${SUBMISSION_FROM} ${whereSql}`, params)!.n
|
const count = get<{ n: number }>(`SELECT COUNT(*) AS n ${SUBMISSION_FROM} ${whereSql}`, params)!.n
|
||||||
@@ -305,20 +311,25 @@ router.get("/random-for-rating", (c) => {
|
|||||||
}
|
}
|
||||||
if (user.role === Role.NORMAL && todayRatingCount(user.id) >= 30) return c.json(null)
|
if (user.role === Role.NORMAL && todayRatingCount(user.id) >= 30) return c.json(null)
|
||||||
|
|
||||||
const base = `SELECT s.id, u.username, t.title AS task_title, t.display AS task_display,
|
// 只在索引上随机挑出 rowid,再读这一行的代码;否则每个候选都要读一遍 html/css/js
|
||||||
t.task_type, s.html, s.css, s.js
|
const base = `SELECT s.rowid AS rid FROM submission_submission s
|
||||||
${SUBMISSION_FROM}
|
|
||||||
WHERE s.user_id != $me
|
WHERE s.user_id != $me
|
||||||
AND NOT EXISTS (SELECT 1 FROM submission_rating r WHERE r.submission_id = s.id AND r.user_id = $me)
|
AND NOT EXISTS (SELECT 1 FROM submission_rating r WHERE r.submission_id = s.id AND r.user_id = $me)
|
||||||
${excludeId ? "AND s.id != $exclude" : ""}`
|
${excludeId ? "AND s.id != $exclude" : ""}`
|
||||||
const params = { me: user.id, ...(excludeId ? { exclude: excludeId } : {}) }
|
const params = { me: user.id, ...(excludeId ? { exclude: excludeId } : {}) }
|
||||||
const pick =
|
const picked =
|
||||||
get(
|
get<{ rid: number }>(
|
||||||
`${base} AND (SELECT COUNT(*) FROM submission_rating r WHERE r.submission_id = s.id) < 5
|
`${base} AND (SELECT COUNT(*) FROM submission_rating r WHERE r.submission_id = s.id) < 5
|
||||||
ORDER BY RANDOM() LIMIT 1`,
|
ORDER BY RANDOM() LIMIT 1`,
|
||||||
params,
|
params,
|
||||||
) ?? get(`${base} ORDER BY RANDOM() LIMIT 1`, params)
|
) ?? get<{ rid: number }>(`${base} ORDER BY RANDOM() LIMIT 1`, params)
|
||||||
if (!pick) return c.json(null)
|
if (!picked) return c.json(null)
|
||||||
|
const pick = get(
|
||||||
|
`SELECT s.id, u.username, t.title AS task_title, t.display AS task_display,
|
||||||
|
t.task_type, s.html, s.css, s.js
|
||||||
|
${SUBMISSION_FROM} WHERE s.rowid = $rid`,
|
||||||
|
{ rid: picked.rid },
|
||||||
|
)!
|
||||||
return c.json({
|
return c.json({
|
||||||
submission_id: uuidOut(pick.id),
|
submission_id: uuidOut(pick.id),
|
||||||
username: pick.username,
|
username: pick.username,
|
||||||
|
|||||||
Reference in new issue
Block a user