fix(阶段5): jieba 资源导入不能用在 dev;补上 /public/upload 的伺服
## jieba:两种形态得走两条路
上一条 commit 把 jieba 的 .node 改成 `with { type: "file" }` 内嵌,只验证了
编译产物,没跑 `bun run` —— 结果 dev 直接起不来:
TypeError: To load Node-API modules, use require() or process.dlopen
instead of import.
`.node` 的资源导入只有打包器认,运行时不认。而且必须是**动态** import:
静态 import 在模块加载时就求值,拿 isCompiled 判断也来不及。
所以分两条路:dev 用包自己的入口(那条路 bun run 下是好的),编译形态才走
内嵌资源。两边都实测过了。.wasm 没这个毛病,两种形态都正常。
withBuiltinDict 因此变成 async,连带 buildWordFrequencies 也是。
jieba 实例改成缓存 Promise 而不是结果,并发进来两个请求只会建一次词典。
## /public/upload 之前根本没人伺服
后台上传图片存盘、返回 `/public/upload/<name>`,但没有任何路由处理这个前缀 ——
题面里插的图片一律 404。补上,并和头像共用一个 serveUpload:只取路径最后一段
且要求它和原样一致,`..`、子目录、编码斜杠都在这里被拒。
生产环境这些请求也走后端(Caddy 反代整段 /public),不让 Caddy 直接读盘,
这样开发和生产是同一条代码路径。
dev 模式 8 项实测:上传图片/头像/默认头像回退 200,不存在、`..`、编码穿越、
子目录、编码斜杠全 404。编译产物在干净目录里 7 项照旧全过。
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -49,25 +49,55 @@ app.onError((error, c) => {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
/** 头像取不到时的占位图,避免每个没设头像的学生都打一次 404 */
|
||||||
|
const DEFAULT_AVATAR_SVG =
|
||||||
|
'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 128 128"><rect width="128" height="128" rx="64" fill="#e2e8f0"/><circle cx="64" cy="48" r="24" fill="#94a3b8"/><path d="M20 120c4-28 22-42 44-42s40 14 44 42" fill="#94a3b8"/></svg>'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 伺服 /public 下的用户上传文件。
|
||||||
|
*
|
||||||
|
* 只取路径最后一段并要求它和原样一致 —— `..`、子目录、编码过的斜杠都会在这里被拒,
|
||||||
|
* 拼接进 resolve() 的永远只是一个纯文件名。
|
||||||
|
*
|
||||||
|
* 生产环境这些请求也走后端(Caddy 把 /public/* 整段反代过来),不让 Caddy 直接读盘:
|
||||||
|
* 这样开发(Vite 代理)和生产是同一条代码路径,少一处只在服务器上才出错的差异。
|
||||||
|
*/
|
||||||
|
async function serveUpload(pathname: string, prefix: string, directory: string) {
|
||||||
|
const decoded = decodeURIComponent(pathname)
|
||||||
|
const filename = basename(decoded)
|
||||||
|
if (!filename || filename !== decoded.slice(prefix.length + 1)) {
|
||||||
|
return new Response("Not found", { status: 404 })
|
||||||
|
}
|
||||||
|
const file = Bun.file(resolve(directory, filename))
|
||||||
|
if (await file.exists()) {
|
||||||
|
// 文件名由后端生成且内容不变,可以放心长缓存
|
||||||
|
return new Response(file, { headers: { "cache-control": "public, max-age=86400" } })
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
const server = Bun.serve<SubmissionSocketData>({
|
const server = Bun.serve<SubmissionSocketData>({
|
||||||
port: config.port,
|
port: config.port,
|
||||||
async fetch(request, bunServer) {
|
async fetch(request, bunServer) {
|
||||||
const url = new URL(request.url)
|
const url = new URL(request.url)
|
||||||
if (url.pathname.startsWith(`${config.avatarUriPrefix}/`)) {
|
if (url.pathname.startsWith(`${config.avatarUriPrefix}/`)) {
|
||||||
const filename = basename(decodeURIComponent(url.pathname))
|
const hit = await serveUpload(url.pathname, config.avatarUriPrefix, config.avatarDirectory)
|
||||||
if (filename !== decodeURIComponent(url.pathname).split("/").at(-1)) {
|
if (hit) return hit
|
||||||
return new Response("Not found", { status: 404 })
|
if (basename(decodeURIComponent(url.pathname)) === "default.png") {
|
||||||
}
|
return new Response(DEFAULT_AVATAR_SVG, {
|
||||||
const file = Bun.file(resolve(config.avatarDirectory, filename))
|
headers: { "content-type": "image/svg+xml", "cache-control": "public, max-age=3600" },
|
||||||
if (await file.exists()) return new Response(file)
|
})
|
||||||
if (filename === "default.png") {
|
|
||||||
return new Response(
|
|
||||||
'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 128 128"><rect width="128" height="128" rx="64" fill="#e2e8f0"/><circle cx="64" cy="48" r="24" fill="#94a3b8"/><path d="M20 120c4-28 22-42 44-42s40 14 44 42" fill="#94a3b8"/></svg>',
|
|
||||||
{ headers: { "content-type": "image/svg+xml", "cache-control": "public, max-age=3600" } },
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
return new Response("Not found", { status: 404 })
|
return new Response("Not found", { status: 404 })
|
||||||
}
|
}
|
||||||
|
// 题面里插的图片。原来没有这一段 —— 后台上传成功、返回 /public/upload/xxx,
|
||||||
|
// 但没有任何路由伺服它,题面图片一律 404。
|
||||||
|
if (url.pathname.startsWith(`${config.uploadUriPrefix}/`)) {
|
||||||
|
return (
|
||||||
|
(await serveUpload(url.pathname, config.uploadUriPrefix, config.uploadDirectory)) ??
|
||||||
|
new Response("Not found", { status: 404 })
|
||||||
|
)
|
||||||
|
}
|
||||||
if (url.pathname === "/ws/submissions" || url.pathname === "/ws/config") {
|
if (url.pathname === "/ws/submissions" || url.pathname === "/ws/config") {
|
||||||
const user = await getRequestSessionUser(request)
|
const user = await getRequestSessionUser(request)
|
||||||
if (!user) return new Response("Unauthorized", { status: 401 })
|
if (!user) return new Response("Unauthorized", { status: 401 })
|
||||||
|
|||||||
@@ -258,7 +258,7 @@ flowchartRoutes.get("/flowcharts/statistics", requireAuth, async (c) => {
|
|||||||
criteriaAverages,
|
criteriaAverages,
|
||||||
personCount: roster.length,
|
personCount: roster.length,
|
||||||
completedCount: submitted.size,
|
completedCount: submitted.size,
|
||||||
wordFrequencies: buildWordFrequencies(texts),
|
wordFrequencies: await buildWordFrequencies(texts),
|
||||||
dataUnaccepted: roster
|
dataUnaccepted: roster
|
||||||
.filter((row) => !submitted.has(row.username))
|
.filter((row) => !submitted.has(row.username))
|
||||||
.map((row) => ({
|
.map((row) => ({
|
||||||
|
|||||||
@@ -36,23 +36,26 @@ const CUSTOM_WORDS = [
|
|||||||
* 词典加载有一次性开销(约 100ms),放在模块级会拖慢 API 冷启动,
|
* 词典加载有一次性开销(约 100ms),放在模块级会拖慢 API 冷启动,
|
||||||
* 而词云只有教师偶尔点一次。改成首次调用时才建。
|
* 而词云只有教师偶尔点一次。改成首次调用时才建。
|
||||||
*/
|
*/
|
||||||
let instance: JiebaInstance | null = null
|
let instance: Promise<JiebaInstance> | null = null
|
||||||
|
|
||||||
function jieba() {
|
function jieba() {
|
||||||
|
// 缓存 Promise 而不是结果:并发进来两个请求也只会建一次词典
|
||||||
if (instance) return instance
|
if (instance) return instance
|
||||||
const built = withBuiltinDict()
|
instance = (async () => {
|
||||||
// 对应旧后端的 jieba.add_word(w, freq=9999)。
|
const built = await withBuiltinDict()
|
||||||
// @node-rs/jieba@2 没有导出 insertWord/addWord,改用用户词典缓冲区,格式为「词 词频」。
|
// 对应旧后端的 jieba.add_word(w, freq=9999)。
|
||||||
built.loadDict(
|
// @node-rs/jieba@2 没有导出 insertWord/addWord,改用用户词典缓冲区,格式为「词 词频」。
|
||||||
Buffer.from(CUSTOM_WORDS.map((word) => `${word} 9999`).join("\n") + "\n"),
|
built.loadDict(
|
||||||
)
|
Buffer.from(CUSTOM_WORDS.map((word) => `${word} 9999`).join("\n") + "\n"),
|
||||||
instance = built
|
)
|
||||||
return built
|
return built
|
||||||
|
})()
|
||||||
|
return instance
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildWordFrequencies(texts: string[], topN = 80) {
|
export async function buildWordFrequencies(texts: string[], topN = 80) {
|
||||||
const counter = new Map<string, number>()
|
const counter = new Map<string, number>()
|
||||||
const cutter = jieba()
|
const cutter = await jieba()
|
||||||
for (const raw of texts) {
|
for (const raw of texts) {
|
||||||
const text = raw.replaceAll("【重点】", "")
|
const text = raw.replaceAll("【重点】", "")
|
||||||
for (const token of cutter.cut(text)) {
|
for (const token of cutter.cut(text)) {
|
||||||
|
|||||||
69
apps/api/src/vendor/jieba.ts
vendored
69
apps/api/src/vendor/jieba.ts
vendored
@@ -1,55 +1,64 @@
|
|||||||
/**
|
/**
|
||||||
* 直接加载 @node-rs/jieba 的原生模块和词典,绕开这个包自己的加载逻辑。
|
* 加载 jieba。开发和编译两种形态走不同的路,因为没有一条路两边都能用。
|
||||||
*
|
*
|
||||||
* 为什么不老老实实 `import { Jieba } from "@node-rs/jieba"`:
|
* ## 编译形态(生产)
|
||||||
*
|
*
|
||||||
* - 它的 `index.js` 在**运行时**探测平台再 `require` 对应的子包;
|
* `@node-rs/jieba` 的 `index.js` 在运行时探测平台再 `require` 对应的子包,
|
||||||
* - 它的 `dict.js` 用 `__dirname` 去读同目录的 `dict.txt`。
|
* `dict.js` 用 `__dirname` 去读同目录的 `dict.txt`。这两件事在
|
||||||
*
|
* `bun build --compile` 之后都不成立 —— 二进制里 `__dirname` 和模块解析根都是
|
||||||
* 这两件事在 `bun build --compile` 之后都不成立 —— 编译产物里 `__dirname` 和模块
|
* `/$bunfs/root`,子包和 dict.txt 都不在那儿。实测编译后直接
|
||||||
* 解析根都是 `/$bunfs/root`,子包和 dict.txt 都不在那儿。实测编译后直接报
|
|
||||||
* `Failed to load native binding`,而且**只在离开仓库目录后才报**(在仓库里跑时
|
* `Failed to load native binding`,而且**只在离开仓库目录后才报**(在仓库里跑时
|
||||||
* 它顺着 cwd 找到了 node_modules,假装没事),是那种在服务器上才炸的坑。
|
* 它顺着 cwd 摸到了 node_modules,假装没事),是那种到服务器上才炸的坑。
|
||||||
*
|
*
|
||||||
* 改成把 `.node` 和 `dict.txt` 用 `with { type: "file" }` 内嵌成资源:编译时它们被
|
* 所以编译形态下把 `.node` 和 `dict.txt` 用 `with { type: "file" }` 内嵌成资源,
|
||||||
* 塞进二进制,运行时 Bun 把它们摊到 `/$bunfs/root/` 下,`require()` 和 `readFileSync`
|
* 运行时 Bun 把它们摊在 `/$bunfs/root/` 下,`require()` 和 `readFileSync` 都能正常拿到。
|
||||||
* 都能正常拿到。开发模式(不编译)下这个写法拿到的就是 node_modules 里的真实路径,
|
*
|
||||||
* 两种模式同一份代码。
|
* ## 开发形态
|
||||||
|
*
|
||||||
|
* 但 `.node` 的资源导入**只有打包器认,运行时不认**:`bun run` 遇到
|
||||||
|
* `import x from "….node" with { type: "file" }` 会报
|
||||||
|
* “To load Node-API modules, use require() or process.dlopen instead of import.”,
|
||||||
|
* 整个服务起不来。所以开发时老老实实用包自己的入口,那条路在 `bun run` 下是好的。
|
||||||
|
*
|
||||||
|
* 两个分支都必须是**动态** import:静态 import 在模块加载时就会求值,
|
||||||
|
* 用 `isCompiled` 判断也来不及,dev 一样会撞上上面那个报错。
|
||||||
*
|
*
|
||||||
* 平台写死 linux-x64-gnu:部署目标是 debian 基底的容器,本机开发也是 x64 glibc。
|
* 平台写死 linux-x64-gnu:部署目标是 debian 基底的容器,本机开发也是 x64 glibc。
|
||||||
* 换平台(比如改用 alpine/musl 基底镜像)必须同步改这里的 import,否则编译能过、
|
* 换基底镜像或 CPU 架构必须同步改这里,否则编译能过、启动就崩。
|
||||||
* 启动就崩 —— 所以下面加了显式的错误提示。
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import addonPath from "@node-rs/jieba-linux-x64-gnu/jieba.linux-x64-gnu.node" with { type: "file" }
|
import { isCompiled } from "../runtime"
|
||||||
import dictPath from "@node-rs/jieba/dict.txt" with { type: "file" }
|
|
||||||
import { readFileSync } from "node:fs"
|
|
||||||
|
|
||||||
export interface JiebaInstance {
|
export interface JiebaInstance {
|
||||||
cut(text: string, hmm?: boolean): string[]
|
cut(text: string, hmm?: boolean): string[]
|
||||||
loadDict(dict: Buffer): void
|
loadDict(dict: Buffer): void
|
||||||
}
|
}
|
||||||
|
|
||||||
interface JiebaAddon {
|
/** 内置词典建一个分词器。idf.txt 用不到,不内嵌,省二进制体积 */
|
||||||
Jieba: { withDict(dict: Buffer): JiebaInstance }
|
export async function withBuiltinDict(): Promise<JiebaInstance> {
|
||||||
}
|
if (!isCompiled) {
|
||||||
|
const { Jieba } = await import("@node-rs/jieba")
|
||||||
|
const { dict } = await import("@node-rs/jieba/dict")
|
||||||
|
return Jieba.withDict(dict) as unknown as JiebaInstance
|
||||||
|
}
|
||||||
|
|
||||||
let addon: JiebaAddon | null = null
|
const { readFileSync } = await import("node:fs")
|
||||||
|
const addonPath = (
|
||||||
|
await import("@node-rs/jieba-linux-x64-gnu/jieba.linux-x64-gnu.node", {
|
||||||
|
with: { type: "file" },
|
||||||
|
})
|
||||||
|
).default as unknown as string
|
||||||
|
const dictPath = (await import("@node-rs/jieba/dict.txt", { with: { type: "file" } }))
|
||||||
|
.default as unknown as string
|
||||||
|
|
||||||
function loadAddon(): JiebaAddon {
|
let addon: { Jieba: { withDict(dict: Buffer): JiebaInstance } }
|
||||||
if (addon) return addon
|
|
||||||
try {
|
try {
|
||||||
addon = require(addonPath as unknown as string) as JiebaAddon
|
addon = require(addonPath)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`加载 jieba 原生模块失败(${addonPath})。若已更换容器基底或 CPU 架构,` +
|
`加载 jieba 原生模块失败(${addonPath})。若已更换容器基底或 CPU 架构,` +
|
||||||
`需同步修改 src/vendor/jieba.ts 里写死的 linux-x64-gnu 导入。原始错误:${String(error)}`,
|
`需同步修改 src/vendor/jieba.ts 里写死的 linux-x64-gnu 导入。原始错误:${String(error)}`,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
return addon
|
return addon.Jieba.withDict(readFileSync(dictPath))
|
||||||
}
|
|
||||||
|
|
||||||
/** 内置词典(dict.txt,约 4.8MB)。idf.txt 用不到,不内嵌,省二进制体积 */
|
|
||||||
export function withBuiltinDict(): JiebaInstance {
|
|
||||||
return loadAddon().Jieba.withDict(readFileSync(dictPath as unknown as string))
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user