diff --git a/CLAUDE.md b/CLAUDE.md index cfa91e7..e28c7d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ The backend was rewritten from Django in 2026-10 to cut memory use (~25 MB vs. 1 - Who can read a submission (`canView` in `routes/submission.ts`): staff, the owner, anyone if it's in the showcase, or any student who has submitted the same task (any class). Students' `GET /submission/` only lists their own. Peer works and the random-rating item are anonymous until the viewer rates them (`PUT /:id/score` returns the author). `flag` now only carries `present` (teacher's in-class presentation queue); the old four colors are no longer shown - `src/services/` — `score.ts` (rating weights super×0.5, admin×0.3, normal×0.2, then Bayesian smoothing with C=3 and zones), `gradebook.ts`, `llm.ts` (OpenAI-compatible streaming via fetch; DeepSeek default, ARK for doubao models; chat context is the last 20 messages with code kept only in the newest reply that has code. Chat replies are written as **做了什么 / 我替你定的 / 想一想** — the decisions the AI made for the student and one open question, to prompt the next round; a reply to a plain question has no code and the WebSocket sends `code: null`. For challenges the AI also gets the task brief (`challengeBrief`) as a checklist only (`BRIEF_RULES`): it builds just what the student asked for and asks about one missing requirement in “想一想”; creations get no brief. The “帮我写清楚” coach gets the task brief, recent prompts and the last AI explanation from `guidanceContext` in `services/prompt.ts`, and judges by what the student added beyond the brief, like the classifier), `classifier.ts` (Bloom-level L1–L6 prompt tagging, runs in background; the model sees the task brief and the previous prompt, temperature 0, prompts under 4 chars get no level), `cascade.ts` (Django's CASCADE deletes done by hand, plus the shared delete rules: `hasRatingsOrAwards`, `deleteRound`), `pyround.ts` (`pyRound`/`pySum` reproduce Python's `round()` and `sum()` so scores match the old system bit-for-bit) - `src/ws.ts` — `/ws/prompt//?token=` (code generation chat; the AI context and the page's chat include `conversation`, `edit` and `manual` messages) and `/ws/guidance//?token=...` (prompt coaching); one message per connection at a time (a message sent while the AI is still replying gets an error, payloads capped at 64 KB); on disconnect the pending user message is deleted and the LLM request aborted - - `scripts/` — one-off CLIs: `create-superuser.ts`, `classify-prompts.ts`, `recalculate-scores.ts` + - `scripts/` — one-off CLIs: `create-superuser.ts`, `classify-prompts.ts`, `recalculate-scores.ts`, `prompt-stats.ts` (read-only before/after comparison of AI-chat behavior: rounds, levels, external-code share, “出问题了” / reflection usage) - Roles: `super`, `admin`, `normal`. Task model is multi-table: `task_task` + one of `task_tutorial` / `task_challenge` / `task_creation` (`task_ptr_id`) - **`client/`** — Vue 3 + TypeScript SPA built with **Vite**; Bun is the package manager and runtime (no Node/npm) @@ -62,6 +62,7 @@ bun run typecheck bun scripts/create-superuser.ts bun scripts/classify-prompts.ts [--task-id N] [--force] [--dry-run] bun scripts/recalculate-scores.ts [--dry-run] +bun scripts/prompt-stats.ts [--from date] [--to date] # dates in Beijing time ``` Config comes from `server/.env` (see `server/.env.example`); dev DB is `server/data/db.sqlite3`, uploads go to the repo-root `media/`. diff --git a/server/scripts/prompt-stats.ts b/server/scripts/prompt-stats.ts new file mode 100644 index 0000000..b99aea2 --- /dev/null +++ b/server/scripts/prompt-stats.ts @@ -0,0 +1,109 @@ +/** + * 对比某个日期前后,学生和 AI 对话的情况(比如改了 AI 的回复方式以后有没有变化)。 + * 日期按北京时间,分界日期那天算“之后”。只读,不改数据。 + * + * bun scripts/prompt-stats.ts <分界日期> [--from 日期] [--to 日期] + * bun scripts/prompt-stats.ts 2026-10-28 --from 2026-09-01 + * + * 层级是打标签时的标准;分类标准改过以后,先跑 classify-prompts.ts --force 重打,前后才好比。 + */ +import { all, toDbTime } from "../src/db" + +const args = process.argv.slice(2) +const opt = (name: string) => { + const i = args.indexOf(name) + return i >= 0 ? args[i + 1] : undefined +} +const split = args[0] +if (!split || !/^\d{4}-\d{2}-\d{2}$/.test(split)) { + console.log("用法:bun scripts/prompt-stats.ts <分界日期> [--from 日期] [--to 日期]") + process.exit(1) +} +const day = (d: string) => toDbTime(new Date(`${d}T00:00:00+08:00`)) +const nextDay = (d: string) => toDbTime(new Date(new Date(`${d}T00:00:00+08:00`).getTime() + 86_400_000)) +const periods = [ + { name: `${opt("--from") ?? "最早"} ~ ${split} 之前`, from: opt("--from") ? day(opt("--from")!) : "", to: day(split) }, + { name: `${split} ~ ${opt("--to") ?? "现在"}`, from: day(split), to: opt("--to") ? nextDay(opt("--to")!) : "9999" }, +] + +// 和 PromptPanel 里一样:只说“不行”“没反应”的短抱怨;“出问题了”拼出来的句子 +const VAGUE = + /不行|不对|没反应|没用|没效果|用不了|不能用|不能玩|动不了|进不去|打不开|没显示|不显示|显示不出|bug|出错|报错|错了|坏了|白屏|空白|有问题/i +const PROBLEM = /^我.+,(想看到.+,)?但实际.+。$/s + +type Msg = { + conversation_id: string + source: string + content: string + prompt_level: number | null + created: string + // 在这个对话的 AI 对话里是第几轮(只数 conversation) + round: number | null +} + +// 批处理脚本,整表扫一遍没关系 +const msgs = all( + `SELECT conversation_id, source, content, prompt_level, created, + CASE WHEN source = 'conversation' + THEN ROW_NUMBER() OVER (PARTITION BY conversation_id, source = 'conversation' ORDER BY created, id) END AS round + FROM prompt_message WHERE role = 'user'`, +) +const subs = all<{ created: string; reflected: number }>( + `SELECT s.created, r.submission_id IS NOT NULL AS reflected + FROM submission_submission s JOIN task_task t ON t.id = s.task_id + LEFT JOIN submission_reflection r ON r.submission_id = s.id + WHERE t.task_type = 'challenge'`, +) + +const pct = (n: number, d: number) => (d ? `${((n / d) * 100).toFixed(1)}%` : "—") +const chars = (s: string) => [...s.trim()].length + +function stats(from: string, to: string) { + const inRange = (t: string) => t >= from && t < to + const ms = msgs.filter((m) => inRange(m.created)) + const byConv = new Map() + for (const m of ms) byConv.set(m.conversation_id, [...(byConv.get(m.conversation_id) ?? []), m]) + + const convs = [...byConv.values()] + const ai = convs.filter((c) => c.some((m) => m.source === "conversation")) + const manualOnly = convs.filter((c) => c.some((m) => m.source === "manual") && !c.some((m) => m.source === "conversation")) + const rounds = ai.map((c) => c.filter((m) => m.source === "conversation").length) + const bucket = (lo: number, hi: number) => rounds.filter((n) => n >= lo && n <= hi).length + + const asked = ms.filter((m) => m.source === "conversation") + const levels = [1, 2, 3, 4, 5, 6].map((l) => asked.filter((m) => m.prompt_level === l).length) + const leveled = levels.reduce((a, b) => a + b, 0) + const later = asked.filter((m) => (m.round ?? 1) >= 2 && m.prompt_level) + const vague = asked.filter((m) => chars(m.content) <= 15 && VAGUE.test(m.content)) + const problem = asked.filter((m) => PROBLEM.test(m.content.trim())) + const manual = ms.filter((m) => m.source === "manual") + const s = subs.filter((x) => inRange(x.created)) + + return [ + ["有对话或版本的作品", `${convs.length}`], + ["只粘贴外部代码", `${manualOnly.length}(${pct(manualOnly.length, convs.length)})`], + ["和平台 AI 对话的作品", `${ai.length}`], + [" 平均轮数", ai.length ? (rounds.reduce((a, b) => a + b, 0) / ai.length).toFixed(2) : "—"], + [" 1 轮", pct(bucket(1, 1), ai.length)], + [" 2–3 轮", pct(bucket(2, 3), ai.length)], + [" 4–6 轮", pct(bucket(4, 6), ai.length)], + [" 7 轮以上", pct(bucket(7, Infinity), ai.length)], + ["提问(AI 对话)", `${asked.length},已分级 ${leveled}`], + ...levels.map((n, i) => [` L${i + 1}`, pct(n, leveled)]), + [" L4–L6", pct(levels[3]! + levels[4]! + levels[5]!, leveled)], + [" 第 2 轮起的 L1", pct(later.filter((m) => m.prompt_level === 1).length, later.length)], + [" 只说“不行”的短抱怨", `${vague.length}(${pct(vague.length, asked.length)})`], + [" 用“出问题了”", `${problem.length}(${pct(problem.length, asked.length)})`], + ["用了“帮我写清楚”的作品", `${convs.filter((c) => c.some((m) => m.source === "guidance")).length}(占和 AI 对话的 ${pct(ai.filter((c) => c.some((m) => m.source === "guidance")).length, ai.length)})`], + ["外部 AI 提示词", `${manual.length},12 字以内 ${pct(manual.filter((m) => chars(m.content) <= 12).length, manual.length)}`], + ["挑战提交", `${s.length},写了反思 ${pct(s.filter((x) => x.reflected).length, s.length)}`], + ] +} + +const [a, b] = periods.map((p) => stats(p.from, p.to)) +// 终端里中文占两格 +const cols = (k: string) => [...k].reduce((n, ch) => n + (ch.codePointAt(0)! >= 0x2e80 ? 2 : 1), 0) +const width = Math.max(...a!.map(([k]) => cols(k!))) + 2 +const pad = (k: string) => k + " ".repeat(width - cols(k)) +console.log(`${pad("")}${periods[0]!.name} → ${periods[1]!.name}`) +for (const [i, [k, v]] of a!.entries()) console.log(`${pad(k!)}${v} → ${b![i]![1]}`)