feat: 加 prompt-stats 脚本,对比某天前后学生和 AI 对话的情况
Deploy / deploy (build, debian, 22) (push) Canceled after 0s
Deploy / deploy (build, debian, 22) (push) Canceled after 0s
轮数分布、层级分布、第 2 轮起的 L1、只粘贴外部代码的比例、“出问题了”“帮我写清楚” 和交作品反思的使用情况。只读。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e9127318cb
commit
489274ba0e
2 files changed
+111
-1
No files matched your search
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* 对比某个日期前后,学生和 AI 对话的情况(比如改了 AI 的回复方式以后有没有变化)。
|
||||
* 日期按北京时间,分界日期那天算“之后”。只读,不改数据。
|
||||
*
|
||||
* bun scripts/prompt-stats.ts <分界日期> [--from 日期] [--to 日期]
|
||||
* bun scripts/prompt-stats.ts 2026-10-28 --from 2026-09-01
|
||||
*
|
||||
* 层级是打标签时的标准;分类标准改过以后,先跑 classify-prompts.ts --force 重打,前后才好比。
|
||||
*/
|
||||
import { all, toDbTime } from "../src/db"
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
const opt = (name: string) => {
|
||||
const i = args.indexOf(name)
|
||||
return i >= 0 ? args[i + 1] : undefined
|
||||
}
|
||||
const split = args[0]
|
||||
if (!split || !/^\d{4}-\d{2}-\d{2}$/.test(split)) {
|
||||
console.log("用法:bun scripts/prompt-stats.ts <分界日期> [--from 日期] [--to 日期]")
|
||||
process.exit(1)
|
||||
}
|
||||
const day = (d: string) => toDbTime(new Date(`${d}T00:00:00+08:00`))
|
||||
const nextDay = (d: string) => toDbTime(new Date(new Date(`${d}T00:00:00+08:00`).getTime() + 86_400_000))
|
||||
const periods = [
|
||||
{ name: `${opt("--from") ?? "最早"} ~ ${split} 之前`, from: opt("--from") ? day(opt("--from")!) : "", to: day(split) },
|
||||
{ name: `${split} ~ ${opt("--to") ?? "现在"}`, from: day(split), to: opt("--to") ? nextDay(opt("--to")!) : "9999" },
|
||||
]
|
||||
|
||||
// 和 PromptPanel 里一样:只说“不行”“没反应”的短抱怨;“出问题了”拼出来的句子
|
||||
const VAGUE =
|
||||
/不行|不对|没反应|没用|没效果|用不了|不能用|不能玩|动不了|进不去|打不开|没显示|不显示|显示不出|bug|出错|报错|错了|坏了|白屏|空白|有问题/i
|
||||
const PROBLEM = /^我.+,(想看到.+,)?但实际.+。$/s
|
||||
|
||||
type Msg = {
|
||||
conversation_id: string
|
||||
source: string
|
||||
content: string
|
||||
prompt_level: number | null
|
||||
created: string
|
||||
// 在这个对话的 AI 对话里是第几轮(只数 conversation)
|
||||
round: number | null
|
||||
}
|
||||
|
||||
// 批处理脚本,整表扫一遍没关系
|
||||
const msgs = all<Msg>(
|
||||
`SELECT conversation_id, source, content, prompt_level, created,
|
||||
CASE WHEN source = 'conversation'
|
||||
THEN ROW_NUMBER() OVER (PARTITION BY conversation_id, source = 'conversation' ORDER BY created, id) END AS round
|
||||
FROM prompt_message WHERE role = 'user'`,
|
||||
)
|
||||
const subs = all<{ created: string; reflected: number }>(
|
||||
`SELECT s.created, r.submission_id IS NOT NULL AS reflected
|
||||
FROM submission_submission s JOIN task_task t ON t.id = s.task_id
|
||||
LEFT JOIN submission_reflection r ON r.submission_id = s.id
|
||||
WHERE t.task_type = 'challenge'`,
|
||||
)
|
||||
|
||||
const pct = (n: number, d: number) => (d ? `${((n / d) * 100).toFixed(1)}%` : "—")
|
||||
const chars = (s: string) => [...s.trim()].length
|
||||
|
||||
function stats(from: string, to: string) {
|
||||
const inRange = (t: string) => t >= from && t < to
|
||||
const ms = msgs.filter((m) => inRange(m.created))
|
||||
const byConv = new Map<string, Msg[]>()
|
||||
for (const m of ms) byConv.set(m.conversation_id, [...(byConv.get(m.conversation_id) ?? []), m])
|
||||
|
||||
const convs = [...byConv.values()]
|
||||
const ai = convs.filter((c) => c.some((m) => m.source === "conversation"))
|
||||
const manualOnly = convs.filter((c) => c.some((m) => m.source === "manual") && !c.some((m) => m.source === "conversation"))
|
||||
const rounds = ai.map((c) => c.filter((m) => m.source === "conversation").length)
|
||||
const bucket = (lo: number, hi: number) => rounds.filter((n) => n >= lo && n <= hi).length
|
||||
|
||||
const asked = ms.filter((m) => m.source === "conversation")
|
||||
const levels = [1, 2, 3, 4, 5, 6].map((l) => asked.filter((m) => m.prompt_level === l).length)
|
||||
const leveled = levels.reduce((a, b) => a + b, 0)
|
||||
const later = asked.filter((m) => (m.round ?? 1) >= 2 && m.prompt_level)
|
||||
const vague = asked.filter((m) => chars(m.content) <= 15 && VAGUE.test(m.content))
|
||||
const problem = asked.filter((m) => PROBLEM.test(m.content.trim()))
|
||||
const manual = ms.filter((m) => m.source === "manual")
|
||||
const s = subs.filter((x) => inRange(x.created))
|
||||
|
||||
return [
|
||||
["有对话或版本的作品", `${convs.length}`],
|
||||
["只粘贴外部代码", `${manualOnly.length}(${pct(manualOnly.length, convs.length)})`],
|
||||
["和平台 AI 对话的作品", `${ai.length}`],
|
||||
[" 平均轮数", ai.length ? (rounds.reduce((a, b) => a + b, 0) / ai.length).toFixed(2) : "—"],
|
||||
[" 1 轮", pct(bucket(1, 1), ai.length)],
|
||||
[" 2–3 轮", pct(bucket(2, 3), ai.length)],
|
||||
[" 4–6 轮", pct(bucket(4, 6), ai.length)],
|
||||
[" 7 轮以上", pct(bucket(7, Infinity), ai.length)],
|
||||
["提问(AI 对话)", `${asked.length},已分级 ${leveled}`],
|
||||
...levels.map((n, i) => [` L${i + 1}`, pct(n, leveled)]),
|
||||
[" L4–L6", pct(levels[3]! + levels[4]! + levels[5]!, leveled)],
|
||||
[" 第 2 轮起的 L1", pct(later.filter((m) => m.prompt_level === 1).length, later.length)],
|
||||
[" 只说“不行”的短抱怨", `${vague.length}(${pct(vague.length, asked.length)})`],
|
||||
[" 用“出问题了”", `${problem.length}(${pct(problem.length, asked.length)})`],
|
||||
["用了“帮我写清楚”的作品", `${convs.filter((c) => c.some((m) => m.source === "guidance")).length}(占和 AI 对话的 ${pct(ai.filter((c) => c.some((m) => m.source === "guidance")).length, ai.length)})`],
|
||||
["外部 AI 提示词", `${manual.length},12 字以内 ${pct(manual.filter((m) => chars(m.content) <= 12).length, manual.length)}`],
|
||||
["挑战提交", `${s.length},写了反思 ${pct(s.filter((x) => x.reflected).length, s.length)}`],
|
||||
]
|
||||
}
|
||||
|
||||
const [a, b] = periods.map((p) => stats(p.from, p.to))
|
||||
// 终端里中文占两格
|
||||
const cols = (k: string) => [...k].reduce((n, ch) => n + (ch.codePointAt(0)! >= 0x2e80 ? 2 : 1), 0)
|
||||
const width = Math.max(...a!.map(([k]) => cols(k!))) + 2
|
||||
const pad = (k: string) => k + " ".repeat(width - cols(k))
|
||||
console.log(`${pad("")}${periods[0]!.name} → ${periods[1]!.name}`)
|
||||
for (const [i, [k, v]] of a!.entries()) console.log(`${pad(k!)}${v} → ${b![i]![1]}`)
|
||||
Reference in new issue
Block a user