自动评分:评分规则和执行引擎

- 每个示例的 checks.yaml:书上评价表的功能项转成操作步骤和期望结果(10 个示例共 23 项)
- 引擎:编译一次,每项在沙箱里单独启动一次程序,按控件名操作、调节实训台、核对结果;
  提示框自动点第一个按钮;全站同时最多跑 2 项
- 不通过时写明期望和实际,如"期望6 秒内 txt_even 的文字包含"发现入侵",实际是"正常"";
  找不到书上规定的控件名时直接指出
- 规则写错时指出第几项第几步
- 测试:参考答案全部通过;编译错误、控件改名、判断写反、忘停定时器、接错 DO 口各自报出对应原因

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
xuyueandClaude Opus 5.5 committed 2026-10-22 18:20:46 +08:00
1 parent 5bb2940969
commit 37fc747b49
16 files changed
+824

No files matched your search

+128
View File
@@ -0,0 +1,128 @@
// 评分规则(每个实验的 checks.yaml):书上评价表的功能项,每项是一串操作步骤和期望结果。
// 规则写错时报出是哪一项、哪一步,方便改。
import { ANALOG_INFO, type AnalogKey, DIGITAL_CHANNELS, type DigitalKey } from '@iot/shared'
export interface Check {
title: string
steps: Step[]
}
export type Step =
| { kind: 'click'; control: string }
| { kind: 'text'; control: string; value: string }
| { kind: 'select'; control: string; item: string }
| { kind: 'board'; analog: Partial<Record<AnalogKey, number>>; digital: Partial<Record<DigitalKey, boolean>> }
| { kind: 'wait'; ms: number }
/** desc:给学生看的期望说明,如"显示当前时间",不写就按期望自动生成 */
| { kind: 'expect'; expect: Expectation; within: number; desc?: string }
export type Expectation =
/** 控件属性等于某个值:text → Text,ledText → LedText(LED 控件),enabled → Enabled,checked → Checked,on → IsOn(灯泡、报警灯) */
| { kind: 'prop'; control: string; prop: string; value: unknown }
| { kind: 'contains'; control: string; text: string }
| { kind: 'matches'; control: string; pattern: RegExp; source: string }
/** 一段时间内控件文字不变(比如定时器已经停止) */
| { kind: 'unchanged'; control: string; ms: number }
| { kind: 'led'; text: string }
| { kind: 'relay'; index: number; on: boolean }
| { kind: 'zigbee'; index: number; on: boolean }
| { kind: 'messageBox'; text: string }
| { kind: 'console'; text: string }
/** 期望默认最多等多久(毫秒);依赖定时器的检查在规则里写 within */
const DEFAULT_WITHIN = 2000
const PROP_KEYS: Record<string, string> = { text: 'Text', ledText: 'LedText', enabled: 'Enabled', checked: 'Checked', on: 'IsOn' }
const STRING_PROPS = new Set(['Text', 'LedText'])
/** "6s"、"500ms" 或毫秒数 */
function duration(value: unknown, where: string): number {
if (typeof value === 'number' && value >= 0) return value
const m = typeof value === 'string' ? /^(\d+(?:\.\d+)?)\s*(ms|s)$/.exec(value.trim()) : null
if (!m) throw new Error(`${where}:时长要写成 6s 或 500ms,现在是 ${JSON.stringify(value)}`)
return Number(m[1]) * (m[2] === 's' ? 1000 : 1)
}
function str(value: unknown, where: string, what: string): string {
if (typeof value === 'string' && value) return value
if (typeof value === 'number') return String(value)
throw new Error(`${where}:缺少${what}`)
}
function bool(value: unknown, where: string, what: string): boolean {
if (typeof value === 'boolean') return value
throw new Error(`${where}:${what}要写 true 或 false`)
}
function parseExpectation(raw: Record<string, unknown>, where: string): Expectation {
if ('led' in raw) return { kind: 'led', text: str(raw.led, where, ' LED 屏应显示的文字') }
if ('messageBox' in raw) return { kind: 'messageBox', text: str(raw.messageBox, where, '提示框应包含的文字') }
if ('console' in raw) return { kind: 'console', text: str(raw.console, where, '控制台应输出的文字') }
if ('relay' in raw) return { kind: 'relay', index: Number(raw.relay), on: bool(raw.on, where, 'on') }
if ('zigbee' in raw) return { kind: 'zigbee', index: Number(raw.zigbee), on: bool(raw.on, where, 'on') }
const control = str(raw.control, where, '控件名 control')
if ('contains' in raw) return { kind: 'contains', control, text: str(raw.contains, where, 'contains') }
if ('matches' in raw) {
const source = str(raw.matches, where, 'matches')
return { kind: 'matches', control, pattern: new RegExp(source), source }
}
if ('unchanged' in raw) return { kind: 'unchanged', control, ms: duration(raw.unchanged, where) }
for (const [key, prop] of Object.entries(PROP_KEYS)) {
if (key in raw) return { kind: 'prop', control, prop, value: STRING_PROPS.has(prop) ? String(raw[key]) : bool(raw[key], where, key) }
}
throw new Error(`${where}:不认识的期望 ${JSON.stringify(raw)}`)
}
function parseBoard(raw: Record<string, unknown>, where: string): Step {
const analog: Partial<Record<AnalogKey, number>> = {}
const digital: Partial<Record<DigitalKey, boolean>> = {}
for (const [key, value] of Object.entries(raw)) {
if (key in ANALOG_INFO && typeof value === 'number') analog[key as AnalogKey] = value
else if ((DIGITAL_CHANNELS as readonly string[]).includes(key) && typeof value === 'boolean') digital[key as DigitalKey] = value
else throw new Error(`${where}:实训台没有 ${key},或者值的类型不对(模拟量写数字,开关量写 true / false)`)
}
return { kind: 'board', analog, digital }
}
function parseStep(raw: unknown, where: string): Step {
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw new Error(`${where}:每一步要写成"操作: 内容"`)
const [[kind, value], ...rest] = Object.entries(raw as Record<string, unknown>)
if (kind === 'expect' && rest.length === 0) {
const body = value as Record<string, unknown>
const { within, desc, ...expectation } = body
return {
kind: 'expect',
expect: parseExpectation(expectation, where),
within: within === undefined ? DEFAULT_WITHIN : duration(within, where),
...(desc === undefined ? {} : { desc: String(desc) }),
}
}
if (rest.length > 0) throw new Error(`${where}:一步只能写一个操作`)
const obj = (value ?? {}) as Record<string, unknown>
switch (kind) {
case 'click':
return { kind, control: str(value, where, '要点击的控件名') }
case 'text':
return { kind, control: str(obj.control, where, '控件名 control'), value: String(obj.value ?? '') }
case 'select':
return { kind, control: str(obj.control, where, '控件名 control'), item: str(obj.item, where, '要选的项 item') }
case 'board':
return parseBoard(obj, where)
case 'wait':
return { kind, ms: duration(value, where) }
default:
throw new Error(`${where}:不认识的操作 ${kind}`)
}
}
/** 解析 checks.yaml 的内容 */
export function parseChecks(yaml: string): Check[] {
const raw = Bun.YAML.parse(yaml)
if (!Array.isArray(raw)) throw new Error('评分规则应该是一个列表,每项以"- title:"开头')
return raw.map((item: { title?: unknown; steps?: unknown }, i: number) => {
const title = str(item?.title, `第 ${i + 1} 项`, '名称 title')
if (!Array.isArray(item.steps) || item.steps.length === 0) throw new Error(`第 ${i + 1} 项"${title}":缺少步骤 steps`)
return { title, steps: item.steps.map((step, j) => parseStep(step, `第 ${i + 1} 项"${title}"第 ${j + 1} 步`)) }
})
}
+171
View File
@@ -0,0 +1,171 @@
// 评分时驱动一个学生程序:在沙箱里启动,像浏览器一样维护控件树,按控件名操作控件、调节实训台。
// 提示框自动点第一个按钮("确定"/"是"),免得程序一直等在 MessageBox.Show 上。
import type { Subprocess } from 'bun'
import type { ControlNode, RunnerInput, RunnerMessage, UserAction } from '@iot/shared'
import { Board } from '../board'
import { readLines } from '../runner-process'
import { runCommand } from '../sandbox'
import type { Step } from './checks'
type BoardStep = Extract<Step, { kind: 'board' }>
export class ProgramDriver {
readonly board = new Board()
/** 当前显示的窗体,按显示先后排列 */
readonly forms: ControlNode[] = []
private readonly nodes = new Map<string, ControlNode>()
private readonly parents = new Map<string, ControlNode>()
readonly messageBoxes: { text: string; caption: string }[] = []
readonly exceptions: string[] = []
console = ''
exited = false
private proc?: Subprocess<'pipe', 'pipe', 'pipe'>
private listeners = new Set<() => void>()
private lastMessageAt = Date.now()
start(projectDir: string) {
const { cmd, env } = runCommand(projectDir, { SIM_BOARD: JSON.stringify(this.board.runnerInputs()) })
const proc = Bun.spawn(cmd, { stdin: 'pipe', stdout: 'pipe', stderr: 'pipe', env })
this.proc = proc
void readLines(proc.stdout, (line) => {
try {
this.receive(JSON.parse(line) as RunnerMessage)
} catch {
this.console += `${line}\n`
}
})
void new Response(proc.stderr).text()
void proc.exited.then(() => {
this.exited = true
this.notify()
})
}
stop() {
this.proc?.kill()
}
private send(message: RunnerInput) {
if (this.exited || !this.proc) return
this.proc.stdin.write(`${JSON.stringify(message)}\n`)
this.proc.stdin.flush()
}
private register(node: ControlNode, parent?: ControlNode) {
this.nodes.set(node.id, node)
if (parent) this.parents.set(node.id, parent)
for (const child of node.children) this.register(child, node)
}
private unregister(node: ControlNode) {
this.nodes.delete(node.id)
this.parents.delete(node.id)
for (const child of node.children) this.unregister(child)
}
private receive(message: RunnerMessage) {
this.lastMessageAt = Date.now()
switch (message.type) {
case 'formShown':
this.forms.push(message.form)
this.register(message.form)
break
case 'formClosed': {
const index = this.forms.findIndex((f) => f.id === message.formId)
if (index >= 0) this.unregister(this.forms.splice(index, 1)[0])
break
}
case 'patch':
for (const change of message.changes) {
const node = this.nodes.get(change.id)
if (node) node.props[change.prop] = change.value
}
break
case 'controlAdded': {
const parent = this.nodes.get(message.parentId)
if (!parent) break
parent.children.splice(message.index, 0, message.node)
this.register(message.node, parent)
break
}
case 'controlRemoved': {
const node = this.nodes.get(message.id)
const parent = this.parents.get(message.id)
if (!node) break
if (parent) parent.children = parent.children.filter((c) => c.id !== node.id)
this.unregister(node)
break
}
case 'messageBox': {
this.messageBoxes.push({ text: message.text, caption: message.caption })
// 自动点第一个按钮
const first = /^(OK|Yes|Retry|Abort)/.exec(message.buttons)?.[1] ?? 'OK'
this.send({ type: 'reply', requestId: message.requestId, result: first })
break
}
case 'device':
this.board.applyDevice(message)
break
case 'console':
this.console += message.text
break
case 'exception':
this.exceptions.push(message.message)
break
}
this.notify()
}
private notify() {
for (const listener of this.listeners) listener()
}
/** 按控件名找控件;多个窗体里有同名控件时取最后显示的窗体里的 */
find(name: string): ControlNode | undefined {
for (const form of [...this.forms].reverse()) {
const found = findIn(form, name)
if (found) return found
}
}
act(node: ControlNode, action: UserAction) {
this.send({ type: 'action', id: node.id, ...action })
}
setBoard(step: BoardStep) {
this.board.applyInput({ type: 'boardInput', analog: step.analog, digital: step.digital })
this.send(this.board.runnerInputs())
}
/** 等到条件成立(立即检查,之后每收到一条消息检查一次),超时返回 false */
waitFor(condition: () => boolean, timeoutMs: number): Promise<boolean> {
if (condition()) return Promise.resolve(true)
return new Promise((resolve) => {
const done = (result: boolean) => {
clearTimeout(timer)
this.listeners.delete(check)
resolve(result)
}
const check = () => {
if (condition()) done(true)
}
const timer = setTimeout(() => done(condition()), timeoutMs)
this.listeners.add(check)
})
}
/** 等程序处理完刚才的操作:从现在起一段时间内没有新消息(最多等 maxMs) */
async settle(quietMs = 150, maxMs = 2000) {
const start = Date.now()
const quietFor = () => Date.now() - Math.max(this.lastMessageAt, start)
while (Date.now() - start < maxMs && quietFor() < quietMs) await Bun.sleep(quietMs - quietFor())
}
}
function findIn(node: ControlNode, name: string): ControlNode | undefined {
if (node.name === name) return node
for (const child of node.children) {
const found = findIn(child, name)
if (found) return found
}
}
+186
View File
@@ -0,0 +1,186 @@
// 自动评分:编译一次学生工程,每个检查项各启动一次程序,按步骤操作并核对结果。
// 服务器只有 2 核,全站同时最多跑 GRADE_CONCURRENCY 个检查,其余排队。
import type { CheckResult, ControlNode, GradeResult } from '@iot/shared'
import { compileService } from '../compile-service'
import type { Check, Expectation, Step } from './checks'
import { ProgramDriver } from './driver'
const GRADE_CONCURRENCY = 2
/** 程序启动后多久内应该显示出窗体 */
const START_TIMEOUT = 10_000
const RELAY_NAMES = ['报警灯', '照明灯', '路灯', '插座 1', '插座 2']
const ZIGBEE_NAMES = ['ZigBee 节点 0', '照明灯 1']
// ---------- 并发限制 ----------
let running = 0
const waiting: (() => void)[] = []
async function withSlot<T>(task: () => Promise<T>): Promise<T> {
if (running >= GRADE_CONCURRENCY) await new Promise<void>((resolve) => waiting.push(resolve))
running++
try {
return await task()
} finally {
running--
waiting.shift()?.()
}
}
/** 给学生工程评分;dir 是已写好源文件的工程目录 */
export async function grade(dir: string, checks: Check[]): Promise<GradeResult> {
const start = Date.now()
const compiled = await compileService.compile(dir)
if (!compiled.ok) {
return {
compiled: false,
results: checks.map((c) => ({ title: c.title, passed: false, message: '编译没有通过,先改正编译错误' })),
ms: Date.now() - start,
}
}
const results = await Promise.all(checks.map((check) => withSlot(() => runCheck(dir, check))))
return { compiled: true, results, ms: Date.now() - start }
}
class Failure extends Error {}
async function runCheck(dir: string, check: Check): Promise<CheckResult> {
const driver = new ProgramDriver()
try {
driver.start(dir)
if (!(await driver.waitFor(() => driver.forms.length > 0 || driver.exited, START_TIMEOUT)) || driver.forms.length === 0) {
throw new Failure(driver.exceptions[0] ? `程序出错:${driver.exceptions[0]}` : '程序启动后没有显示窗体')
}
for (const step of check.steps) {
await runStep(driver, step)
if (driver.exceptions.length > 0) throw new Failure(`程序出错:${driver.exceptions[0]}`)
}
return { title: check.title, passed: true }
} catch (error) {
if (error instanceof Failure) return { title: check.title, passed: false, message: error.message }
throw error
} finally {
driver.stop()
}
}
function need(driver: ProgramDriver, name: string): ControlNode {
const node = driver.find(name)
if (!node) throw new Failure(`找不到名为 ${name} 的控件(书上规定的控件名,注意大小写)`)
return node
}
async function runStep(driver: ProgramDriver, step: Step) {
switch (step.kind) {
case 'click': {
const node = need(driver, step.control)
if (node.props.Enabled === false) throw new Failure(`${step.control} 是灰色的(Enabled 为 false),点不了`)
driver.act(node, { action: 'click' })
await driver.settle()
break
}
case 'text':
driver.act(need(driver, step.control), { action: 'text', value: step.value })
await driver.settle()
break
case 'select': {
const node = need(driver, step.control)
if (node.type !== 'ComboBox') throw new Failure(`${step.control} 不是下拉框`)
// 和浏览器一样先展开(教材在下拉框的 Click 里填充串口列表),再选
driver.act(node, { action: 'dropDown' })
await driver.settle()
const items = ((node.props.Items as unknown[] | undefined) ?? []).map(String)
const index = items.indexOf(step.item)
if (index < 0) {
throw new Failure(`下拉框 ${step.control} 里没有"${step.item}"(现在的选项:${items.length ? items.join('、') : '空'})`)
}
driver.act(node, { action: 'select', index })
await driver.settle()
break
}
case 'board':
driver.setBoard(step)
await driver.settle()
break
case 'wait':
await Bun.sleep(step.ms)
break
case 'expect':
await expectStep(driver, step)
break
}
}
const seconds = (ms: number) => `${Math.round(ms / 100) / 10} 秒`
const quote = (value: unknown) => (value === undefined || value === null || value === '' ? '空的' : `"${String(value)}"`)
async function expectStep(driver: ProgramDriver, step: Extract<Step, { kind: 'expect' }>) {
const { expect: e, within } = step
const wait = within > 2000 ? `${seconds(within)}内` : ''
if (e.kind === 'unchanged') {
const node = need(driver, e.control)
const before = node.props.Text
await Bun.sleep(e.ms)
if (node.props.Text !== before) {
throw new Failure(`期望 ${e.control} ${step.desc ?? `在 ${seconds(e.ms)}内保持不变`},实际从${quote(before)}变成了${quote(node.props.Text)}`)
}
return
}
const ok = await driver.waitFor(() => satisfied(driver, e), within)
if (!ok) throw new Failure(describeFailure(driver, e, step.desc, wait))
}
function satisfied(driver: ProgramDriver, e: Exclude<Expectation, { kind: 'unchanged' }>): boolean {
const props = (name: string) => driver.find(name)?.props
switch (e.kind) {
case 'prop':
return props(e.control)?.[e.prop] === e.value
case 'contains':
return String(props(e.control)?.Text ?? '').includes(e.text)
case 'matches':
return e.pattern.test(String(props(e.control)?.Text ?? ''))
case 'led':
return driver.board.state.ledText.includes(e.text)
case 'relay':
return driver.board.state.relays[e.index] === e.on
case 'zigbee':
return driver.board.state.zigbee[e.index] === e.on
case 'messageBox':
return driver.messageBoxes.some((box) => box.text.includes(e.text))
case 'console':
return driver.console.includes(e.text)
}
}
function describeFailure(driver: ProgramDriver, e: Exclude<Expectation, { kind: 'unchanged' }>, desc: string | undefined, wait: string): string {
const prefix = wait ? `${wait}` : ''
const onOff = (on: boolean) => (on ? '打开' : '关闭')
switch (e.kind) {
case 'prop': {
const actual = need(driver, e.control).props[e.prop]
if (e.prop === 'Text' || e.prop === 'LedText') return `期望${prefix} ${e.control} ${desc ?? `显示"${e.value}"`},实际是${quote(actual)}`
if (e.prop === 'Enabled') return `期望${prefix} ${e.control} ${desc ?? (e.value ? '可用' : '变成灰色不可用')},实际${actual === false ? '不可用' : '可用'}`
if (e.prop === 'Checked') return `期望${prefix} ${e.control} ${desc ?? (e.value ? '被选中' : '没有被选中')},实际${actual ? '被选中' : '没有被选中'}`
if (e.prop === 'IsOn') return `期望${prefix}窗体上的 ${e.control} ${desc ?? (e.value ? '亮' : '灭')},实际是${actual ? '亮' : '灭'}的`
return `期望${prefix} ${e.control} 的 ${e.prop} 是 ${String(e.value)},实际是 ${String(actual)}`
}
case 'contains':
return `期望${prefix} ${e.control} ${desc ?? `的文字包含"${e.text}"`},实际是${quote(need(driver, e.control).props.Text)}`
case 'matches':
return `期望${prefix} ${e.control} ${desc ?? `的文字符合 ${e.pattern}`},实际是${quote(need(driver, e.control).props.Text)}`
case 'led':
return `期望${prefix}实训台 LED 屏${desc ?? `显示"${e.text}"`},实际${driver.board.state.ledText ? `显示"${driver.board.state.ledText}"` : '什么也没显示'}`
case 'relay':
return `期望${prefix}实训台的${RELAY_NAMES[e.index] ?? `DO${e.index}`}(ADAM-4150 DO${e.index})${desc ?? onOff(e.on)},实际是${onOff(driver.board.state.relays[e.index])}的`
case 'zigbee':
return `期望${prefix}实训台的${ZIGBEE_NAMES[e.index] ?? `ZigBee 继电器 ${e.index}`}${desc ?? onOff(e.on)},实际是${onOff(driver.board.state.zigbee[e.index])}的`
case 'messageBox': {
const shown = driver.messageBoxes.map((b) => `"${b.text}"`).join('、')
return `期望${prefix}弹出${desc ?? `包含"${e.text}"的提示框`},${shown ? `实际弹出的是 ${shown}` : '实际没有弹出提示框'}`
}
case 'console':
return `期望${prefix}控制台${desc ?? `输出"${e.text}"`},实际输出${quote(driver.console.trim())}`
}
}