xuyue/OnlineJudge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更改项目结构

Browse Source
This commit is contained in:
virusdefender
2015-08-12 16:49:25 +08:00
parent 702d51f495
commit 92ab7e5fb2
15 changed files with 39 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
judge/controller/README.md
View File

@@ -1 +0,0 @@
celery -A judge.controller worker -l DEBUG

1
judge/controller/__init__.py
View File

@@ -1 +0,0 @@
# coding=utf-8

5
judge/controller/celery.py
View File

@@ -1,5 +0,0 @@
# coding=utf-8
from __future__ import absolute_import
from celery import Celery
app = Celery("judge", broker="redis://localhost:6379/0", include=["judge.controller.tasks"])

13
judge/controller/tasks.py
View File

@@ -1,13 +0,0 @@
# coding=utf-8
from __future__ import absolute_import
from judge.controller.celery import app
import subprocess32 as subprocess
@app.task
def judge(solution_id, time_limit, memory_limit, test_case_id):
subprocess.call("/usr/bin/docker run -t -i --privileged -v /var/test_case/:/var/judger/test_case/ "
"-v /var/code/:/var/judger/code/ judger python judge/judger/run.py "
"--solution_id %s --time_limit %s --memory_limit %s --test_case_id %s" %
(solution_id, str(time_limit), str(memory_limit), test_case_id),
timeout=(time_limit / 100) * 20)

0
judge/judger/__init__.py
View File

173
judge/judger/client.py
View File

@@ -1,173 +0,0 @@
# coding=utf-8
import json
import commands
import hashlib
from multiprocessing import Pool
from settings import max_running_number, lrun_gid, lrun_uid, judger_workspace
from language import languages
from result import result
from compiler import compile_
from judge_exceptions import JudgeClientError, CompileError
from utils import parse_lrun_output
# 下面这个函数作为代理访问实例变量，否则Python2会报错，是Python2的已知问题
# http://stackoverflow.com/questions/1816958/cant-pickle-type-instancemethod-when-using-pythons-multiprocessing-pool-ma/7309686
def _run(instance, test_case_id):
return instance._judge_one(test_case_id)
class JudgeClient(object):
def __init__(self, language_code, exe_path, max_cpu_time,
max_real_time, max_memory, test_case_dir):
"""
:param language_code: 语言编号
:param exe_path: 可执行文件路径
:param max_cpu_time: 最大cpu时间，单位ms
:param max_real_time: 最大执行时间，单位ms
:param max_memory: 最大内存，单位MB
:param test_case_dir: 测试用例文件夹路径
:return:返回结果list
"""
self._language = languages[str(language_code)]
self._exe_path = exe_path
self._max_cpu_time = max_cpu_time
self._max_real_time = max_real_time
self._max_memory = max_memory
self._test_case_dir = test_case_dir
# 进程池
self._pool = Pool(processes=max_running_number)
# 测试用例配置项
self._test_case_info = self._load_test_case_info()
def _load_test_case_info(self):
# 读取测试用例信息 转换为dict
try:
f = open(self._test_case_dir + "info")
return json.loads(f.read())
except IOError:
raise JudgeClientError("Test case config file not found")
except ValueError:
raise JudgeClientError("Test case config file format error")
def _generate_command(self, test_case_id):
"""
设置相关运行限制 进制访问网络 如果启用tmpfs 就把代码输出写入tmpfs，否则写入硬盘
"""
# todo 系统调用白名单 chroot等参数
command = "lrun" + \
" --max-cpu-time " + str(self._max_cpu_time / 1000.0) + \
" --max-real-time " + str(self._max_real_time / 1000.0) + \
" --max-memory " + str(self._max_memory * 1000 * 1000) + \
" --network false" + \
" --uid " + str(lrun_uid) + \
" --gid " + str(lrun_gid)
execute_command = self._language["execute_command"].format(exe_path=self._exe_path)
command += (" " +
execute_command +
# 0就是stdin
" 0<" + self._test_case_dir + str(test_case_id) + ".in" +
# 1就是stdout
" 1>" + judger_workspace + str(test_case_id) + ".out" +
# 3是stderr，包含lrun的输出和程序的异常输出
" 3>&2")
return command
def _parse_lrun_output(self, output):
# 要注意的是 lrun把结果输出到了stderr，所以有些情况下lrun的输出可能与程序的一些错误输出的混合的，要先分离一下
error = None
# 倒序找到MEMORY的位置
output_start = output.rfind("MEMORY")
if output_start == -1:
raise JudgeClientError("Lrun result parse error")
# 如果不是0，说明lrun输出前面有输出，也就是程序的stderr有内容
if output_start != 0:
error = output[0:output_start]
# 分离出lrun的输出
output = output[output_start:]
return error, parse_lrun_output(output)
def _compare_output(self, test_case_id):
test_case_md5 = self._test_case_info["test_cases"][str(test_case_id)]["output_md5"]
output_path = judger_workspace + str(test_case_id) + ".out"
try:
f = open(output_path, "rb")
except IOError:
# 文件不存在等引发的异常 返回结果错误
return False
# 计算输出文件的md5 和之前测试用例文件的md5进行比较
md5 = hashlib.md5()
while True:
data = f.read(2 ** 8)
if not data:
break
md5.update(data)
# 对比文件是否一致
# todo 去除最后的空行
return md5.hexdigest() == test_case_md5
def _judge_one(self, test_case_id):
# 运行lrun程序 接收返回值
command = self._generate_command(test_case_id)
status_code, output = commands.getstatusoutput(command)
if status_code:
raise JudgeClientError(output)
error, run_result = self._parse_lrun_output(output)
run_result["test_case_id"] = test_case_id
# 如果返回值非0 或者信号量不是0 或者程序的stderr有输出 代表非正常结束
if run_result["exit_code"] or run_result["term_sig"] or run_result["siginaled"] or error:
run_result["result"] = result["runtime_error"]
return run_result
# 代表内存或者时间超过限制了
if run_result["exceed"]:
if run_result["exceed"] == "memory":
run_result["result"] = result["memory_limit_exceeded"]
elif run_result["exceed"] in ["cpu_time", "real_time"]:
run_result["result"] = result["time_limit_exceeded"]
else:
raise JudgeClientError("Error exceeded type: " + run_result["exceed"])
return run_result
# 下面就是代码正常运行了 需要判断代码的输出是否正确
if self._compare_output(test_case_id):
run_result["result"] = result["accepted"]
else:
run_result["result"] = result["wrong_answer"]
return run_result
def run(self):
# 添加到任务队列
_results = []
results = []
for i in range(self._test_case_info["test_case_number"]):
_results.append(self._pool.apply_async(_run, (self, i + 1)))
self._pool.close()
self._pool.join()
for item in _results:
# 注意多进程中的异常只有在get()的时候才会被引发
# http://stackoverflow.com/questions/22094852/how-to-catch-exceptions-in-workers-in-multiprocessing
try:
results.append(item.get())
except Exception as e:
# todo logging
print e
results.append({"result": result["system_error"]})
return results
def __getstate__(self):
# 不同的pool之间进行pickle的时候要排除自己，否则报错
# http://stackoverflow.com/questions/25382455/python-notimplementederror-pool-objects-cannot-be-passed-between-processes
self_dict = self.__dict__.copy()
del self_dict['_pool']
return self_dict

36
judge/judger/compiler.py
View File

@@ -1,36 +0,0 @@
# coding=utf-8
import commands
from settings import lrun_uid, lrun_gid
from judge_exceptions import CompileError, JudgeClientError
from utils import parse_lrun_output
def compile_(language_item, src_path, exe_path):
compile_command = language_item["compile_command"].format(src_path=src_path, exe_path=exe_path)
# 防止编译器卡死 或者 include </dev/random>等
execute_command = "lrun" + \
" --max-real-time 5" + \
" --uid " + str(lrun_uid) + \
" --gid " + str(lrun_gid) + \
" " + \
compile_command + \
" 3>&2"
status, output = commands.getstatusoutput(execute_command)
output_start = output.rfind("MEMORY")
if output_start == -1:
raise JudgeClientError("Error running compiler in lrun")
# 返回值不为0 或者 stderr中lrun的输出之前有东西
if status or output_start:
raise CompileError(output[0:output_start])
parse_result = parse_lrun_output(output)
if parse_result["exit_code"] or parse_result["term_sig"] or parse_result["siginaled"] or parse_result["exceed"]:
raise CompileError("Compile error")
return exe_path

9
judge/judger/judge_exceptions.py
View File

@@ -1,9 +0,0 @@
# coding=utf-8
class JudgeClientError(Exception):
pass
class CompileError(Exception):
pass

28
judge/judger/language.py
View File

@@ -1,28 +0,0 @@
# coding=utf-8
languages = {
1: {
"name": "c",
"src_name": "main.c",
"code": 1,
"compile_command": "gcc -DONLINE_JUDGE -O2 -w -std=c99 -pipe {src_path} -lm -o {exe_path}main",
"execute_command": "{exe_path}main"
},
2: {
"name": "cpp",
"src_name": "main.cpp",
"code": 2,
"compile_command": "g++ -DONLINE_JUDGE -O2 -w -std=c++11 -pipe {src_path} -lm -o {exe_path}main",
"execute_command": "{exe_path}main"
},
3: {
"name": "java",
"src_name": "Main.java",
"code": 3,
"compile_command": "javac {src_path} -d {exe_path}",
"execute_command": "java -cp {exe_path} Main"
}
}

14
judge/judger/result.py
View File

@@ -1,14 +0,0 @@
# coding=utf-8
result = {
"accepted": 0,
"runtime_error": 1,
"time_limit_exceeded": 2,
"memory_limit_exceeded": 3,
"compile_error": 4,
"format_error": 5,
"wrong_answer": 6,
"system_error": 7,
"waiting": 8
}

57
judge/judger/run.py
View File

@@ -1,57 +0,0 @@
# coding=utf-8
import sys
import pymongo
from bson.objectid import ObjectId
from client import JudgeClient
from language import languages
from compiler import compile_
from result import result
from settings import judger_workspace
from oj import settings
# 简单的解析命令行参数
# 参数有 -solution_id -time_limit -memory_limit -test_case_id
# 获取到的值是['xxx.py', '-solution_id', '1111', '-time_limit', '1000', '-memory_limit', '100', '-test_case_id', 'aaaa']
args = sys.argv
solution_id = args[2]
time_limit = args[4]
memory_limit = args[6]
test_case_id = args[8]
mongodb_setting = settings.DATABASES["mongodb"]
connection = pymongo.MongoClient(host=mongodb_setting["HOST"], port=mongodb_setting["PORT"])
collection = connection["oj"]["oj_submission"]
submission = collection.find_one({"_id": ObjectId(solution_id)})
if not submission:
exit()
# 将代码写入文件
language = languages[submission["language"]]
src_path = judger_workspace + "run/" + language["src_name"]
f = open(src_path, "w")
f.write(submission["code"])
f.close()
# 编译
try:
exe_path = compile_(language, src_path, judger_workspace + "run/")
except Exception as e:
print e
print [{"result": result["compile_error"]}]
exit()
client = JudgeClient(language_code=language,
exe_path=exe_path,
max_cpu_time=int(time_limit),
max_real_time=int(time_limit) * 2,
max_memory=int(memory_limit),
test_case_dir="/var/judger/test_case/" + str(test_case_id) + "/")
print client.run()

15
judge/judger/settings.py
View File

@@ -1,15 +0,0 @@
# coding=utf-8
# 单个判题端最多同时运行的程序个数，因为判题端会同时运行多组测试数据，比如一共有5组测试数据
# 如果MAX_RUNNING_NUMBER大于等于5，那么这5组数据就会同时进行评测，然后返回结果。
# 如果MAX_RUNNING_NUMBER小于5，为3，那么就会同时运行前三组测试数据，然后再运行后两组数据
# 这样可以避免同时运行的程序过多导致的cpu占用太高
max_running_number = 10
# lrun运行用户的uid
lrun_uid = 1001
# lrun用户组gid
lrun_gid = 1002
#judger工作目录
judger_workspace = "/var/judger/"

40
judge/judger/utils.py
View File

@@ -1,40 +0,0 @@
# coding=utf-8
from judge_exceptions import JudgeClientError
def parse_lrun_output(output):
lines = output.split("\n")
if len(lines) != 7:
raise JudgeClientError("Lrun result parse error")
result = {}
# 将lrun输出的各种带下划线 不带下划线的字符串统一处理
translate = {"MEMORY": "memory",
"CPUTIME": "cpu_time",
"CPU_TIME": "cpu_time",
"REALTIME": "real_time",
"REAL_TIME": "real_time",
"TERMSIG": "term_sig",
"SIGNALED": "siginaled",
"EXITCODE": "exit_code",
"EXCEED": "exceed"}
for line in lines:
name = line[:9].strip(" ")
value = line[9:]
if name == "MEMORY":
result[translate[name]] = int(value)
elif name == "CPUTIME":
result[translate[name]] = int(float(value) * 1000)
elif name == "REALTIME":
result[translate[name]] = int(float(value) * 1000)
elif name == "EXITCODE":
result[translate[name]] = int(value)
elif name == "TERMSIG":
result[translate[name]] = int(value)
elif name == "SIGNALED":
result[translate[name]] = int(value)
elif name == "EXCEED":
if value == "none":
result[translate[name]] = None
else:
result[translate[name]] = translate[value]
return result