重构
This commit is contained in:
virusdefender
176
judge/client.py
176
judge/client.py
@@ -1,176 +0,0 @@
|
||||
# coding=utf-8
|
||||
import os
|
||||
import json
|
||||
import hashlib
|
||||
import judger
|
||||
import spj_client
|
||||
|
||||
from multiprocessing import Pool
|
||||
|
||||
from settings import max_running_number
|
||||
from language import languages
|
||||
from result import result
|
||||
from judge_exceptions import JudgeClientError
|
||||
from logger import logger
|
||||
|
||||
|
||||
# 下面这个函数作为代理访问实例变量，否则Python2会报错，是Python2的已知问题
|
||||
# http://stackoverflow.com/questions/1816958/cant-pickle-type-instancemethod-when-using-pythons-multiprocessing-pool-ma/7309686
|
||||
def _run(instance, test_case_id):
|
||||
return instance._judge_one(test_case_id)
|
||||
|
||||
|
||||
class JudgeClient(object):
|
||||
def __init__(self, language_code, exe_path, max_cpu_time, max_memory, test_case_dir, judge_base_path, spj_path):
|
||||
"""
|
||||
:param language_code: 语言编号
|
||||
:param exe_path: 可执行文件路径
|
||||
:param max_cpu_time: 最大cpu时间，单位ms
|
||||
:param max_memory: 最大内存，单位字节，直接传给judger.run方法
|
||||
:param test_case_dir: 测试用例文件夹路径
|
||||
:return:返回结果list
|
||||
"""
|
||||
self._language = languages[language_code]
|
||||
self._exe_path = exe_path
|
||||
self._max_cpu_time = max_cpu_time
|
||||
|
||||
# 如果是Java, 就不在judger中限制内存分配了, 而是转移到Java运行参数中,
|
||||
# 参见 https://github.com/QingdaoU/OnlineJudge/issues/23
|
||||
# 这里给出3倍的限制, 是为了防止出现OutOfMemory异常导致误判为Runtime Error,
|
||||
# 如果实际使用超过了3倍, 就只能得到Runtime Error的结果了
|
||||
# 而最后会比较Java实际使用的内存和1.5倍的设定内存的大小
|
||||
self._real_max_memory = max_memory
|
||||
if self._language["name"] == "java":
|
||||
self._max_memory = judger.MEMORY_UNLIMITED
|
||||
self.execute_command = self._language["execute_command"].\
|
||||
format(exe_path=self._exe_path, max_memory=max_memory * 3).split(" ")
|
||||
else:
|
||||
self._max_memory = self._real_max_memory
|
||||
self.execute_command = self._language["execute_command"].format(exe_path=self._exe_path).split(" ")
|
||||
|
||||
self._test_case_dir = test_case_dir
|
||||
# 进程池
|
||||
self._pool = Pool(processes=max_running_number)
|
||||
# 测试用例配置项
|
||||
self._test_case_info = self._load_test_case_info()
|
||||
self._judge_base_path = judge_base_path
|
||||
self._spj_path = spj_path
|
||||
|
||||
def _load_test_case_info(self):
|
||||
# 读取测试用例信息 转换为dict
|
||||
try:
|
||||
f = open(os.path.join(self._test_case_dir, "info"))
|
||||
return json.loads(f.read())
|
||||
except IOError:
|
||||
raise JudgeClientError("Test case config file not found")
|
||||
except ValueError:
|
||||
raise JudgeClientError("Test case config file format error")
|
||||
|
||||
def _compare_output(self, test_case_id):
|
||||
test_case_config = self._test_case_info["test_cases"][str(test_case_id)]
|
||||
output_path = os.path.join(self._judge_base_path, str(test_case_id) + ".out")
|
||||
|
||||
try:
|
||||
f = open(output_path, "rb")
|
||||
except IOError:
|
||||
# 文件不存在等引发的异常 返回结果错误
|
||||
return "", False
|
||||
|
||||
if "striped_output_md5" not in test_case_config:
|
||||
# 计算输出文件的md5 和之前测试用例文件的md5进行比较
|
||||
# 兼容之前没有striped_output_md5的测试用例
|
||||
# 现在比较的是完整的文件
|
||||
md5 = hashlib.md5()
|
||||
while True:
|
||||
data = f.read(2 ** 8)
|
||||
if not data:
|
||||
break
|
||||
md5.update(data)
|
||||
output_md5 = md5.hexdigest()
|
||||
|
||||
return output_md5, output_md5 == test_case_config["output_md5"]
|
||||
else:
|
||||
# 这时候需要去除用户输出最后的空格和换行 再去比较md5
|
||||
md5 = hashlib.md5()
|
||||
# 比较和返回去除空格后的md5比较结果
|
||||
md5.update(f.read().rstrip())
|
||||
output_md5 = md5.hexdigest()
|
||||
return output_md5, output_md5 == test_case_config["striped_output_md5"]
|
||||
|
||||
def _judge_one(self, test_case_id):
|
||||
in_file = os.path.join(self._test_case_dir, str(test_case_id) + ".in")
|
||||
out_file = os.path.join(self._judge_base_path, str(test_case_id) + ".out")
|
||||
run_result = judger.run(path=self.execute_command[0],
|
||||
max_cpu_time=self._max_cpu_time,
|
||||
max_memory=self._max_memory,
|
||||
in_file=in_file,
|
||||
out_file=out_file,
|
||||
args=self.execute_command[1:],
|
||||
env=["PATH=" + os.environ["PATH"]],
|
||||
use_sandbox=self._language["use_sandbox"],
|
||||
use_nobody=True)
|
||||
run_result["test_case"] = test_case_id
|
||||
|
||||
# 对Java的特殊处理, 详见__init__函数中注释
|
||||
if self._language["name"] == "java" and run_result["memory"] > self._real_max_memory * 1.5:
|
||||
run_result["flag"] = 3
|
||||
|
||||
# 将judger返回的结果标志转换为本系统中使用的
|
||||
if run_result["flag"] == 0:
|
||||
if self._spj_path is None:
|
||||
output_md5, r = self._compare_output(test_case_id)
|
||||
if r:
|
||||
run_result["result"] = result["accepted"]
|
||||
else:
|
||||
run_result["result"] = result["wrong_answer"]
|
||||
run_result["output_md5"] = output_md5
|
||||
else:
|
||||
spj_result = spj_client.spj(path=self._spj_path,
|
||||
max_cpu_time=3 * self._max_cpu_time,
|
||||
max_memory=3 * self._real_max_memory,
|
||||
in_path=in_file,
|
||||
user_out_path=out_file)
|
||||
if spj_result["spj_result"] == spj_client.AC:
|
||||
run_result["result"] = result["accepted"]
|
||||
elif spj_result["spj_result"] == spj_client.WA:
|
||||
run_result["result"] = result["wrong_answer"]
|
||||
else:
|
||||
run_result["result"] = result["system_error"]
|
||||
run_result["error"] = "SPJ Crashed, return: %d, signal: %d" % \
|
||||
(spj_result["spj_result"], spj_result["signal"])
|
||||
|
||||
elif run_result["flag"] in [1, 2]:
|
||||
run_result["result"] = result["time_limit_exceeded"]
|
||||
elif run_result["flag"] == 3:
|
||||
run_result["result"] = result["memory_limit_exceeded"]
|
||||
elif run_result["flag"] == 4:
|
||||
run_result["result"] = result["runtime_error"]
|
||||
elif run_result["flag"] == 5:
|
||||
run_result["result"] = result["system_error"]
|
||||
return run_result
|
||||
|
||||
def run(self):
|
||||
# 添加到任务队列
|
||||
_results = []
|
||||
results = []
|
||||
for i in range(self._test_case_info["test_case_number"]):
|
||||
_results.append(self._pool.apply_async(_run, (self, i + 1)))
|
||||
self._pool.close()
|
||||
self._pool.join()
|
||||
for item in _results:
|
||||
# 注意多进程中的异常只有在get()的时候才会被引发
|
||||
# http://stackoverflow.com/questions/22094852/how-to-catch-exceptions-in-workers-in-multiprocessing
|
||||
try:
|
||||
results.append(item.get())
|
||||
except Exception as e:
|
||||
logger.error("system error")
|
||||
logger.error(e)
|
||||
results.append({"result": result["system_error"]})
|
||||
return results
|
||||
|
||||
def __getstate__(self):
|
||||
# 不同的pool之间进行pickle的时候要排除自己，否则报错
|
||||
# http://stackoverflow.com/questions/25382455/python-notimplementederror-pool-objects-cannot-be-passed-between-processes
|
||||
self_dict = self.__dict__.copy()
|
||||
del self_dict['_pool']
|
||||
return self_dict
|
||||
@@ -1,40 +0,0 @@
|
||||
# coding=utf-8
|
||||
import os
|
||||
import judger
|
||||
from judge_exceptions import CompileError
|
||||
from logger import logger
|
||||
|
||||
|
||||
def compile_(language_item, src_path, exe_path, judge_base_path, compile_spj=False):
|
||||
command_item = "spj_compile_command" if compile_spj else "compile_command"
|
||||
compile_command = language_item[command_item].format(src_path=src_path, exe_path=exe_path).split(" ")
|
||||
compiler = compile_command[0]
|
||||
compile_args = compile_command[1:]
|
||||
compiler_output_file = os.path.join(judge_base_path, "compiler.out")
|
||||
|
||||
compile_result = judger.run(path=compiler,
|
||||
in_file="/dev/null",
|
||||
out_file=compiler_output_file,
|
||||
max_cpu_time=language_item["compile_max_cpu_time"],
|
||||
max_memory=language_item["compile_max_memory"],
|
||||
args=compile_args,
|
||||
env=["PATH=" + os.environ["PATH"]],
|
||||
use_sandbox=False,
|
||||
use_nobody=True)
|
||||
|
||||
compile_output_handler = open(compiler_output_file)
|
||||
compile_output = compile_output_handler.read().strip()
|
||||
compile_output_handler.close()
|
||||
|
||||
if compile_result["flag"] != 0:
|
||||
logger.error("Compiler error")
|
||||
logger.error(compile_output)
|
||||
logger.error(str(compile_result))
|
||||
if compile_output:
|
||||
raise CompileError(compile_output)
|
||||
else:
|
||||
raise CompileError("Compile error, info: " + str(compile_result))
|
||||
else:
|
||||
if "error" in compile_output:
|
||||
raise CompileError(compile_output)
|
||||
return exe_path
|
||||
@@ -1,9 +0,0 @@
|
||||
# coding=utf-8
|
||||
|
||||
|
||||
class JudgeClientError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CompileError(Exception):
|
||||
pass
|
||||
@@ -1,42 +0,0 @@
|
||||
# coding=utf-8
|
||||
|
||||
|
||||
languages = {
|
||||
1: {
|
||||
"name": "c",
|
||||
"src_name": "main.c",
|
||||
"code": 1,
|
||||
"compile_max_cpu_time": 3000,
|
||||
"compile_max_memory": 128 * 1024 * 1024,
|
||||
"compile_command": "/usr/bin/gcc -DONLINE_JUDGE -O2 -w -fmax-errors=3 -std=c99 {src_path} -lm -o {exe_path}/main",
|
||||
"spj_compile_command": "/usr/bin/gcc -DONLINE_JUDGE -O2 -Werror -fmax-errors=3 -std=c99 {src_path} -lm -o {exe_path}",
|
||||
"execute_command": "{exe_path}/main",
|
||||
"use_sandbox": True
|
||||
},
|
||||
2: {
|
||||
"name": "cpp",
|
||||
"src_name": "main.cpp",
|
||||
"code": 2,
|
||||
"compile_max_cpu_time": 3000,
|
||||
"compile_max_memory": 256 * 1024 * 1024,
|
||||
"compile_command": "/usr/bin/g++ -DONLINE_JUDGE -O2 -w -fmax-errors=3 -std=c++11 {src_path} -lm -o {exe_path}/main",
|
||||
"spj_compile_command": "/usr/bin/g++ -DONLINE_JUDGE -O2 -Werror -fmax-errors=3 -std=c++11 {src_path} -lm -o {exe_path}",
|
||||
"execute_command": "{exe_path}/main",
|
||||
"use_sandbox": True
|
||||
},
|
||||
3: {
|
||||
"name": "java",
|
||||
"src_name": "Main.java",
|
||||
"code": 3,
|
||||
"compile_max_cpu_time": 3000,
|
||||
"compile_max_memory": 1024 * 1024 * 1024,
|
||||
"compile_command": "/usr/bin/javac {src_path} -d {exe_path} -J-Xss1m -J-XX:MaxPermSize=16M "
|
||||
"-J-XX:PermSize=8M -J-Xms16m -J-Xmx1024m -encoding UTF8",
|
||||
"execute_command": "/usr/bin/java -cp {exe_path} -Xss1M -XX:MaxPermSize=16M "
|
||||
"-XX:PermSize=8M -Xms16M -Xmx{max_memory} -Djava.security.manager "
|
||||
"-Djava.security.policy==policy -Djava.awt.headless=true Main",
|
||||
"use_sandbox": False
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# coding=utf-8
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG,
|
||||
format='%(asctime)s [%(threadName)s:%(thread)d] [%(name)s:%(lineno)d] [%(module)s:%(funcName)s] [%(levelname)s]- %(message)s',
|
||||
filename='log/judge.log')
|
||||
|
||||
logger = logging
|
||||
@@ -1,15 +0,0 @@
|
||||
# coding=utf-8
|
||||
|
||||
|
||||
# 这个映射关系是前后端通用的,判题服务器提供接口,也应该遵守这个,可能需要一些转换
|
||||
result = {
|
||||
"accepted": 0,
|
||||
"runtime_error": 1,
|
||||
"time_limit_exceeded": 2,
|
||||
"memory_limit_exceeded": 3,
|
||||
"compile_error": 4,
|
||||
"format_error": 5,
|
||||
"wrong_answer": 6,
|
||||
"system_error": 7,
|
||||
"waiting": 8
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
# coding=utf-8
|
||||
import os
|
||||
import socket
|
||||
import shutil
|
||||
|
||||
from logger import logger
|
||||
from client import JudgeClient
|
||||
from language import languages
|
||||
from compiler import compile_
|
||||
from result import result
|
||||
from settings import judger_workspace
|
||||
|
||||
|
||||
class JudgeInstanceRunner(object):
|
||||
|
||||
def run(self, token, submission_id, language_code, code, time_limit, memory_limit, test_case_id,
|
||||
spj, spj_language, spj_code, spj_version):
|
||||
language = languages[language_code]
|
||||
host_name = socket.gethostname()
|
||||
judge_base_path = os.path.join(judger_workspace, "run", submission_id)
|
||||
|
||||
if not token or token != os.environ.get("rpc_token"):
|
||||
if token:
|
||||
logger.info("Invalid token: " + token)
|
||||
return {"code": 2, "data": {"error": "Invalid token", "server": host_name}}
|
||||
|
||||
try:
|
||||
os.mkdir(judge_base_path)
|
||||
os.chmod(judge_base_path, 0777)
|
||||
|
||||
# 将代码写入文件
|
||||
src_path = os.path.join(judge_base_path, language["src_name"])
|
||||
f = open(src_path, "w")
|
||||
f.write(code.encode("utf8"))
|
||||
f.close()
|
||||
except Exception as e:
|
||||
shutil.rmtree(judge_base_path, ignore_errors=True)
|
||||
return {"code": 2, "data": {"error": str(e), "server": host_name}}
|
||||
|
||||
# 编译
|
||||
try:
|
||||
exe_path = compile_(language_item=language, src_path=src_path,
|
||||
exe_path=judge_base_path, judge_base_path=judge_base_path, compile_spj=False)
|
||||
except Exception as e:
|
||||
shutil.rmtree(judge_base_path, ignore_errors=True)
|
||||
return {"code": 1, "data": {"error": str(e), "server": host_name}}
|
||||
|
||||
test_case_dir = os.path.join(judger_workspace, "test_case", test_case_id)
|
||||
|
||||
# SPJ相关
|
||||
if spj:
|
||||
spj_path = os.path.join(test_case_dir, "spj-" + spj_version)
|
||||
if "spj-" + spj_version not in os.listdir(test_case_dir):
|
||||
spj_language_item = languages[spj_language]
|
||||
spj_code_path = os.path.join(test_case_dir, "spj-" + spj_language_item["src_name"])
|
||||
|
||||
f = open(spj_code_path, "w")
|
||||
f.write(spj_code.encode("utf8"))
|
||||
f.close()
|
||||
|
||||
try:
|
||||
compile_(language_item=languages[spj_language], src_path=spj_code_path,
|
||||
exe_path=spj_path,
|
||||
judge_base_path=judge_base_path, compile_spj=True)
|
||||
except Exception as e:
|
||||
return {"code": 2, "data": {"error": "SPJ Compile error: " + str(e), "server": host_name}}
|
||||
else:
|
||||
spj_path = None
|
||||
|
||||
# 运行
|
||||
try:
|
||||
client = JudgeClient(language_code=language_code,
|
||||
exe_path=exe_path,
|
||||
max_cpu_time=int(time_limit),
|
||||
max_memory=int(memory_limit) * 1024 * 1024,
|
||||
test_case_dir=test_case_dir,
|
||||
judge_base_path=judge_base_path, spj_path=spj_path)
|
||||
judge_result = {"result": result["accepted"], "info": client.run(),
|
||||
"accepted_answer_time": None, "server": host_name}
|
||||
|
||||
for item in judge_result["info"]:
|
||||
if item["result"] != 0:
|
||||
judge_result["result"] = item["result"]
|
||||
break
|
||||
else:
|
||||
l = sorted(judge_result["info"], key=lambda k: k["cpu_time"])
|
||||
judge_result["accepted_answer_time"] = l[-1]["cpu_time"]
|
||||
return {"code": 0, "data": judge_result}
|
||||
except Exception as e:
|
||||
return {"code": 2, "data": {"error": str(e), "server": host_name}}
|
||||
finally:
|
||||
shutil.rmtree(judge_base_path, ignore_errors=True)
|
||||
@@ -1,13 +0,0 @@
|
||||
# coding=utf-8
|
||||
import SocketServer
|
||||
from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
|
||||
from runner import JudgeInstanceRunner
|
||||
|
||||
|
||||
class AsyncXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer):
|
||||
pass
|
||||
|
||||
|
||||
server = AsyncXMLRPCServer(('0.0.0.0', 8080), SimpleXMLRPCRequestHandler, allow_none=True)
|
||||
server.register_instance(JudgeInstanceRunner())
|
||||
server.serve_forever()
|
||||
@@ -1,9 +0,0 @@
|
||||
# coding=utf-8
|
||||
# 单个判题端最多同时运行的程序个数，因为判题端会同时运行多组测试数据，比如一共有5组测试数据
|
||||
# 如果MAX_RUNNING_NUMBER大于等于5，那么这5组数据就会同时进行评测，然后返回结果。
|
||||
# 如果MAX_RUNNING_NUMBER小于5，为3，那么就会同时运行前三组测试数据，然后再运行后两组数据
|
||||
# 这样可以避免同时运行的程序过多导致的cpu占用太高
|
||||
max_running_number = 10
|
||||
|
||||
# judger工作目录
|
||||
judger_workspace = "/var/judger/"
|
||||
@@ -1,26 +0,0 @@
|
||||
# coding=utf-8
|
||||
import os
|
||||
import judger
|
||||
|
||||
WA = 1
|
||||
AC = 0
|
||||
SPJ_ERROR = -1
|
||||
|
||||
|
||||
def file_exists(path):
|
||||
return os.path.exists(path)
|
||||
|
||||
|
||||
def spj(path, max_cpu_time, max_memory, in_path, user_out_path):
|
||||
if file_exists(in_path) and file_exists(user_out_path):
|
||||
result = judger.run(path=path, in_file=in_path, out_file="/tmp/spj.out",
|
||||
max_cpu_time=max_cpu_time, max_memory=max_memory,
|
||||
args=[in_path, user_out_path], env=["PATH=" + os.environ.get("PATH", "")],
|
||||
use_sandbox=True, use_nobody=True)
|
||||
if result["signal"] == 0 and result["exit_status"] in [AC, WA, SPJ_ERROR]:
|
||||
result["spj_result"] = result["exit_status"]
|
||||
else:
|
||||
result["spj_result"] = SPJ_ERROR
|
||||
return result
|
||||
else:
|
||||
raise ValueError("in_path or user_out_path does not exist")
|
||||
Reference in New Issue
Block a user