修改判题的目录结构

judge/judger/client.py

@@ -0,0 +1,173 @@
+# coding=utf-8
+import json
+import commands
+import hashlib
+from multiprocessing import Pool
+
+from settings import max_running_number, lrun_gid, lrun_uid, judger_workspace
+from language import languages
+from result import result
+from compiler import compile_
+from judge_exceptions import JudgeClientError, CompileError
+from utils import parse_lrun_output
+
+
+# 下面这个函数作为代理访问实例变量，否则Python2会报错，是Python2的已知问题
+# http://stackoverflow.com/questions/1816958/cant-pickle-type-instancemethod-when-using-pythons-multiprocessing-pool-ma/7309686
+def _run(instance, test_case_id):
+    return instance._judge_one(test_case_id)
+
+
+class JudgeClient(object):
+    def __init__(self, language_code, exe_path, max_cpu_time,
+                 max_real_time, max_memory, test_case_dir):
+        """
+        :param language_code: 语言编号
+        :param exe_path: 可执行文件路径
+        :param max_cpu_time: 最大cpu时间，单位ms
+        :param max_real_time: 最大执行时间，单位ms
+        :param max_memory: 最大内存，单位MB
+        :param test_case_dir: 测试用例文件夹路径
+        :return:返回结果list
+        """
+        self._language = languages[language_code]
+        self._exe_path = exe_path
+        self._max_cpu_time = max_cpu_time
+        self._max_real_time = max_real_time
+        self._max_memory = max_memory
+        self._test_case_dir = test_case_dir
+        # 进程池
+        self._pool = Pool(processes=max_running_number)
+        # 测试用例配置项
+        self._test_case_info = self._load_test_case_info()
+
+    def _load_test_case_info(self):
+        # 读取测试用例信息 转换为dict
+        try:
+            f = open(self._test_case_dir + "info")
+            return json.loads(f.read())
+        except IOError:
+            raise JudgeClientError("Test case config file not found")
+        except ValueError:
+            raise JudgeClientError("Test case config file format error")
+
+    def _generate_command(self, test_case_id):
+        """
+        设置相关运行限制 进制访问网络 如果启用tmpfs 就把代码输出写入tmpfs，否则写入硬盘
+        """
+        # todo 系统调用白名单 chroot等参数
+        command = "lrun" + \
+                  " --max-cpu-time " + str(self._max_cpu_time / 1000.0) + \
+                  " --max-real-time " + str(self._max_real_time / 1000.0 * 2) + \
+                  " --max-memory " + str(self._max_memory * 1000 * 1000) + \
+                  " --network false" + \
+                  " --uid " + str(lrun_uid) + \
+                  " --gid " + str(lrun_gid)
+
+        execute_command = self._language["execute_command"].format(exe_path=self._exe_path)
+
+        command += (" " +
+                    execute_command +
+                    # 0就是stdin
+                    " 0<" + self._test_case_dir + str(test_case_id) + ".in" +
+                    # 1就是stdout
+                    " 1>" + judger_workspace + str(test_case_id) + ".out" +
+                    # 3是stderr，包含lrun的输出和程序的异常输出
+                    " 3>&2")
+        return command
+
+    def _parse_lrun_output(self, output):
+        # 要注意的是 lrun把结果输出到了stderr，所以有些情况下lrun的输出可能与程序的一些错误输出的混合的，要先分离一下
+        error = None
+        # 倒序找到MEMORY的位置
+        output_start = output.rfind("MEMORY")
+        if output_start == -1:
+            raise JudgeClientError("Lrun result parse error")
+        # 如果不是0，说明lrun输出前面有输出，也就是程序的stderr有内容
+        if output_start != 0:
+            error = output[0:output_start]
+        # 分离出lrun的输出
+        output = output[output_start:]
+
+        return error, parse_lrun_output(output)
+
+    def _compare_output(self, test_case_id):
+        test_case_md5 = self._test_case_info["test_cases"][str(test_case_id)]["output_md5"]
+        output_path = judger_workspace + str(test_case_id) + ".out"
+
+        try:
+            f = open(output_path, "rb")
+        except IOError:
+            # 文件不存在等引发的异常 返回结果错误
+            return False
+
+        # 计算输出文件的md5 和之前测试用例文件的md5进行比较
+        md5 = hashlib.md5()
+        while True:
+            data = f.read(2 ** 8)
+            if not data:
+                break
+            md5.update(data)
+
+        # 对比文件是否一致
+        # todo 去除最后的空行
+        return md5.hexdigest() == test_case_md5
+
+    def _judge_one(self, test_case_id):
+        # 运行lrun程序 接收返回值
+        command = self._generate_command(test_case_id)
+        status_code, output = commands.getstatusoutput(command)
+        if status_code:
+            raise JudgeClientError(output)
+        error, run_result = self._parse_lrun_output(output)
+
+        run_result["test_case_id"] = test_case_id
+
+        # 如果返回值非0 或者信号量不是0 或者程序的stderr有输出 代表非正常结束
+        if run_result["exit_code"] or run_result["term_sig"] or run_result["siginaled"] or error:
+            run_result["result"] = result["runtime_error"]
+            return run_result
+
+        # 代表内存或者时间超过限制了
+        if run_result["exceed"]:
+            if run_result["exceed"] == "memory":
+                run_result["result"] = result["memory_limit_exceeded"]
+            elif run_result["exceed"] in ["cpu_time", "real_time"]:
+                run_result["result"] = result["time_limit_exceeded"]
+            else:
+                raise JudgeClientError("Error exceeded type: " + run_result["exceed"])
+            return run_result
+
+        # 下面就是代码正常运行了 需要判断代码的输出是否正确
+        if self._compare_output(test_case_id):
+            run_result["result"] = result["accepted"]
+        else:
+            run_result["result"] = result["wrong_answer"]
+
+        return run_result
+
+    def run(self):
+        # 添加到任务队列
+        _results = []
+        results = []
+        for i in range(self._test_case_info["test_case_number"]):
+            _results.append(self._pool.apply_async(_run, (self, i + 1)))
+        self._pool.close()
+        self._pool.join()
+        for item in _results:
+            # 注意多进程中的异常只有在get()的时候才会被引发
+            # http://stackoverflow.com/questions/22094852/how-to-catch-exceptions-in-workers-in-multiprocessing
+            try:
+                results.append(item.get())
+            except Exception as e:
+                # todo logging
+                print e
+                results.append({"result": result["system_error"]})
+        return results
+
+    def __getstate__(self):
+        # 不同的pool之间进行pickle的时候要排除自己，否则报错
+        # http://stackoverflow.com/questions/25382455/python-notimplementederror-pool-objects-cannot-be-passed-between-processes
+        self_dict = self.__dict__.copy()
+        del self_dict['_pool']
+        return self_dict

judge/judger/compiler.py

@@ -0,0 +1,35 @@
+# coding=utf-8
+import commands
+
+from settings import lrun_uid, lrun_gid
+from judge_exceptions import CompileError, JudgeClientError
+from utils import parse_lrun_output
+
+
+def compile_(language_item, src_path, exe_path):
+    compile_command = language_item["compile_command"].format(src_path=src_path, exe_path=exe_path)
+
+    # 防止编译器卡死 或者 include </dev/random>等
+    execute_command = "lrun" + \
+                      " --max-real-time 5" + \
+                      " --uid " + str(lrun_uid) + \
+                      " --gid " + str(lrun_gid) + \
+                      " " + \
+                      compile_command + \
+                      " 3>&2"
+    status, output = commands.getstatusoutput(execute_command)
+
+    output_start = output.rfind("MEMORY")
+
+    if output_start == -1:
+        raise JudgeClientError("Error running compiler in lrun")
+
+    # 返回值不为0 或者 stderr中lrun的输出之前有东西
+    if status or output_start:
+        raise CompileError(output[0:output_start])
+
+    parse_result = parse_lrun_output(output)
+
+    if parse_result["exit_code"] or parse_result["term_sig"] or parse_result["siginaled"] or parse_result["exceed"]:
+        raise CompileError("Compile error")
+    return exe_path

judge/judger/judge_exceptions.py

@@ -0,0 +1,9 @@
+# coding=utf-8
+
+
+class JudgeClientError(Exception):
+    pass
+
+
+class CompileError(Exception):
+    pass

judge/judger/language.py

@@ -0,0 +1,28 @@
+# coding=utf-8
+
+
+languages = {
+    1: {
+        "name": "c",
+        "src_name": "main.c",
+        "code": 1,
+        "compile_command": "gcc -DONLINE_JUDGE -O2 -w -std=c99 {src_path} -lm -o {exe_path}main",
+        "execute_command": "{exe_path}main"
+    },
+    2: {
+        "name": "cpp",
+        "src_name": "main.cpp",
+        "code": 2,
+        "compile_command": "g++ -DONLINE_JUDGE -O2 -w -std=c++11 {src_path} -lm -o {exe_path}main",
+        "execute_command": "{exe_path}main"
+    },
+    3: {
+        "name": "java",
+        "src_name": "Main.java",
+        "code": 3,
+        "compile_command": "javac {src_path} -d {exe_path}",
+        "execute_command": "java -cp {exe_path} Main"
+    }
+}
+
+

judge/judger/result.py

@@ -0,0 +1,14 @@
+# coding=utf-8
+
+
+result = {
+    "accepted": 0,
+    "runtime_error": 1,
+    "time_limit_exceeded": 2,
+    "memory_limit_exceeded": 3,
+    "compile_error": 4,
+    "format_error": 5,
+    "wrong_answer": 6,
+    "system_error": 7,
+    "waiting": 8
+}

judge/judger/run.py

@@ -0,0 +1,83 @@
+# coding=utf-8
+import sys
+import os
+import pymongo
+
+from bson.objectid import ObjectId
+
+from client import JudgeClient
+from language import languages
+from compiler import compile_
+from result import result
+from settings import judger_workspace
+
+sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
+
+from judger_controller.settings import mongodb_config
+
+
+# 简单的解析命令行参数
+# 参数有 -solution_id -time_limit -memory_limit -test_case_id
+# 获取到的值是['xxx.py', '-solution_id', '1111', '-time_limit', '1000', '-memory_limit', '100', '-test_case_id', 'aaaa']
+args = sys.argv
+submission_id = args[2]
+time_limit = args[4]
+memory_limit = args[6]
+test_case_id = args[8]
+
+connection = pymongo.MongoClient(host=mongodb_config["host"], port=mongodb_config["port"])
+collection = connection["oj"]["oj_submission"]
+
+submission = collection.find_one({"_id": ObjectId(submission_id)})
+if not submission:
+    exit()
+connection.close()
+
+# 将代码写入文件
+language = languages[submission["language"]]
+src_path = judger_workspace + "run/" + language["src_name"]
+f = open(src_path, "w")
+f.write(submission["code"].encode("utf8"))
+f.close()
+
+# 编译
+try:
+    exe_path = compile_(language, src_path, judger_workspace + "run/")
+except Exception as e:
+    print e
+    connection = pymongo.MongoClient(host=mongodb_config["host"], port=mongodb_config["port"])
+    collection = connection["oj"]["oj_submission"]
+    data = {"result": result["compile_error"], "info": str(e)}
+    collection.find_one_and_update({"_id": ObjectId(submission_id)}, {"$set": data})
+    connection.close()
+    exit()
+
+print "Compile successfully"
+# 运行
+try:
+    client = JudgeClient(language_code=submission["language"],
+                         exe_path=exe_path,
+                         max_cpu_time=int(time_limit),
+                         max_real_time=int(time_limit) * 2,
+                         max_memory=int(memory_limit),
+                         test_case_dir=judger_workspace + "test_case/" + test_case_id + "/")
+    judge_result = {"result": result["accepted"], "info": client.run()}
+
+    for item in judge_result["info"]:
+        if item["result"]:
+            judge_result["result"] = item["result"]
+            break
+    else:
+        l = sorted(judge_result["info"], key=lambda k: k["cpu_time"])
+        judge_result["accepted_answer_info"] = {"time": l[-1]["cpu_time"]}
+
+except Exception as e:
+    print e
+    judge_result = {"result": result["system_error"], "info": str(e)}
+
+print "Run successfully"
+print judge_result
+connection = pymongo.MongoClient(host=mongodb_config["host"], port=mongodb_config["port"])
+collection = connection["oj"]["oj_submission"]
+collection.find_one_and_update({"_id": ObjectId(submission_id)}, {"$set": judge_result})
+connection.close()

judge/judger/settings.py

@@ -0,0 +1,17 @@
+# coding=utf-8
+# 单个判题端最多同时运行的程序个数，因为判题端会同时运行多组测试数据，比如一共有5组测试数据
+# 如果MAX_RUNNING_NUMBER大于等于5，那么这5组数据就会同时进行评测，然后返回结果。
+# 如果MAX_RUNNING_NUMBER小于5，为3，那么就会同时运行前三组测试数据，然后再运行后两组数据
+# 这样可以避免同时运行的程序过多导致的cpu占用太高
+max_running_number = 10
+
+# lrun运行用户的uid
+lrun_uid = 1001
+
+# lrun用户组gid
+lrun_gid = 1002
+
+# judger工作目录
+judger_workspace = "/var/judger/"
+
+

judge/judger/utils.py

@@ -0,0 +1,40 @@
+# coding=utf-8
+from judge_exceptions import JudgeClientError
+
+
+def parse_lrun_output(output):
+    lines = output.split("\n")
+    if len(lines) != 7:
+        raise JudgeClientError("Lrun result parse error")
+    result = {}
+    # 将lrun输出的各种带下划线 不带下划线的字符串统一处理
+    translate = {"MEMORY": "memory",
+                 "CPUTIME": "cpu_time",
+                 "CPU_TIME": "cpu_time",
+                 "REALTIME": "real_time",
+                 "REAL_TIME": "real_time",
+                 "TERMSIG": "term_sig",
+                 "SIGNALED": "siginaled",
+                 "EXITCODE": "exit_code",
+                 "EXCEED": "exceed"}
+    for line in lines:
+        name = line[:9].strip(" ")
+        value = line[9:]
+        if name == "MEMORY":
+            result[translate[name]] = int(value)
+        elif name == "CPUTIME":
+            result[translate[name]] = int(float(value) * 1000)
+        elif name == "REALTIME":
+            result[translate[name]] = int(float(value) * 1000)
+        elif name == "EXITCODE":
+            result[translate[name]] = int(value)
+        elif name == "TERMSIG":
+            result[translate[name]] = int(value)
+        elif name == "SIGNALED":
+            result[translate[name]] = int(value)
+        elif name == "EXCEED":
+            if value == "none":
+                result[translate[name]] = None
+            else:
+                result[translate[name]] = translate[value]
+    return result

judge/judger_controller/README.md

@@ -0,0 +1 @@
+celery -A judge.controller worker -l DEBUG

judge/judger_controller/__init__.py

@@ -0,0 +1 @@
+# coding=utf-8

judge/judger_controller/celery.py

@@ -0,0 +1,10 @@
+# coding=utf-8
+from __future__ import absolute_import
+from celery import Celery
+from .settings import redis_config
+
+app = Celery("judge", broker="redis://" +
+                             redis_config["host"] + ":" +
+                             str(redis_config["port"]) +
+                             "/" + str(redis_config["db"]),
+             include=["judge.judger_controller.tasks"])

judge/judger_controller/settings.py

@@ -0,0 +1,25 @@
+# coding=utf-8
+redis_config = {
+    "host": "127.0.0.1",
+    "port": 6379,
+    "db": 0
+}
+
+
+docker_config = {
+    "image_name": "judger",
+    "docker_path": "docker",
+    "shell": True
+}
+
+
+test_case_dir = "/Users/virusdefender/Desktop/test_case/"
+source_code_dir = "/Users/virusdefender/Desktop/"
+
+
+mongodb_config = {
+    "host": "192.168.59.3",
+    "username": "root",
+    "password": "root",
+    "port": 27017
+}

judge/judger_controller/tasks.py

@@ -0,0 +1,31 @@
+# coding=utf-8
+# from __future__ import absolute_import
+import pymongo
+from bson import ObjectId
+import subprocess32 as subprocess
+from ..judger.result import result
+from ..judger_controller.celery import app
+from settings import docker_config, source_code_dir, test_case_dir, mongodb_config
+
+
+@app.task
+def judge(submission_id, time_limit, memory_limit, test_case_id):
+    try:
+        command = "%s run -t -i --privileged --rm=true " \
+                  "-v %s:/var/judger/test_case/ " \
+                  "-v %s:/var/judger/code/ " \
+                  "%s " \
+                  "python judge/judger/run.py " \
+                  "--solution_id %s --time_limit %s --memory_limit %s --test_case_id %s" % \
+                  (docker_config["docker_path"],
+                   test_case_dir,
+                   source_code_dir,
+                   docker_config["image_name"],
+                   submission_id, str(time_limit), str(memory_limit), test_case_id)
+        subprocess.call(command, timeout=(time_limit / 1000.0 * 10), shell=docker_config["shell"])
+    except Exception as e:
+        connection = pymongo.MongoClient(host=mongodb_config["host"], port=mongodb_config["port"])
+        collection = connection["oj"]["oj_submission"]
+        data = {"result": result["system_error"], "info": str(e)}
+        collection.find_one_and_update({"_id": ObjectId(submission_id)}, {"$set": data})
+        connection.close()