From f56d75cf9139a22a7196464a7ab426738d4561b2 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sun, 20 Nov 2016 19:18:25 +0800 Subject: [PATCH] add judge_server related api --- conf/migrations/0003_judgeserver.py | 33 ++++++++++ conf/migrations/0004_auto_20161120_1834.py | 25 ++++++++ conf/migrations/0005_judgeservertoken.py | 25 ++++++++ conf/models.py | 22 +++++++ conf/serializers.py | 27 ++++++-- conf/tests.py | 53 +++++++++++++++- conf/urls/admin.py | 3 +- conf/urls/oj.py | 4 +- conf/views.py | 73 +++++++++++++++++++++- 9 files changed, 254 insertions(+), 11 deletions(-) create mode 100644 conf/migrations/0003_judgeserver.py create mode 100644 conf/migrations/0004_auto_20161120_1834.py create mode 100644 conf/migrations/0005_judgeservertoken.py diff --git a/conf/migrations/0003_judgeserver.py b/conf/migrations/0003_judgeserver.py new file mode 100644 index 0000000..f8bf93e --- /dev/null +++ b/conf/migrations/0003_judgeserver.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.10 on 2016-11-19 14:27 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('conf', '0002_auto_20161119_1657'), + ] + + operations = [ + migrations.CreateModel( + name='JudgeServer', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('hostname', models.CharField(max_length=64)), + ('ip', models.CharField(max_length=32)), + ('judger_version', models.CharField(max_length=24)), + ('cpu_core', models.IntegerField()), + ('memory_usage', models.FloatField()), + ('cpu_usage', models.FloatField()), + ('last_heartbeat', models.DateTimeField()), + ('create_time', models.DateTimeField(auto_now_add=True)), + ('task_number', models.IntegerField()), + ], + options={ + 'db_table': 'judge_server', + }, + ), + ] diff --git a/conf/migrations/0004_auto_20161120_1834.py b/conf/migrations/0004_auto_20161120_1834.py new file mode 100644 index 0000000..f6d88da --- /dev/null +++ b/conf/migrations/0004_auto_20161120_1834.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.10 on 2016-11-20 10:34 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('conf', '0003_judgeserver'), + ] + + operations = [ + migrations.AddField( + model_name='judgeserver', + name='service_url', + field=models.CharField(blank=True, max_length=128, null=True), + ), + migrations.AlterField( + model_name='judgeserver', + name='ip', + field=models.CharField(blank=True, max_length=32, null=True), + ), + ] diff --git a/conf/migrations/0005_judgeservertoken.py b/conf/migrations/0005_judgeservertoken.py new file mode 100644 index 0000000..7c93010 --- /dev/null +++ b/conf/migrations/0005_judgeservertoken.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.10 on 2016-11-20 11:00 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('conf', '0004_auto_20161120_1834'), + ] + + operations = [ + migrations.CreateModel( + name='JudgeServerToken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('token', models.CharField(max_length=32)), + ], + options={ + 'db_table': 'judge_server_token', + }, + ), + ] diff --git a/conf/models.py b/conf/models.py index 0446ab2..3b09443 100644 --- a/conf/models.py +++ b/conf/models.py @@ -28,3 +28,25 @@ class WebsiteConfig(models.Model): class Meta: db_table = "website_config" + +class JudgeServer(models.Model): + hostname = models.CharField(max_length=64) + ip = models.CharField(max_length=32, blank=True, null=True) + judger_version = models.CharField(max_length=24) + cpu_core = models.IntegerField() + memory_usage = models.FloatField() + cpu_usage = models.FloatField() + last_heartbeat = models.DateTimeField() + create_time = models.DateTimeField(auto_now_add=True) + task_number = models.IntegerField(default=0) + service_url = models.CharField(max_length=128, blank=True, null=True) + + class Meta: + db_table = "judge_server" + + +class JudgeServerToken(models.Model): + token = models.CharField(max_length=32) + + class Meta: + db_table = "judge_server_token" diff --git a/conf/serializers.py b/conf/serializers.py index d6d977e..37640b3 100644 --- a/conf/serializers.py +++ b/conf/serializers.py @@ -1,12 +1,12 @@ -from utils.api import serializers +from utils.api import serializers, DateTimeTZField -from .models import SMTPConfig, WebsiteConfig +from .models import SMTPConfig, WebsiteConfig, JudgeServer class EditSMTPConfigSerializer(serializers.Serializer): server = serializers.CharField(max_length=128) port = serializers.IntegerField(default=25) - email = serializers.CharField(max_length=128) + email = serializers.EmailField(max_length=128) password = serializers.CharField(max_length=128, required=False, allow_null=True, allow_blank=True) tls = serializers.BooleanField() @@ -37,4 +37,23 @@ class CreateEditWebsiteConfigSerializer(serializers.Serializer): class WebsiteConfigSerializer(serializers.ModelSerializer): class Meta: model = WebsiteConfig - exclude = ["id"] \ No newline at end of file + exclude = ["id"] + + +class JudgeServerSerializer(serializers.ModelSerializer): + create_time = DateTimeTZField() + last_heartbeat = DateTimeTZField() + + class Meta: + model = JudgeServer + + +class JudgeServerHeartbeatSerializer(serializers.Serializer): + hostname = serializers.CharField(max_length=64) + judger_version = serializers.CharField(max_length=24) + cpu_core = serializers.IntegerField(min_value=1) + memory = serializers.FloatField(min_value=0, max_value=100) + cpu = serializers.FloatField(min_value=0, max_value=100) + action = serializers.ChoiceField(choices=("heartbeat", )) + service_url = serializers.CharField(max_length=128, required=False) + diff --git a/conf/tests.py b/conf/tests.py index f6760e5..f677771 100644 --- a/conf/tests.py +++ b/conf/tests.py @@ -1,6 +1,7 @@ -from utils.api.tests import APITestCase +import hashlib -from .models import SMTPConfig, WebsiteConfig +from utils.api.tests import APITestCase +from .models import SMTPConfig, JudgeServerToken, JudgeServer class SMTPConfigTest(APITestCase): @@ -73,3 +74,51 @@ class WebsiteConfigAPITest(APITestCase): resp = self.client.get(url) self.assertSuccess(resp) self.assertEqual(resp.data["data"]["name_shortcut"], "oj") + + +class JudgeServerStatusAPITest(APITestCase): + def setUp(self): + self.url = self.reverse("judge_server_api") + self.user = self.create_super_admin() + + def test_get_judge_server_status(self): + self.assertFalse(JudgeServerToken.objects.exists()) + resp = self.client.get(self.url) + self.assertSuccess(resp) + self.assertListEqual(resp.data["data"]["servers"], []) + self.assertEqual(JudgeServerToken.objects.first().token, resp.data["data"]["token"]) + + +class JudgeServerHeartbeatest(APITestCase): + def setUp(self): + self.url = self.reverse("judge_server_heartbeat_api") + self.data = {"hostname": "testhostname", "judger_version": "1.0.4", "cpu_core": 4, + "cpu": 90.5, "memory": 80.3, "action": "heartbeat"} + self.token = "test" + self.hashed_token = hashlib.sha256(self.token.encode("utf-8")).hexdigest() + JudgeServerToken.objects.create(token=self.token) + + def test_new_heartbeat(self): + resp = self.client.post(self.url, data=self.data, **{"HTTP_X_JUDGE_SERVER_TOKEN": self.hashed_token}) + self.assertSuccess(resp) + server = JudgeServer.objects.first() + self.assertEqual(server.ip, "127.0.0.1") + self.assertEqual(server.service_url ,None) + + def test_new_heartbeat_service_url(self): + service_url = "http://1.2.3.4:8000/api/judge" + data = self.data + data["service_url"] = service_url + resp = self.client.post(self.url, data=self.data, **{"HTTP_X_JUDGE_SERVER_TOKEN": self.hashed_token}) + self.assertSuccess(resp) + server = JudgeServer.objects.first() + self.assertEqual(server.ip, None) + self.assertEqual(server.service_url, service_url) + + def test_update_heartbeat(self): + self.test_new_heartbeat() + data = self.data + data["judger_version"] = "2.0.0" + resp = self.client.post(self.url, data=data, **{"HTTP_X_JUDGE_SERVER_TOKEN": self.hashed_token}) + self.assertSuccess(resp) + self.assertEqual(JudgeServer.objects.get(hostname=self.data["hostname"]).judger_version, data["judger_version"]) diff --git a/conf/urls/admin.py b/conf/urls/admin.py index a6e8006..456e6c0 100644 --- a/conf/urls/admin.py +++ b/conf/urls/admin.py @@ -1,8 +1,9 @@ from django.conf.urls import url -from ..views import WebsiteConfigAPI, SMTPAPI +from ..views import WebsiteConfigAPI, SMTPAPI, JudgeServerAPI urlpatterns = [ url(r'^smtp$', SMTPAPI.as_view(), name="smtp_admin_api"), url(r'^website$', WebsiteConfigAPI.as_view(), name="website_config_api"), + url(r'^judge_server', JudgeServerAPI.as_view(), name="judge_server_api") ] diff --git a/conf/urls/oj.py b/conf/urls/oj.py index a6cee9a..3e3c196 100644 --- a/conf/urls/oj.py +++ b/conf/urls/oj.py @@ -1,7 +1,9 @@ from django.conf.urls import url -from ..views import WebsiteConfigAPI +from ..views import WebsiteConfigAPI, JudgeServerHeartbeatAPI + urlpatterns = [ url(r'^website$', WebsiteConfigAPI.as_view(), name="website_info_api"), + url(r'^judge_server_heartbeat$', JudgeServerHeartbeatAPI.as_view(), name="judge_server_heartbeat_api") ] diff --git a/conf/views.py b/conf/views.py index 59fce41..1a4bb11 100644 --- a/conf/views.py +++ b/conf/views.py @@ -1,11 +1,16 @@ -from utils.api import APIView, validate_serializer +import hashlib + +from django.utils import timezone from account.decorators import super_admin_required +from utils.api import APIView, CSRFExemptAPIView, validate_serializer +from utils.shortcuts import rand_str -from .models import SMTPConfig, WebsiteConfig +from .models import SMTPConfig, WebsiteConfig, JudgeServer, JudgeServerToken from .serializers import (WebsiteConfigSerializer, CreateEditWebsiteConfigSerializer, CreateSMTPConfigSerializer, EditSMTPConfigSerializer, - SMTPConfigSerializer, TestSMTPConfigSerializer) + SMTPConfigSerializer, TestSMTPConfigSerializer, + JudgeServerSerializer, JudgeServerHeartbeatSerializer) class SMTPAPI(APIView): @@ -63,3 +68,65 @@ class WebsiteConfigAPI(APIView): WebsiteConfig.objects.all().delete() config = WebsiteConfig.objects.create(**data) return self.success(WebsiteConfigSerializer(config).data) + + +class JudgeServerAPI(APIView): + @super_admin_required + def get(self, request): + judge_server_token = JudgeServerToken.objects.first() + if not judge_server_token: + token = rand_str(12) + JudgeServerToken.objects.create(token=token) + else: + token = judge_server_token.token + servers = JudgeServer.objects.all().order_by("-last_heartbeat") + return self.success({"token": token, + "servers": JudgeServerSerializer(servers, many=True).data}) + + @super_admin_required + def delete(self, request): + pass + + +class JudgeServerHeartbeatAPI(CSRFExemptAPIView): + @validate_serializer(JudgeServerHeartbeatSerializer) + def post(self, request): + judge_server_token = JudgeServerToken.objects.first() + if not judge_server_token: + return self.error("Web server token not set") + token = judge_server_token.token + data = request.data + judge_server_token = request.META.get("HTTP_X_JUDGE_SERVER_TOKEN") + if hashlib.sha256(token.encode("utf-8")).hexdigest() != judge_server_token: + return self.error("Invalid token") + service_url = data.get("service_url") + if service_url: + ip = None + else: + ip = request.META["REMOTE_ADDR"] + + try: + server = JudgeServer.objects.get(hostname=data["hostname"]) + server.judger_version = data["judger_version"] + server.cpu_core = data["cpu_core"] + server.memory_usage = data["memory"] + server.cpu_usage = data["cpu"] + server.service_url= service_url + server.ip = ip + server.last_heartbeat = timezone.now() + server.save() + except JudgeServer.DoesNotExist: + JudgeServer.objects.create(hostname=data["hostname"], + judger_version=data["judger_version"], + cpu_core=data["cpu_core"], + memory_usage=data["memory"], + cpu_usage=data["cpu"], + ip=ip, + service_url=service_url, + last_heartbeat=timezone.now(), + ) + return self.success() + + + +