Integrate two coordinators

alibaba · May 28, 2024 · bc0ffaa · bc0ffaa
1 parent 40a5a7b
commit bc0ffaa
Show file tree

Hide file tree

Showing 131 changed files with 614 additions and 1,996 deletions.
diff --git a/coordinator/gscoordinator/coordinator.py b/coordinator/gscoordinator/coordinator.py
@@ -26,10 +26,12 @@
 import sys
 from concurrent import futures
 
+import connexion
 import grpc
 from graphscope.config import Config
 from graphscope.proto import coordinator_service_pb2_grpc
 
+from gscoordinator.flex.encoder import JSONEncoder
 from gscoordinator.monitor import Monitor
 from gscoordinator.servicer import init_graphscope_one_service_servicer
 from gscoordinator.utils import GS_GRPC_MAX_MESSAGE_LENGTH
@@ -144,6 +146,16 @@ def start_server(
 
     server.start()
 
+    # OpenApi server
+    app = connexion.App(__name__, specification_dir="./flex/openapi/")
+    app.app.json_encoder = JSONEncoder
+    app.add_api(
+        "openapi.yaml",
+        arguments={"title": "GraphScope FLEX HTTP SERVICE API"},
+        pythonic_params=True,
+    )
+    app.run(port=config.coordinator.http_port)
+
     if config.coordinator.monitor:
         try:
             Monitor.startServer(config.coordinator.monitor_port, "127.0.0.1")

diff --git a/coordinator/gscoordinator/flex/__init__.py b/coordinator/gscoordinator/flex/__init__.py
@@ -0,0 +1,17 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/coordinator/gscoordinator/flex/controllers/__init__.py b/coordinator/gscoordinator/flex/controllers/__init__.py
@@ -0,0 +1,17 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/...ordinator/controllers/alert_controller.py → ...ator/flex/controllers/alert_controller.py b/...ordinator/controllers/alert_controller.py → ...ator/flex/controllers/alert_controller.py
@@ -3,14 +3,14 @@
 from typing import Tuple
 from typing import Union
 
-from gs_flex_coordinator.models.create_alert_receiver_request import CreateAlertReceiverRequest  # noqa: E501
-from gs_flex_coordinator.models.create_alert_rule_request import CreateAlertRuleRequest  # noqa: E501
-from gs_flex_coordinator.models.error import Error  # noqa: E501
-from gs_flex_coordinator.models.get_alert_message_response import GetAlertMessageResponse  # noqa: E501
-from gs_flex_coordinator.models.get_alert_receiver_response import GetAlertReceiverResponse  # noqa: E501
-from gs_flex_coordinator.models.get_alert_rule_response import GetAlertRuleResponse  # noqa: E501
-from gs_flex_coordinator.models.update_alert_message_status_request import UpdateAlertMessageStatusRequest  # noqa: E501
-from gs_flex_coordinator import util
+from gscoordinator.flex.models.create_alert_receiver_request import CreateAlertReceiverRequest  # noqa: E501
+from gscoordinator.flex.models.create_alert_rule_request import CreateAlertRuleRequest  # noqa: E501
+from gscoordinator.flex.models.error import Error  # noqa: E501
+from gscoordinator.flex.models.get_alert_message_response import GetAlertMessageResponse  # noqa: E501
+from gscoordinator.flex.models.get_alert_receiver_response import GetAlertReceiverResponse  # noqa: E501
+from gscoordinator.flex.models.get_alert_rule_response import GetAlertRuleResponse  # noqa: E501
+from gscoordinator.flex.models.update_alert_message_status_request import UpdateAlertMessageStatusRequest  # noqa: E501
+from gscoordinator.flex import util
 
 
 def create_alert_receiver(create_alert_receiver_request):  # noqa: E501

diff --git a/...tor/controllers/data_source_controller.py → ...lex/controllers/data_source_controller.py b/...tor/controllers/data_source_controller.py → ...lex/controllers/data_source_controller.py
@@ -3,12 +3,12 @@
 from typing import Tuple
 from typing import Union
 
-from gs_flex_coordinator.models.error import Error  # noqa: E501
-from gs_flex_coordinator.models.schema_mapping import SchemaMapping  # noqa: E501
-from gs_flex_coordinator import util
+from gscoordinator.flex.models.error import Error  # noqa: E501
+from gscoordinator.flex.models.schema_mapping import SchemaMapping  # noqa: E501
+from gscoordinator.flex import util
 
-from gs_flex_coordinator.core import client_wrapper
-from gs_flex_coordinator.core import handle_api_exception
+from gscoordinator.flex.core import client_wrapper
+from gscoordinator.flex.core import handle_api_exception
 
 
 @handle_api_exception()

diff --git a/...ator/controllers/deployment_controller.py → ...flex/controllers/deployment_controller.py b/...ator/controllers/deployment_controller.py → ...flex/controllers/deployment_controller.py
@@ -3,13 +3,13 @@
 from typing import Tuple
 from typing import Union
 
-from gs_flex_coordinator.models.error import Error  # noqa: E501
-from gs_flex_coordinator.models.running_deployment_info import RunningDeploymentInfo  # noqa: E501
-from gs_flex_coordinator.models.running_deployment_status import RunningDeploymentStatus  # noqa: E501
-from gs_flex_coordinator import util
+from gscoordinator.flex.models.error import Error  # noqa: E501
+from gscoordinator.flex.models.running_deployment_info import RunningDeploymentInfo  # noqa: E501
+from gscoordinator.flex.models.running_deployment_status import RunningDeploymentStatus  # noqa: E501
+from gscoordinator.flex import util
 
-from gs_flex_coordinator.core import client_wrapper
-from gs_flex_coordinator.core import handle_api_exception
+from gscoordinator.flex.core import client_wrapper
+from gscoordinator.flex.core import handle_api_exception
 
 
 @handle_api_exception()

diff --git a/...ordinator/controllers/graph_controller.py → ...ator/flex/controllers/graph_controller.py b/...ordinator/controllers/graph_controller.py → ...ator/flex/controllers/graph_controller.py
@@ -3,18 +3,18 @@
 from typing import Tuple
 from typing import Union
 
-from gs_flex_coordinator.models.create_edge_type import CreateEdgeType  # noqa: E501
-from gs_flex_coordinator.models.create_graph_request import CreateGraphRequest  # noqa: E501
-from gs_flex_coordinator.models.create_graph_response import CreateGraphResponse  # noqa: E501
-from gs_flex_coordinator.models.create_graph_schema_request import CreateGraphSchemaRequest  # noqa: E501
-from gs_flex_coordinator.models.create_vertex_type import CreateVertexType  # noqa: E501
-from gs_flex_coordinator.models.error import Error  # noqa: E501
-from gs_flex_coordinator.models.get_graph_response import GetGraphResponse  # noqa: E501
-from gs_flex_coordinator.models.get_graph_schema_response import GetGraphSchemaResponse  # noqa: E501
-from gs_flex_coordinator import util
-
-from gs_flex_coordinator.core import client_wrapper
-from gs_flex_coordinator.core import handle_api_exception
+from gscoordinator.flex.models.create_edge_type import CreateEdgeType  # noqa: E501
+from gscoordinator.flex.models.create_graph_request import CreateGraphRequest  # noqa: E501
+from gscoordinator.flex.models.create_graph_response import CreateGraphResponse  # noqa: E501
+from gscoordinator.flex.models.create_graph_schema_request import CreateGraphSchemaRequest  # noqa: E501
+from gscoordinator.flex.models.create_vertex_type import CreateVertexType  # noqa: E501
+from gscoordinator.flex.models.error import Error  # noqa: E501
+from gscoordinator.flex.models.get_graph_response import GetGraphResponse  # noqa: E501
+from gscoordinator.flex.models.get_graph_schema_response import GetGraphSchemaResponse  # noqa: E501
+from gscoordinator.flex import util
+
+from gscoordinator.flex.core import client_wrapper
+from gscoordinator.flex.core import handle_api_exception
 
 
 def create_edge_type(graph_id, create_edge_type=None):  # noqa: E501

diff --git a/...coordinator/controllers/job_controller.py → ...inator/flex/controllers/job_controller.py b/...coordinator/controllers/job_controller.py → ...inator/flex/controllers/job_controller.py
@@ -3,14 +3,14 @@
 from typing import Tuple
 from typing import Union
 
-from gs_flex_coordinator.models.create_dataloading_job_response import CreateDataloadingJobResponse  # noqa: E501
-from gs_flex_coordinator.models.dataloading_job_config import DataloadingJobConfig  # noqa: E501
-from gs_flex_coordinator.models.error import Error  # noqa: E501
-from gs_flex_coordinator.models.job_status import JobStatus  # noqa: E501
-from gs_flex_coordinator import util
-
-from gs_flex_coordinator.core import client_wrapper
-from gs_flex_coordinator.core import handle_api_exception
+from gscoordinator.flex.models.create_dataloading_job_response import CreateDataloadingJobResponse  # noqa: E501
+from gscoordinator.flex.models.dataloading_job_config import DataloadingJobConfig  # noqa: E501
+from gscoordinator.flex.models.error import Error  # noqa: E501
+from gscoordinator.flex.models.job_status import JobStatus  # noqa: E501
+from gscoordinator.flex import util
+
+from gscoordinator.flex.core import client_wrapper
+from gscoordinator.flex.core import handle_api_exception
 
 
 @handle_api_exception()

diff --git a/...inator/controllers/security_controller.py → ...r/flex/controllers/security_controller.py b/...inator/controllers/security_controller.py → ...r/flex/controllers/security_controller.py
diff --git a/...dinator/controllers/service_controller.py → ...or/flex/controllers/service_controller.py b/...dinator/controllers/service_controller.py → ...or/flex/controllers/service_controller.py
@@ -3,13 +3,13 @@
 from typing import Tuple
 from typing import Union
 
-from gs_flex_coordinator.models.error import Error  # noqa: E501
-from gs_flex_coordinator.models.service_status import ServiceStatus  # noqa: E501
-from gs_flex_coordinator.models.start_service_request import StartServiceRequest  # noqa: E501
-from gs_flex_coordinator import util
+from gscoordinator.flex.models.error import Error  # noqa: E501
+from gscoordinator.flex.models.service_status import ServiceStatus  # noqa: E501
+from gscoordinator.flex.models.start_service_request import StartServiceRequest  # noqa: E501
+from gscoordinator.flex import util
 
-from gs_flex_coordinator.core import client_wrapper
-from gs_flex_coordinator.core import handle_api_exception
+from gscoordinator.flex.core import client_wrapper
+from gscoordinator.flex.core import handle_api_exception
 
 
 def get_service_status_by_id(graph_id):  # noqa: E501

diff --git a/...ontrollers/stored_procedure_controller.py → ...ontrollers/stored_procedure_controller.py b/...ontrollers/stored_procedure_controller.py → ...ontrollers/stored_procedure_controller.py
@@ -3,15 +3,15 @@
 from typing import Tuple
 from typing import Union
 
-from gs_flex_coordinator.models.create_stored_proc_request import CreateStoredProcRequest  # noqa: E501
-from gs_flex_coordinator.models.create_stored_proc_response import CreateStoredProcResponse  # noqa: E501
-from gs_flex_coordinator.models.error import Error  # noqa: E501
-from gs_flex_coordinator.models.get_stored_proc_response import GetStoredProcResponse  # noqa: E501
-from gs_flex_coordinator.models.update_stored_proc_request import UpdateStoredProcRequest  # noqa: E501
-from gs_flex_coordinator import util
-
-from gs_flex_coordinator.core import client_wrapper
-from gs_flex_coordinator.core import handle_api_exception
+from gscoordinator.flex.models.create_stored_proc_request import CreateStoredProcRequest  # noqa: E501
+from gscoordinator.flex.models.create_stored_proc_response import CreateStoredProcResponse  # noqa: E501
+from gscoordinator.flex.models.error import Error  # noqa: E501
+from gscoordinator.flex.models.get_stored_proc_response import GetStoredProcResponse  # noqa: E501
+from gscoordinator.flex.models.update_stored_proc_request import UpdateStoredProcRequest  # noqa: E501
+from gscoordinator.flex import util
+
+from gscoordinator.flex.core import client_wrapper
+from gscoordinator.flex.core import handle_api_exception
 
 
 @handle_api_exception()

diff --git a/...ordinator/controllers/utils_controller.py → ...ator/flex/controllers/utils_controller.py b/...ordinator/controllers/utils_controller.py → ...ator/flex/controllers/utils_controller.py
@@ -3,12 +3,12 @@
 from typing import Tuple
 from typing import Union
 
-from gs_flex_coordinator.models.error import Error  # noqa: E501
-from gs_flex_coordinator.models.upload_file_response import UploadFileResponse  # noqa: E501
-from gs_flex_coordinator import util
+from gscoordinator.flex.models.error import Error  # noqa: E501
+from gscoordinator.flex.models.upload_file_response import UploadFileResponse  # noqa: E501
+from gscoordinator.flex import util
 
-from gs_flex_coordinator.core import client_wrapper
-from gs_flex_coordinator.core import handle_api_exception
+from gscoordinator.flex.core import client_wrapper
+from gscoordinator.flex.core import handle_api_exception
 
 
 @handle_api_exception()

diff --git a/...ator/gs_flex_coordinator/core/__init__.py → ...nator/gscoordinator/flex/core/__init__.py b/...ator/gs_flex_coordinator/core/__init__.py → ...nator/gscoordinator/flex/core/__init__.py
@@ -27,5 +27,5 @@
     level=logging.INFO,
 )
 
-from gs_flex_coordinator.core.client_wrapper import client_wrapper  # noqa: F401, E402
-from gs_flex_coordinator.core.utils import handle_api_exception  # noqa: F401, E402
+from gscoordinator.flex.core.client_wrapper import client_wrapper  # noqa: F401, E402
+from gscoordinator.flex.core.utils import handle_api_exception  # noqa: F401, E402
diff --git a/...s_flex_coordinator/core/alert/__init__.py → ...gscoordinator/flex/core/alert/__init__.py b/...s_flex_coordinator/core/alert/__init__.py → ...gscoordinator/flex/core/alert/__init__.py
@@ -16,4 +16,4 @@
 # limitations under the License.
 #
 
-from gs_flex_coordinator.core.alert.alert_manager import alert_manager
+from gscoordinator.flex.core.alert.alert_manager import alert_manager
diff --git a/...x_coordinator/core/alert/alert_manager.py → ...rdinator/flex/core/alert/alert_manager.py b/...x_coordinator/core/alert/alert_manager.py → ...rdinator/flex/core/alert/alert_manager.py
@@ -23,16 +23,16 @@
 from typing import List
 from typing import Union
 
-from gs_flex_coordinator.core.alert.alert_receiver import DingTalkReceiver
-from gs_flex_coordinator.core.alert.builtin_rules import init_builtin_alert_rules
-from gs_flex_coordinator.core.alert.message_collector import AlertMessageCollector
-from gs_flex_coordinator.core.config import ALERT_WORKSPACE
-from gs_flex_coordinator.core.scheduler import schedule
-from gs_flex_coordinator.core.utils import decode_datetimestr
-from gs_flex_coordinator.core.utils import encode_datetime
-from gs_flex_coordinator.models import AlertMessage
-from gs_flex_coordinator.models import AlertReceiver
-from gs_flex_coordinator.models import AlertRule
+from gscoordinator.flex.core.alert.alert_receiver import DingTalkReceiver
+from gscoordinator.flex.core.alert.builtin_rules import init_builtin_alert_rules
+from gscoordinator.flex.core.alert.message_collector import AlertMessageCollector
+from gscoordinator.flex.core.config import ALERT_WORKSPACE
+from gscoordinator.flex.core.scheduler import schedule
+from gscoordinator.flex.core.utils import decode_datetimestr
+from gscoordinator.flex.core.utils import encode_datetime
+from gscoordinator.flex.models import AlertMessage
+from gscoordinator.flex.models import AlertReceiver
+from gscoordinator.flex.models import AlertRule
 
 
 class AlertManager(object):

diff --git a/...x_coordinator/core/alert/alert_message.py → ...rdinator/flex/core/alert/alert_message.py b/...x_coordinator/core/alert/alert_message.py → ...rdinator/flex/core/alert/alert_message.py
@@ -18,7 +18,7 @@
 
 import datetime
 
-from gs_flex_coordinator.core.utils import encode_datetime
+from gscoordinator.flex.core.utils import encode_datetime
 
 
 class AlertMessage(object):

diff --git a/..._coordinator/core/alert/alert_receiver.py → ...dinator/flex/core/alert/alert_receiver.py b/..._coordinator/core/alert/alert_receiver.py → ...dinator/flex/core/alert/alert_receiver.py
@@ -21,9 +21,9 @@
 
 import requests
 
-from gs_flex_coordinator.core.alert.alert_message import AlertMessage
-from gs_flex_coordinator.core.config import INSTANCE_NAME
-from gs_flex_coordinator.core.utils import random_string
+from gscoordinator.flex.core.alert.alert_message import AlertMessage
+from gscoordinator.flex.core.config import INSTANCE_NAME
+from gscoordinator.flex.core.utils import random_string
 
 
 class DingTalkReceiver(object):

diff --git a/...flex_coordinator/core/alert/alert_rule.py → ...coordinator/flex/core/alert/alert_rule.py b/...flex_coordinator/core/alert/alert_rule.py → ...coordinator/flex/core/alert/alert_rule.py
@@ -21,9 +21,9 @@
 from abc import ABCMeta
 from abc import abstractmethod
 
-from gs_flex_coordinator.core.alert.alert_message import AlertMessage
-from gs_flex_coordinator.core.scheduler import cancel_job
-from gs_flex_coordinator.core.scheduler import schedule
+from gscoordinator.flex.core.alert.alert_message import AlertMessage
+from gscoordinator.flex.core.scheduler import cancel_job
+from gscoordinator.flex.core.scheduler import schedule
 
 
 class AlertRule(metaclass=ABCMeta):

diff --git a/...x_coordinator/core/alert/builtin_rules.py → ...rdinator/flex/core/alert/builtin_rules.py b/...x_coordinator/core/alert/builtin_rules.py → ...rdinator/flex/core/alert/builtin_rules.py
@@ -23,11 +23,11 @@
 import psutil
 from gremlin_python.driver.client import Client
 
-from gs_flex_coordinator.core import client_wrapper
-from gs_flex_coordinator.core.alert.alert_rule import AlertRule
-from gs_flex_coordinator.core.alert.message_collector import AlertMessageCollector
-from gs_flex_coordinator.core.config import CLUSTER_TYPE
-from gs_flex_coordinator.core.config import SOLUTION
+from gscoordinator.flex.core import client_wrapper
+from gscoordinator.flex.core.alert.alert_rule import AlertRule
+from gscoordinator.flex.core.alert.message_collector import AlertMessageCollector
+from gscoordinator.flex.core.config import CLUSTER_TYPE
+from gscoordinator.flex.core.config import SOLUTION
 
 
 class HighDiskUtilizationAlert(AlertRule):

diff --git a/...ordinator/core/alert/message_collector.py → ...ator/flex/core/alert/message_collector.py b/...ordinator/core/alert/message_collector.py → ...ator/flex/core/alert/message_collector.py
@@ -21,11 +21,11 @@
 import os
 import pickle
 
-from gs_flex_coordinator.core.alert.alert_message import AlertMessage
-from gs_flex_coordinator.core.config import ALERT_WORKSPACE
-from gs_flex_coordinator.core.scheduler import cancel_job
-from gs_flex_coordinator.core.scheduler import schedule
-from gs_flex_coordinator.core.utils import decode_datetimestr
+from gscoordinator.flex.core.alert.alert_message import AlertMessage
+from gscoordinator.flex.core.config import ALERT_WORKSPACE
+from gscoordinator.flex.core.scheduler import cancel_job
+from gscoordinator.flex.core.scheduler import schedule
+from gscoordinator.flex.core.utils import decode_datetimestr
 
 
 class OneDayAlertMessageCollector(object):