Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(config): support starting pd nodes & store nodes #16

Merged
merged 9 commits into from
Sep 24, 2024
Merged

feat(config): support starting pd nodes & store nodes #16

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
4309b8e
feat(config): support starting pd nodes & store nodes
HJ-Young Sep 19, 2024
bc79e1a
Merge remote-tracking branch 'origin/master'
HJ-Young Sep 19, 2024
13c5d2b
fix(config): support different backend =
HJ-Young Sep 20, 2024
b08ae0d
fix(config): pass the test
HJ-Young Sep 21, 2024
6172236
fix(config): fix path in deploy_graph.py
HJ-Young Sep 22, 2024
873a288
fix(config): wait for pd & store to start
HJ-Young Sep 23, 2024
819ef3d
chore(readme): update README.md
HJ-Young Sep 23, 2024
2cafc3c
fix(config): fix the config file
HJ-Young Sep 23, 2024
96320b5
Update src/deploy_start.py
imbajin Sep 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 110 additions & 1 deletion src/common/deploy_graph.py
View file
Open in desktop
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could use 100 or 120 as the line wrapping standard, no need wrap too early

Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import subprocess
import sys

from config.basic_config import admin_password
from src.config.basic_config import admin_password

current_path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(current_path + '/../../')
Expand Down Expand Up @@ -118,6 +118,50 @@ def set_server_properties(package_dir_path, host, server_port, gremlin_port):
}
''')

def set_pd_properties(package_dir_path, host, grpc_port, rest_port, store_list, raft_port, raft_list):
"""
修改 pd 组件配置
:return:
"""
# 修改 application 文件
application_conf = package_dir_path + '/conf/application.yml'
alter_properties(application_conf,
'8686',
'%d' % grpc_port)
alter_properties(application_conf,
'8620',
'%d' % rest_port)
new_store_list = ','.join([f"{host}:{port}" for port in store_list])
alter_properties(application_conf,
'initial-store-list: 127.0.0.1:8500',
'initial-store-list: %s' % new_store_list)
alter_properties(application_conf,
'address: 127.0.0.1:8610',
'address: %s:%d' % (host, raft_port))
new_raft_list = ','.join([f"{host}:{port}" for port in raft_list])
alter_properties(application_conf,
'peers-list: 127.0.0.1:8610',
'peers-list: %s' % new_raft_list)

def set_store_properties(package_dir_path, host, pd_list, grpc_port, raft_port, rest_port):
application_conf = package_dir_path + '/conf/application.yml'
new_pd_list = ','.join([f"{host}:{port}" for port in pd_list])
alter_properties(application_conf,
'address: localhost:8686',
'address: %s' % new_pd_list)
alter_properties(application_conf,
'port: 8500',
'port: %d' % grpc_port)
alter_properties(application_conf,
'address: 127.0.0.1:8510',
'address: %s:%d' % (host, raft_port))
alter_properties(application_conf,
'address: 127.0.0.1:8610',
'address: %s:%d' % (host, raft_port))
alter_properties(application_conf,
'port: 8520',
'port: %d' % rest_port)


def set_hubble_properties(package_dir_path, host, port):
"""
Expand All @@ -140,6 +184,16 @@ def start_graph(package_dir_path, graph_type):
f'&& echo "{pa}" | ./bin/init-store.sh '
'&& ./bin/start-hugegraph.sh' % package_dir_path
)
elif graph_type == 'pd':
os.system(
'cd %s '
'&& ./bin/start-hugegraph-pd.sh' % package_dir_path
)
elif graph_type == 'store':
os.system(
'cd %s '
'&& ./bin/start-hugegraph-store.sh' % package_dir_path
)
else:
os.system(
f'chmod -R 755 {package_dir_path}'
Expand Down Expand Up @@ -168,6 +222,16 @@ def __init__(self, obj):
self.loader_git = obj.loader_git
self.tools_git = obj.tools_git
self.hubble_git = obj.hubble_git
self.host = obj.host
self.pd_grpc_port = obj.pd_grpc_port
self.pd_rest_port = obj.pd_rest_port
self.store_list = obj.store_list
self.pd_raft_port = obj.pd_raft_port
self.raft_list = obj.raft_list
self.pd_list = obj.pd_list
self.store_grpc_port = obj.store_grpc_port
self.store_raft_port = obj.store_raft_port
self.store_rest_port = obj.store_rest_port

@staticmethod
def server(conf):
Expand All @@ -189,6 +253,51 @@ def server(conf):
)
start_graph(gen_dir, 'server')

@staticmethod
def pd(conf):
"""
:return:
"""
is_exists_path(conf.codebase_path)
get_code(conf.codebase_path, conf.server_git, conf.pd_local_repo)
compile_package(conf.project_path)
unzip_targz(conf.pd_path, conf.pd_tar_path.split('/')[-1])

gen_dir = os.path.join(conf.codebase_path, conf.pd_gen_dir)
# start graph_server
set_pd_properties(
gen_dir,
conf.host,
conf.pd_grpc_port,
conf.pd_rest_port,
conf.store_list,
conf.pd_raft_port,
conf.raft_list
)
start_graph(gen_dir, 'pd')

@staticmethod
def store(conf):
"""
:return:
"""
is_exists_path(conf.codebase_path)
get_code(conf.codebase_path, conf.server_git, conf.store_local_repo)
compile_package(conf.project_path)
unzip_targz(conf.store_path, conf.store_tar_path.split('/')[-1])

gen_dir = os.path.join(conf.codebase_path, conf.store_gen_dir)
# start graph_server
set_store_properties(
gen_dir,
conf.host,
conf.pd_list,
conf.store_grpc_port,
conf.store_raft_port,
conf.store_rest_port
)
start_graph(gen_dir, 'store')

@staticmethod
def toolchain(conf):
is_exists_path(conf.codebase_path)
Expand Down
26 changes: 23 additions & 3 deletions src/config/basic_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,25 @@
# apache release version
is_incubating = 'incubating-'
# TODO: consider user * instead of fixed version?
server_release_version = '1.5.0'
hugegraph_release_version = '1.5.0'
toolchain_release_version = '1.3.0'
pd_local_repo = 'incubator-hugegraph/hugegraph-pd'
store_local_repo = 'incubator-hugegraph/hugegraph-store'
server_local_repo = 'incubator-hugegraph/hugegraph-server'
toolchain_local_repo = 'incubator-hugegraph-toolchain'
server_gen_dir = f'incubator-hugegraph/hugegraph-server/apache-hugegraph-server-{is_incubating}{server_release_version}'
pd_gen_dir = f'incubator-hugegraph/hugegraph-pd/apache-hugegraph-pd-{is_incubating}{hugegraph_release_version}'
store_gen_dir = f'incubator-hugegraph/hugegraph-store/apache-hugegraph-store-{is_incubating}{hugegraph_release_version}'
server_gen_dir = f'incubator-hugegraph/hugegraph-server/apache-hugegraph-server-{is_incubating}{hugegraph_release_version}'
toolchain_gen_dir = f'incubator-hugegraph-toolchain/apache-hugegraph-toolchain-{is_incubating}{toolchain_release_version}'
toolchain_obj_template = 'apache-hugegraph-{tool_name}-' + is_incubating + f'{toolchain_release_version}'

project_path = os.path.join(codebase_path, 'incubator-hugegraph')
pd_path = os.path.join(codebase_path, pd_local_repo)
store_path = os.path.join(codebase_path, store_local_repo)
server_path = os.path.join(codebase_path, server_local_repo)
server_tar_path = os.path.join(codebase_path, 'incubator-hugegraph/hugegraph-server', f'apache-hugegraph-server-{is_incubating}{server_release_version}' + '.tar.gz')
pd_tar_path = os.path.join(codebase_path, pd_gen_dir + '.tar.gz')
store_tar_path = os.path.join(codebase_path, store_gen_dir + '.tar.gz')
server_tar_path = os.path.join(codebase_path, server_gen_dir + '.tar.gz')
toolchain_path = os.path.join(codebase_path, toolchain_local_repo)
loader_path = os.path.join(codebase_path, toolchain_gen_dir, toolchain_obj_template.format(tool_name='loader'))
hubble_path = os.path.join(codebase_path, toolchain_gen_dir, toolchain_obj_template.format(tool_name='hubble'))
Expand All @@ -49,6 +57,18 @@
admin_password = {'admin': 'admin'}
test_password = {'tester': '123456'}

host = '127.0.0.1'
pd_grpc_port = 8686
pd_rest_port = 8620
store_list = [8500]
pd_raft_port = 8610
raft_list = [8610]

pd_list = [8686]
store_grpc_port = 8500
store_raft_port = 8510
store_rest_port = 8520

# toolchain (includes loader, hubble, tools)
toolchain_git = {
'branch': '3b58fc6',
Expand Down
8 changes: 7 additions & 1 deletion src/deploy_start.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,21 @@ def graph_deploy(param, conf_obj):
Deploy.server(conf_obj)
elif param == 'toolchain':
Deploy.toolchain(conf_obj)
elif param == 'pd':
Deploy.pd(conf_obj)
elif param == 'store':
Deploy.store(conf_obj)
else:
Deploy.pd(conf_obj)
Deploy.store(conf_obj)
Deploy.server(conf_obj)
Deploy.toolchain(conf_obj)


if __name__ == "__main__":
param_size = len(sys.argv)
if param_size == 2 \
and sys.argv[1] in ['all', 'server', 'toolchain']:
and sys.argv[1] in ['all', 'server', 'toolchain', 'pd', 'store']:
graph_deploy(sys.argv[1], basic_config)
else:
print('failed: 执行脚本参数为[all,server,toolchain]')
Expand Down
Loading