diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index 2dbc48cd8..e59a46b1c 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -1,6 +1,12 @@ name: third-party dependencies check -on: [push, pull_request] +on: + push: + branches: + - master + - /^release-.*$/ + - /^v[0-9]\..*$/ + pull_request: jobs: build: diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml index 129302667..2f2f3d7db 100644 --- a/.github/workflows/hubble-ci.yml +++ b/.github/workflows/hubble-ci.yml @@ -84,7 +84,7 @@ jobs: $TRAVIS_DIR/install-hugegraph.sh $SERVER_VERSION - name: Unit test - run: mvn test -P unit-test -pl hugegraph-hubble -ntp + run: mvn test -P unit-test -pl hugegraph-hubble/hubble-be -ntp - name: API test env: diff --git a/hugegraph-dist/scripts/dependency/known-dependencies.txt b/hugegraph-dist/scripts/dependency/known-dependencies.txt index 0063fb694..e56e92b52 100644 --- a/hugegraph-dist/scripts/dependency/known-dependencies.txt +++ b/hugegraph-dist/scripts/dependency/known-dependencies.txt @@ -11,7 +11,6 @@ annotations-17.0.0.jar ant-1.9.1.jar ant-launcher-1.9.1.jar antlr-runtime-3.5.2.jar -aopalliance-repackaged-2.5.0-b42.jar aopalliance-repackaged-3.0.1.jar apache-curator-2.12.0.pom arrow-format-0.8.0.jar @@ -58,7 +57,6 @@ commons-math3-3.1.1.jar commons-math3-3.4.1.jar commons-net-3.1.jar commons-net-3.6.jar -commons-text-1.4.jar commons-text-1.6.jar commons-text-1.9.jar curator-client-2.12.0.jar @@ -137,11 +135,8 @@ hive-storage-api-2.7.0.jar hive-storage-api-2.7.2.jar hive-upgrade-acid-3.1.3.jar hive-vector-code-gen-3.1.3.jar -hk2-api-2.5.0-b42.jar hk2-api-3.0.1.jar -hk2-locator-2.5.0-b42.jar hk2-locator-3.0.1.jar -hk2-utils-2.5.0-b42.jar hk2-utils-3.0.1.jar hppc-0.7.2.jar htrace-core4-4.1.0-incubating.jar @@ -171,9 +166,11 @@ jackson-databind-2.7.8.jar jackson-databind-2.9.9.3.jar jackson-datatype-jdk8-2.9.9.jar jackson-datatype-jsr310-2.9.9.jar +jackson-jaxrs-base-2.9.9.jar jackson-jaxrs-base-2.14.0-rc1.jar jackson-jaxrs-json-provider-2.14.0-rc1.jar jackson-jaxrs-json-provider-2.7.8.jar +jackson-jaxrs-json-provider-2.9.9.jar jackson-mapper-asl-1.9.13.jar jackson-module-jaxb-annotations-2.14.0-rc1.jar jackson-module-jaxb-annotations-2.12.3.jar @@ -197,14 +194,11 @@ javax.activation-api-1.2.0.jar javax.annotation-api-1.3.2.jar javax.el-3.0.0.jar javax.el-3.0.1-b12.jar -javax.inject-1.jar -javax.inject-2.5.0-b42.jar javax.json-1.0.jar javax.servlet-api-3.1.0.jar javax.servlet-api-4.0.1.jar javax.servlet.jsp-2.3.2.jar javax.servlet.jsp-api-2.3.1.jar -javax.ws.rs-api-2.1.jar javolution-5.5.1.jar jaxb-api-2.2.11.jar jaxb-api-2.3.1.jar @@ -215,24 +209,14 @@ jcip-annotations-1.0-1.jar jcodings-1.0.18.jar jcommander-1.72.jar jcommander-1.78.jar -jersey-apache-connector-2.27.jar jersey-apache-connector-3.0.3.jar -jersey-client-2.27.jar jersey-client-3.0.3.jar -jersey-common-2.27.jar jersey-common-3.0.3.jar -jersey-container-servlet-2.27.jar jersey-container-servlet-3.0.3.jar -jersey-container-servlet-core-2.27.jar jersey-container-servlet-core-3.0.3.jar -jersey-entity-filtering-2.27.jar jersey-entity-filtering-3.0.3.jar -jersey-hk2-2.27.jar jersey-hk2-3.0.3.jar -jersey-media-jaxb-2.27.jar -jersey-media-json-jackson-2.27.jar jersey-media-json-jackson-3.0.3.jar -jersey-server-2.27.jar jersey-server-3.0.3.jar jetty-client-9.4.33.v20201020.jar jetty-http-9.3.19.v20170502.jar @@ -279,7 +263,6 @@ jsp-api-2.1.jar jsqlparser-3.1.jar jsr305-3.0.0.jar jsr305-3.0.1.jar -jsr305-3.0.2.jar jul-to-slf4j-1.7.28.jar junit-4.12.jar junit-4.13.1.jar @@ -307,10 +290,10 @@ log4j-1.2.17.jar log4j-api-2.11.2.jar log4j-api-2.18.0.jar log4j-core-2.18.0.jar +log4j-core-2.11.2.jar +log4j-jul-2.11.2.jar log4j-slf4j-impl-2.18.0.jar -log4j-to-slf4j-2.11.2.jar -logback-classic-1.2.3.jar -logback-core-1.2.3.jar +log4j-slf4j-impl-2.11.2.jar lombok-1.18.8.jar lz4-java-1.4.0.jar lz4-java-1.7.1.jar @@ -345,7 +328,6 @@ orc-core-1.5.8.jar orc-core-1.6.14.jar orc-shims-1.5.8.jar orc-shims-1.6.14.jar -osgi-resource-locator-1.0.1.jar osgi-resource-locator-1.0.3.jar ow2-asm-6.2.jar paranamer-2.3.jar @@ -362,7 +344,6 @@ sfm-util-8.2.1.jar slf4j-api-1.7.25.jar slf4j-api-1.7.28.jar slf4j-api-1.7.30.jar -slf4j-log4j12-1.7.28.jar slf4j-log4j12-1.7.30.jar snakeyaml-1.23.jar snappy-java-1.0.5.jar @@ -379,7 +360,7 @@ spring-boot-starter-actuator-2.1.8.RELEASE.jar spring-boot-starter-cache-2.1.8.RELEASE.jar spring-boot-starter-jdbc-2.1.8.RELEASE.jar spring-boot-starter-json-2.1.8.RELEASE.jar -spring-boot-starter-logging-2.1.8.RELEASE.jar +spring-boot-starter-log4j2-2.1.8.RELEASE.jar spring-boot-starter-tomcat-2.1.8.RELEASE.jar spring-boot-starter-web-2.1.8.RELEASE.jar spring-context-5.1.9.RELEASE.jar diff --git a/hugegraph-hubble/hubble-be/pom.xml b/hugegraph-hubble/hubble-be/pom.xml index 28a8821aa..deebfd8a8 100644 --- a/hugegraph-hubble/hubble-be/pom.xml +++ b/hugegraph-hubble/hubble-be/pom.xml @@ -18,9 +18,19 @@ + + org.springframework.boot + spring-boot-starter-log4j2 + org.springframework.boot spring-boot-starter-jdbc + + + org.springframework.boot + spring-boot-starter-logging + + org.springframework.boot @@ -63,6 +73,11 @@ com.github.ben-manes.caffeine caffeine + + org.apache.hugegraph + hugegraph-common + ${version} + org.apache.hugegraph @@ -97,6 +112,18 @@ com.oracle ojdbc8 + + org.apache.hugegraph + hugegraph-common + + + slf4j-log4j12 + org.slf4j + + + log4j + log4j + diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/config/WebMvcConfig.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/config/WebMvcConfig.java index 6ced26e8f..bd740a5bb 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/config/WebMvcConfig.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/config/WebMvcConfig.java @@ -20,7 +20,6 @@ package org.apache.hugegraph.config; import org.apache.hugegraph.handler.CustomInterceptor; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.InterceptorRegistry; import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; @@ -43,7 +42,6 @@ public void addInterceptors(InterceptorRegistry registry) { .addPathPatterns("/**"); } - @Bean public CustomInterceptor customInterceptor() { return new CustomInterceptor(); } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/ExceptionAdvisor.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/ExceptionAdvisor.java index 0b2a9bd78..be961e5a1 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/ExceptionAdvisor.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/ExceptionAdvisor.java @@ -44,6 +44,7 @@ public class ExceptionAdvisor { @ExceptionHandler(InternalException.class) @ResponseStatus(HttpStatus.OK) public Response exceptionHandler(InternalException e) { + log.error("InternalException:", e); String message = this.handleMessage(e.getMessage(), e.args()); return Response.builder() .status(Constant.STATUS_INTERNAL_ERROR) @@ -55,6 +56,7 @@ public Response exceptionHandler(InternalException e) { @ExceptionHandler(ExternalException.class) @ResponseStatus(HttpStatus.OK) public Response exceptionHandler(ExternalException e) { + log.error("ExternalException:", e); String message = this.handleMessage(e.getMessage(), e.args()); return Response.builder() .status(e.status()) @@ -66,6 +68,7 @@ public Response exceptionHandler(ExternalException e) { @ExceptionHandler(ParameterizedException.class) @ResponseStatus(HttpStatus.OK) public Response exceptionHandler(ParameterizedException e) { + log.error("ParameterizedException", e); String message = this.handleMessage(e.getMessage(), e.args()); return Response.builder() .status(Constant.STATUS_BAD_REQUEST) @@ -77,6 +80,7 @@ public Response exceptionHandler(ParameterizedException e) { @ExceptionHandler(Exception.class) @ResponseStatus(HttpStatus.OK) public Response exceptionHandler(Exception e) { + log.error("Exception:", e); String message = this.handleMessage(e.getMessage(), null); return Response.builder() .status(Constant.STATUS_BAD_REQUEST) @@ -88,6 +92,7 @@ public Response exceptionHandler(Exception e) { @ExceptionHandler(IllegalGremlinException.class) @ResponseStatus(HttpStatus.OK) public Response exceptionHandler(IllegalGremlinException e) { + log.error("IllegalGremlinException:", e); String message = this.handleMessage(e.getMessage(), e.args()); return Response.builder() .status(Constant.STATUS_ILLEGAL_GREMLIN) diff --git a/hugegraph-hubble/hubble-be/src/main/resources/log4j2.xml b/hugegraph-hubble/hubble-be/src/main/resources/log4j2.xml new file mode 100644 index 000000000..fabc9216e --- /dev/null +++ b/hugegraph-hubble/hubble-be/src/main/resources/log4j2.xml @@ -0,0 +1,75 @@ + + + + + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphConnectionTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphConnectionTest.java new file mode 100644 index 000000000..aa2196093 --- /dev/null +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphConnectionTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hugegraph.unit; + +import org.apache.hugegraph.HugeGraphHubble; +import org.apache.hugegraph.common.Constant; +import org.apache.hugegraph.common.Response; +import org.apache.hugegraph.entity.GraphConnection; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@TestPropertySource("classpath:application.properties") +@SpringBootTest(classes = HugeGraphHubble.class, webEnvironment = + SpringBootTest.WebEnvironment.RANDOM_PORT) +public class GraphConnectionTest { + + private static final String HOST = "127.0.0.1"; + private static final int PORT = 8080; + + @Autowired + private TestRestTemplate testRestTemplate; + + @Test + public void testGraphConnect() { + GraphConnection entry = + GraphConnection.builder().host(HOST).port(PORT).name("test").graph( + "hugegraph").build(); + Response response = testRestTemplate.postForObject( + Constant.API_VERSION + "graph-connections", + entry, Response.class); + Assert.assertEquals(response.getMessage(), 200, response.getStatus()); + } +} diff --git a/hugegraph-hubble/hubble-be/src/test/resources/application.properties b/hugegraph-hubble/hubble-be/src/test/resources/application.properties index 6efb9e8b5..cc264dfda 100644 --- a/hugegraph-hubble/hubble-be/src/test/resources/application.properties +++ b/hugegraph-hubble/hubble-be/src/test/resources/application.properties @@ -2,7 +2,7 @@ server.servlet.context-path=/api/v1.1 server.port=8088 spring.datasource.driver-class-name=org.h2.Driver -spring.datasource.url=jdbc:h2:mem:db +spring.datasource.url=jdbc:h2:mem:db;DB_CLOSE_ON_EXIT=FALSE spring.datasource.username=sa spring.datasource.password= spring.datasource.schema=classpath:database/schema.sql @@ -28,8 +28,3 @@ mybatis.configuration.use-generated-keys=true mybatis.configuration.default-executor-type=reuse mybatis.configuration.default-statement-timeout=600 -logging.level.org.springframework=WARN -logging.level.org.apache.hugegraph.mapper=DEBUG -logging.level.org.apache.hugegraph.service=INFO -logging.file=logs/hugegraph-hubble.log -logging.file.max-size=10MB diff --git a/hugegraph-hubble/hubble-be/src/test/resources/database/schema.sql b/hugegraph-hubble/hubble-be/src/test/resources/database/schema.sql index cec66668d..cc03c3e78 100644 --- a/hugegraph-hubble/hubble-be/src/test/resources/database/schema.sql +++ b/hugegraph-hubble/hubble-be/src/test/resources/database/schema.sql @@ -12,8 +12,11 @@ CREATE TABLE IF NOT EXISTS `graph_connection` ( `graph` VARCHAR(48) NOT NULL, `host` VARCHAR(48) NOT NULL DEFAULT 'localhost', `port` INT NOT NULL DEFAULT '8080', + `timeout` INT NOT NULL, `username` VARCHAR(48), `password` VARCHAR(48), + `enabled` BOOLEAN NOT NULL DEFAULT true, + `disable_reason` VARCHAR(65535) NOT NULL DEFAULT '', `create_time` DATETIME(6) NOT NULL, PRIMARY KEY (`id`), UNIQUE (`name`), @@ -22,19 +25,95 @@ CREATE TABLE IF NOT EXISTS `graph_connection` ( CREATE TABLE IF NOT EXISTS `execute_history` ( `id` INT NOT NULL AUTO_INCREMENT, + `conn_id` INT NOT NULL, + `async_id` LONG NOT NULL DEFAULT 0, `execute_type` TINYINT NOT NULL, `content` VARCHAR(65535) NOT NULL, `execute_status` TINYINT NOT NULL, + `async_status` TINYINT NOT NULL DEFAULT 0, `duration` LONG NOT NULL, `create_time` DATETIME(6) NOT NULL, PRIMARY KEY (`id`) ); +CREATE INDEX IF NOT EXISTS `execute_history_conn_id` ON `execute_history`(`conn_id`); CREATE TABLE IF NOT EXISTS `gremlin_collection` ( `id` INT NOT NULL AUTO_INCREMENT, + `conn_id` INT NOT NULL, `name` VARCHAR(48) NOT NULL, `content` VARCHAR(65535) NOT NULL, `create_time` DATETIME(6) NOT NULL, PRIMARY KEY (`id`), - UNIQUE (`name`) + UNIQUE (`conn_id`, `name`) ); +CREATE INDEX IF NOT EXISTS `gremlin_collection_conn_id` ON `gremlin_collection`(`conn_id`); + +CREATE TABLE IF NOT EXISTS `file_mapping` ( + `id` INT NOT NULL AUTO_INCREMENT, + `conn_id` INT NOT NULL, + `job_id` INT NOT NULL DEFAULT 0, + `name` VARCHAR(128) NOT NULL, + `path` VARCHAR(256) NOT NULL, + `total_lines` LONG NOT NULL, + `total_size` LONG NOT NULL, + `file_status` TINYINT NOT NULL DEFAULT 0, + `file_setting` VARCHAR(65535) NOT NULL, + `vertex_mappings` VARCHAR(65535) NOT NULL, + `edge_mappings` VARCHAR(65535) NOT NULL, + `load_parameter` VARCHAR(65535) NOT NULL, + `create_time` DATETIME(6) NOT NULL, + `update_time` DATETIME(6) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE (`conn_id`, `job_id`, `name`) +); +CREATE INDEX IF NOT EXISTS `file_mapping_conn_id` ON `file_mapping`(`conn_id`); + +CREATE TABLE IF NOT EXISTS `load_task` ( + `id` INT NOT NULL AUTO_INCREMENT, + `conn_id` INT NOT NULL, + `job_id` INT NOT NULL DEFAULT 0, + `file_id` INT NOT NULL, + `file_name` VARCHAR(128) NOT NULL, + `options` VARCHAR(65535) NOT NULL, + `vertices` VARCHAR(512) NOT NULL, + `edges` VARCHAR(512) NOT NULL, + `file_total_lines` LONG NOT NULL, + `load_status` TINYINT NOT NULL, + `file_read_lines` LONG NOT NULL, + `last_duration` LONG NOT NULL, + `curr_duration` LONG NOT NULL, + `create_time` DATETIME(6) NOT NULL, + PRIMARY KEY (`id`) +); + +CREATE TABLE IF NOT EXISTS `job_manager` ( + `id` INT NOT NULL AUTO_INCREMENT, + `conn_id` INT NOT NULL DEFAULT 0, + `job_name` VARCHAR(100) NOT NULL DEFAULT '', + `job_remarks` VARCHAR(200) NOT NULL DEFAULT '', + `job_size` LONG NOT NULL DEFAULT 0, + `job_status` TINYINT NOT NULL DEFAULT 0, + `job_duration` LONG NOT NULL DEFAULT 0, + `update_time` DATETIME(6) NOT NULL, + `create_time` DATETIME(6) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE (`job_name`, `conn_id`) +); + +CREATE TABLE IF NOT EXISTS `async_task` ( + `id` INT NOT NULL AUTO_INCREMENT, + `conn_id` INT NOT NULL DEFAULT 0, + `task_id` INT NOT NULL DEFAULT 0, + `task_name` VARCHAR(100) NOT NULL DEFAULT '', + `task_reason` VARCHAR(200) NOT NULL DEFAULT '', + `task_type` TINYINT NOT NULL DEFAULT 0, + `algorithm_name` VARCHAR(48) NOT NULL DEFAULT '', + `task_content` VARCHAR(65535) NOT NULL DEFAULT '', + `task_status` TINYINT NOT NULL DEFAULT 0, + `task_duration` LONG NOT NULL DEFAULT 0, + `create_time` DATETIME(6) NOT NULL, + PRIMARY KEY (`id`) +); + +CREATE INDEX IF NOT EXISTS `load_task_conn_id` ON `load_task`(`conn_id`); +CREATE INDEX IF NOT EXISTS `load_task_file_id` ON `load_task`(`file_id`); diff --git a/hugegraph-hubble/hubble-be/src/test/resources/log4j2.xml b/hugegraph-hubble/hubble-be/src/test/resources/log4j2.xml new file mode 100644 index 000000000..fabc9216e --- /dev/null +++ b/hugegraph-hubble/hubble-be/src/test/resources/log4j2.xml @@ -0,0 +1,75 @@ + + + + + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hugegraph-hubble/pom.xml b/hugegraph-hubble/pom.xml index b17ddcc4c..d8cba158d 100644 --- a/hugegraph-hubble/pom.xml +++ b/hugegraph-hubble/pom.xml @@ -41,10 +41,18 @@ 2.1.8.RELEASE + 3.0.3 + + org.glassfish.jersey + jersey-bom + ${jersey.version} + pom + import + org.springframework.boot spring-boot-dependencies