diff --git a/BCLOUD b/BCLOUD
new file mode 100644
index 000000000..abcf9582e
--- /dev/null
+++ b/BCLOUD
@@ -0,0 +1 @@
+BUILD_SUBMITTER -x -e CENTOS6U3 -m baidu/xbu-data/hugegraph-loader -c "export MAVEN_HOME=/home/scmtools/buildkit/maven/apache-maven-3.3.9/ && export JAVA_HOME=/home/scmtools/buildkit/java/jdk1.8.0_25/ && export PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH && cd baidu/xbu-data/hugegraph-loader && sh build.sh && mkdir output && cp BCLOUD ./output/" -u ./
diff --git a/BUILDING.md b/BUILDING.md
new file mode 100644
index 000000000..a85b754df
--- /dev/null
+++ b/BUILDING.md
@@ -0,0 +1,25 @@
+Building hugegraph-loader
+
+--------------
+
+Required:
+
+* Java 8 (0.9 and later)
+
+* Maven
+
+To build without executing tests:
+
+```
+
+mvn clean install -DskipTests=true
+
+```
+
+To build with default tests:
+
+```
+
+mvn clean install
+
+```
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 000000000..e06d20818
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,202 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
diff --git a/README.md b/README.md
new file mode 100644
index 000000000..9b55472bd
--- /dev/null
+++ b/README.md
@@ -0,0 +1,18 @@
+# hugegraph-loader
+
+hugegraph-loader is a customizable command line utility for loading small to medium size graph datasets into the HugeGraph database from files with various input formats.
+
+## Features
+
+- Various input formats, such as json, csv, and text with any delimiters.
+- Diverse options, with which users can manage the data loading intuitively.
+- Detecting schema from data automatically, reduce the complex work of schema management.
+- Advanced customized operations with groovy script, users can configure how to construct vertices and edges by themselves.
+
+## Learn More
+
+The [project homepage](https://hugegraph.github.io/hugegraph-doc/) contains more information about hugegraph-loader.
+
+## License
+
+hugegraph-loader is licensed under Apache 2.0 License.
\ No newline at end of file
diff --git a/assembly/descriptor/assembly.xml b/assembly/descriptor/assembly.xml
new file mode 100644
index 000000000..e1a372891
--- /dev/null
+++ b/assembly/descriptor/assembly.xml
@@ -0,0 +1,54 @@
+
+ distribution
+ false
+
+
+ dir
+
+
+
+
+ ${assembly.static.dir}/bin
+ bin
+
+ *
+
+ 755
+
+
+ ${assembly.static.dir}
+ /
+ false
+
+
+ ${project.basedir}
+ /
+
+ README*
+ LICENSE*
+ NOTICE*
+
+
+
+ ${project.build.directory}
+ lib
+
+ *.jar
+
+
+
+
+
+
+
+ /lib
+ false
+ runtime
+ false
+
+ *:*:jar:*
+
+
+
+
+
\ No newline at end of file
diff --git a/assembly/static/bin/hugegraph-loader b/assembly/static/bin/hugegraph-loader
new file mode 100755
index 000000000..b4cb52c37
--- /dev/null
+++ b/assembly/static/bin/hugegraph-loader
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+abs_path() {
+ SOURCE="${BASH_SOURCE[0]}"
+ while [ -h "$SOURCE" ]; do
+ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
+ done
+ echo "$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+}
+
+BIN=`abs_path`
+TOP="$(cd $BIN/../ && pwd)"
+CONF="$TOP/conf"
+LIB="$TOP/lib"
+LOG="$TOP/logs"
+
+# Use the unofficial bash strict mode to avoid subtle bugs impossible.
+# Don't use -u option for now because LOADER_HOME might not yet defined.
+set -eo pipefail
+
+export VARS=${@:1}
+
+# Use JAVA_HOME if set, otherwise look for java in PATH
+if [ -n "$JAVA_HOME" ]; then
+ # Why we can't have nice things: Solaris combines x86 and x86_64
+ # installations in the same tree, using an unconventional path for the
+ # 64bit JVM. Since we prefer 64bit, search the alternate path first,
+ # (see https://issues.apache.org/jira/browse/CASSANDRA-4638).
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr
+ exit 1;
+fi
+
+# Add the slf4j-log4j12 binding
+CP=$(find -L $LIB -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':')
+# Add the jars in lib that start with "hugegraph"
+CP="$CP":$(find -L $LIB -name 'hugegraph*.jar' | sort | tr '\n' ':')
+# Add the remaining jars in lib.
+CP="$CP":$(find -L $LIB -name '*.jar' \
+ \! -name 'hugegraph*' \
+ \! -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':')
+
+export LOADER_CLASSPATH="${CLASSPATH:-}:$CP"
+
+# Xmx needs to be set so that it is big enough to cache all the vertexes in the run
+export JVM_OPTS="$JVM_OPTS -Xmx10g -cp $LOADER_CLASSPATH"
+
+# Uncomment to enable debugging
+#JVM_OPTS="$JVM_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=1414"
+
+exec "$JAVA" -Dname="HugeGraphLoader" -Dlog4j.configurationFile="$CONF/log4j2.xml" \
+$JVM_OPTS com.baidu.hugegraph.loader.HugeGraphLoader $VARS
diff --git a/assembly/static/conf/log4j2.xml b/assembly/static/conf/log4j2.xml
new file mode 100644
index 000000000..0ffd92504
--- /dev/null
+++ b/assembly/static/conf/log4j2.xml
@@ -0,0 +1,68 @@
+
+
+
+
+ UTF-8
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${log_charset}
+ %m%n
+
+
+
+
+ ${log_charset}
+ %m%n
+
+
+
+
+ ${log_charset}
+ %m%n
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/assembly/static/example/edge_created.json b/assembly/static/example/edge_created.json
new file mode 100644
index 000000000..c441ee672
--- /dev/null
+++ b/assembly/static/example/edge_created.json
@@ -0,0 +1,4 @@
+{"aname": "marko", "bname": "lop", "date": "20171210", "weight": 0.4}
+{"aname": "josh", "bname": "lop", "date": "20091111", "weight": 0.4}
+{"aname": "josh", "bname": "ripple", "date": "20171210", "weight": 1.0}
+{"aname": "peter", "bname": "lop", "date": "20170324", "weight": 0.2}
\ No newline at end of file
diff --git a/assembly/static/example/edge_knows.json b/assembly/static/example/edge_knows.json
new file mode 100644
index 000000000..216719278
--- /dev/null
+++ b/assembly/static/example/edge_knows.json
@@ -0,0 +1,2 @@
+{"source_name": "marko", "target_name": "vadas", "date": "20160110", "weight": 0.5}
+{"source_name": "marko", "target_name": "josh", "date": "20130220", "weight": 1.0}
\ No newline at end of file
diff --git a/assembly/static/example/schema.groovy b/assembly/static/example/schema.groovy
new file mode 100644
index 000000000..0fd5cf410
--- /dev/null
+++ b/assembly/static/example/schema.groovy
@@ -0,0 +1,24 @@
+// Define schema
+schema.propertyKey("name").asText().ifNotExist().create();
+schema.propertyKey("age").asInt().ifNotExist().create();
+schema.propertyKey("city").asText().ifNotExist().create();
+schema.propertyKey("weight").asDouble().ifNotExist().create();
+schema.propertyKey("lang").asText().ifNotExist().create();
+schema.propertyKey("date").asText().ifNotExist().create();
+schema.propertyKey("price").asDouble().ifNotExist().create();
+
+schema.vertexLabel("person").properties("name", "age", "city").primaryKeys("name").ifNotExist().create();
+schema.vertexLabel("software").properties("name", "lang", "price").primaryKeys("name").ifNotExist().create();
+
+schema.indexLabel("personByName").onV("person").by("name").secondary().ifNotExist().create();
+schema.indexLabel("personByAge").onV("person").by("age").range().ifNotExist().create();
+schema.indexLabel("personByCity").onV("person").by("city").secondary().ifNotExist().create();
+schema.indexLabel("personByAgeAndCity").onV("person").by("age", "city").secondary().ifNotExist().create();
+schema.indexLabel("softwareByPrice").onV("software").by("price").range().ifNotExist().create();
+
+schema.edgeLabel("knows").sourceLabel("person").targetLabel("person").properties("date", "weight").ifNotExist().create();
+schema.edgeLabel("created").sourceLabel("person").targetLabel("software").properties("date", "weight").ifNotExist().create();
+
+schema.indexLabel("createdByDate").onE("created").by("date").secondary().ifNotExist().create();
+schema.indexLabel("createdByWeight").onE("created").by("weight").range().ifNotExist().create();
+schema.indexLabel("knowsByWeight").onE("knows").by("weight").range().ifNotExist().create();
\ No newline at end of file
diff --git a/assembly/static/example/struct.json b/assembly/static/example/struct.json
new file mode 100644
index 000000000..63bc7f4a8
--- /dev/null
+++ b/assembly/static/example/struct.json
@@ -0,0 +1,59 @@
+{
+ "vertices": [
+ {
+ "label": "person",
+ "input": {
+ "type": "file",
+ "path": "example/vertex_person.csv",
+ "format": "CSV",
+ "header": ["name", "age", "city"],
+ "charset": "UTF-8"
+ },
+ "mapping": {
+ "name": "name",
+ "age": "age",
+ "city": "city"
+ }
+ },
+ {
+ "label": "software",
+ "input": {
+ "type": "file",
+ "path": "example/vertex_software.text",
+ "format": "TEXT",
+ "delimiter": "|",
+ "charset": "GBK"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "label": "knows",
+ "source": ["source_name"],
+ "target": ["target_name"],
+ "input": {
+ "type": "file",
+ "path": "example/edge_knows.json",
+ "format": "JSON"
+ },
+ "mapping": {
+ "source_name": "name",
+ "target_name": "name"
+ }
+ },
+ {
+ "label": "created",
+ "source": ["aname"],
+ "target": ["bname"],
+ "input": {
+ "type": "file",
+ "path": "example/edge_created.json",
+ "format": "JSON"
+ },
+ "mapping": {
+ "aname": "name",
+ "bname": "name"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/assembly/static/example/vertex_person.csv b/assembly/static/example/vertex_person.csv
new file mode 100644
index 000000000..00a34c0c8
--- /dev/null
+++ b/assembly/static/example/vertex_person.csv
@@ -0,0 +1,5 @@
+marko,29,Beijing
+vadas,27,Hongkong
+josh,32,Beijing
+peter,35,Shanghai
+"li,nary",26,"Wu,han"
\ No newline at end of file
diff --git a/assembly/static/example/vertex_software.text b/assembly/static/example/vertex_software.text
new file mode 100644
index 000000000..20b93ebb0
--- /dev/null
+++ b/assembly/static/example/vertex_software.text
@@ -0,0 +1,3 @@
+name|lang|price
+lop|java|328
+ripple|java|199
\ No newline at end of file
diff --git a/build.sh b/build.sh
new file mode 100644
index 000000000..0d403d93b
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+HUGEGRAPH_LOADER_RELEASE_PATH="${PWD}/output/"
+
+export MAVEN_HOME="/home/scmtools/buildkit/maven/apache-maven-3.3.9/"
+export JAVA_HOME="/home/scmtools/buildkit/java/jdk1.8.0_25/"
+export PATH="$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH"
+
+mvn clean compile
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 000000000..7cd550f28
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,152 @@
+
+
+ 4.0.0
+
+ com.baidu.hugegraph
+ hugegraph-loader
+ 0.6.1
+
+
+ hugegraph-loader
+ ${release.name}-${project.version}
+ ${project.basedir}/assembly
+ ${assembly.dir}/descriptor
+ ${assembly.dir}/static
+ bash
+ UTF-8
+ 1.8
+ 1.8
+ 2.8.2
+ 4.12
+ 21.0
+ 3.6
+
+
+
+
+ junit
+ junit
+ ${junit.version}
+ test
+
+
+ com.baidu.hugegraph
+ hugegraph-common
+ 1.4.4
+
+
+ com.baidu.hugegraph
+ hugegraph-client
+ 1.5.6
+
+
+ commons-io
+ commons-io
+ 2.6
+
+
+ org.codehaus.groovy
+ groovy-all
+ 2.4.6
+
+
+ com.beust
+ jcommander
+ 1.72
+
+
+ com.opencsv
+ opencsv
+ 4.2
+
+
+
+
+
+
+ maven-compiler-plugin
+ 3.1
+
+ ${compiler.source}
+ ${compiler.target}
+
+ 500
+
+
+ -Xlint:unchecked
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+ 2.4
+
+
+ assembly-hugegraph-loader
+ package
+
+ single
+
+
+ false
+ false
+ ${project.basedir}
+ ${assembly.descriptor.dir}/assembly.xml
+ ${final.name}
+
+
+
+
+
+ maven-antrun-plugin
+
+
+ package
+
+ run
+
+
+
+
+ tar -zcvf \
+ ${project.basedir}/${final.name}.tar.gz ${final.name} || exit 1
+ rm -f ${project.basedir}/dist.sh
+ echo -n "hugegraph-loader tar.gz available at: "
+ echo "${project.basedir}/${final.name}.tar.gz"
+
+
+
+
+
+
+
+
+
+
+ maven-clean-plugin
+ 3.0.0
+
+
+
+ ${project.basedir}
+
+ *.tar.gz
+ ${final.name}/**
+
+ false
+
+
+ ${final.name}
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/src/main/java/com/baidu/hugegraph/loader/HugeGraphLoader.java b/src/main/java/com/baidu/hugegraph/loader/HugeGraphLoader.java
new file mode 100644
index 000000000..9c61e89b6
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/HugeGraphLoader.java
@@ -0,0 +1,270 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+
+import com.baidu.hugegraph.driver.HugeClient;
+import com.baidu.hugegraph.loader.exception.LoadException;
+import com.baidu.hugegraph.loader.exception.ParseException;
+import com.baidu.hugegraph.loader.executor.GroovyExecutor;
+import com.baidu.hugegraph.loader.executor.HugeClients;
+import com.baidu.hugegraph.loader.executor.LoadLogger;
+import com.baidu.hugegraph.loader.executor.LoadOptions;
+import com.baidu.hugegraph.loader.executor.LoadSummary;
+import com.baidu.hugegraph.loader.parser.EdgeParser;
+import com.baidu.hugegraph.loader.parser.VertexParser;
+import com.baidu.hugegraph.loader.reader.InputReader;
+import com.baidu.hugegraph.loader.reader.InputReaderFactory;
+import com.baidu.hugegraph.loader.source.EdgeSource;
+import com.baidu.hugegraph.loader.source.GraphSource;
+import com.baidu.hugegraph.loader.source.VertexSource;
+import com.baidu.hugegraph.loader.task.TaskManager;
+import com.baidu.hugegraph.structure.graph.Edge;
+import com.baidu.hugegraph.structure.graph.Vertex;
+import com.baidu.hugegraph.util.E;
+import com.baidu.hugegraph.util.Log;
+import com.beust.jcommander.JCommander;
+
+public class HugeGraphLoader {
+
+ private static final Logger LOG = Log.logger(HugeGraphLoader.class);
+ private static final LoadLogger LOG_PARSE = LoadLogger.logger("parseError");
+
+ private final JCommander commander;
+ private final TaskManager taskManager;
+ private final GraphSource graphSource;
+
+ private long parseFailureNum = 0L;
+
+ public static void main(String[] args) {
+ HugeGraphLoader loader = new HugeGraphLoader(args);
+ LoadSummary summary = loader.load();
+ summary.print();
+ }
+
+ private HugeGraphLoader(String[] args) {
+ LoadOptions options = LoadOptions.instance();
+ this.commander = JCommander.newBuilder().addObject(options).build();
+ this.parseAndCheckOptions(args);
+ this.taskManager = new TaskManager(options);
+ this.graphSource = GraphSource.of(options.file);
+ }
+
+ private void parseAndCheckOptions(String[] args) {
+ this.commander.parse(args);
+ // Check options
+ LoadOptions options = LoadOptions.instance();
+ // Check option "-f"
+ E.checkArgument(!StringUtils.isEmpty(options.file),
+ "Must specified entrance groovy file");
+ File scriptFile = new File(options.file);
+ if (!scriptFile.canRead()) {
+ LOG.error("Script file must be readable: '{}'",
+ scriptFile.getAbsolutePath());
+ this.exitWithUsage(-1);
+ }
+ // Check option "-g"
+ E.checkArgument(!StringUtils.isEmpty(options.graph),
+ "Must specified a graph");
+ // Check option "-h"
+ if (!options.host.startsWith("http://")) {
+ options.host = "http://" + options.host;
+ }
+ }
+
+ private LoadSummary load() {
+ // Create schema
+ this.createSchema();
+
+ LoadSummary summary = new LoadSummary();
+ // Prepare to load vertices
+ Instant begTime = Instant.now();
+ System.out.print("Vertices has been imported: 0\b\b");
+ // Load vertices
+ this.loadVertices();
+ Instant endTime = Instant.now();
+ Duration duration = Duration.between(begTime, endTime);
+ summary.parseFailureVertices(this.parseFailureNum);
+ summary.insertFailureVertices(this.taskManager.failureNum());
+ summary.insertSuccessVertices(this.taskManager.successNum());
+ summary.vertexLoadTime(duration);
+ System.out.println(" " + summary.insertSuccessVertices());
+ // Reset counters
+ this.resetCounters();
+
+ // Prepare to load edges ...
+ begTime = Instant.now();
+ System.out.print("Edges has been imported: 0\b\b");
+ // Load edges
+ this.loadEdges();
+ endTime = Instant.now();
+ duration = Duration.between(begTime, endTime);
+ summary.parseFailureEdges(this.parseFailureNum);
+ summary.insertFailureEdges(this.taskManager.failureNum());
+ summary.insertSuccessEdges(this.taskManager.successNum());
+ summary.edgeLoadTime(duration);
+ System.out.println(" " + summary.insertSuccessEdges());
+ // Reset counters
+ this.resetCounters();
+
+ LoadOptions options = LoadOptions.instance();
+ // Shutdown task manager
+ this.taskManager.shutdown(options.shutdownTimeout);
+ return summary;
+ }
+
+ private void resetCounters() {
+ this.taskManager.cleanup();
+ this.parseFailureNum = 0L;
+ }
+
+ private void createSchema() {
+ LoadOptions options = LoadOptions.instance();
+ File schemaFile = FileUtils.getFile(options.schema);
+ HugeClient client = HugeClients.get(options);
+ GroovyExecutor groovyExecutor = new GroovyExecutor();
+ groovyExecutor.bind("schema", client.schema());
+ String script;
+ try {
+ script = FileUtils.readFileToString(schemaFile, "UTF-8");
+ } catch (IOException e) {
+ throw new LoadException("Read schema file '%s' error",
+ e, options.schema);
+ }
+ groovyExecutor.execute(script, client);
+ }
+
+ private void loadVertices() {
+ LoadOptions options = LoadOptions.instance();
+ List vertexSources = this.graphSource.vertexSources();
+ for (VertexSource source : vertexSources) {
+ InputReader reader = InputReaderFactory.create(source.input());
+ VertexParser parser = new VertexParser(source, reader);
+ this.loadVertex(parser);
+ try {
+ parser.close();
+ } catch (Exception e) {
+ LOG.warn("Failed to close parser for vertex source {}", source);
+ }
+ }
+ // Waiting async worker threads finish
+ this.taskManager.waitFinished(options.timeout);
+ }
+
+ private void loadVertex(VertexParser parser) {
+ LoadOptions options = LoadOptions.instance();
+ int batchSize = options.batchSize;
+ List batch = new ArrayList<>(batchSize);
+ while (parser.hasNext()) {
+ try {
+ Vertex vertex = parser.next();
+ batch.add(vertex);
+ } catch (ParseException e) {
+ if (options.testMode) {
+ throw e;
+ }
+ LOG.error("Vertex parse error", e);
+ LOG_PARSE.error(e);
+ if (++this.parseFailureNum >= options.maxParseErrors) {
+ exitWithInfo("vertices", options.maxParseErrors);
+ }
+ continue;
+ }
+ if (batch.size() >= batchSize) {
+ this.taskManager.submitVertexBatch(batch);
+ batch = new ArrayList<>(batchSize);
+ }
+ }
+ if (batch.size() > 0) {
+ this.taskManager.submitVertexBatch(batch);
+ }
+ }
+
+ private void loadEdges() {
+ LoadOptions options = LoadOptions.instance();
+ List edgeSources = this.graphSource.edgeSources();
+ for (EdgeSource source : edgeSources) {
+ InputReader reader = InputReaderFactory.create(source.input());
+ EdgeParser parser = new EdgeParser(source, reader);
+ this.loadEdge(parser);
+ try {
+ parser.close();
+ } catch (Exception e) {
+ LOG.warn("Failed to close parser for edge source {}", source);
+ }
+ }
+ // Waiting async worker threads finish
+ this.taskManager.waitFinished(options.timeout);
+ }
+
+ private void loadEdge(EdgeParser parser) {
+ LoadOptions options = LoadOptions.instance();
+ int batchSize = options.batchSize;
+ List batch = new ArrayList<>(batchSize);
+ while (parser.hasNext()) {
+ try {
+ Edge edge = parser.next();
+ batch.add(edge);
+ } catch (ParseException e) {
+ if (options.testMode) {
+ throw e;
+ }
+ LOG.error("Edge parse error", e);
+ LOG_PARSE.error(e);
+ if (++this.parseFailureNum >= options.maxParseErrors) {
+ exitWithInfo("edges", options.maxParseErrors);
+ }
+ continue;
+ }
+ if (batch.size() >= batchSize) {
+ this.taskManager.submitEdgeBatch(batch);
+ batch = new ArrayList<>(batchSize);
+ }
+ }
+ if (batch.size() > 0) {
+ this.taskManager.submitEdgeBatch(batch);
+ }
+ }
+
+ private void exitWithUsage(int status) {
+ this.commander.usage();
+ System.exit(status);
+ }
+
+ private static void exitWithInfo(String type, int parseErrors) {
+ LOG.error("Too many {} parse error ... Stopping", type);
+ // Print an empty line.
+ System.out.println();
+ System.out.println(String.format(
+ "Error: More than %s %s parsing error ... Stopping",
+ parseErrors, type));
+ System.exit(0);
+ }
+}
diff --git a/src/main/java/com/baidu/hugegraph/loader/exception/InsertException.java b/src/main/java/com/baidu/hugegraph/loader/exception/InsertException.java
new file mode 100644
index 000000000..154b6d840
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/exception/InsertException.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.exception;
+
+import com.baidu.hugegraph.structure.GraphElement;
+
+public class InsertException extends IllegalArgumentException {
+
+ private final GraphElement element;
+
+ public InsertException(GraphElement element, String message) {
+ super(message);
+ this.element = element;
+ }
+
+ public InsertException(GraphElement element, String message,
+ Throwable cause) {
+ super(message, cause);
+ this.element = element;
+ }
+
+ public InsertException(GraphElement element, String message,
+ Object... args) {
+ super(String.format(message, args));
+ this.element = element;
+ }
+
+ public InsertException(GraphElement element, String message,
+ Throwable cause, Object... args) {
+ super(String.format(message, args), cause);
+ this.element = element;
+ }
+
+ public GraphElement element() {
+ return this.element;
+ }
+}
diff --git a/src/main/java/com/baidu/hugegraph/loader/exception/LoadException.java b/src/main/java/com/baidu/hugegraph/loader/exception/LoadException.java
new file mode 100644
index 000000000..ed72d5e4f
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/exception/LoadException.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.exception;
+
+public class LoadException extends RuntimeException {
+
+ public LoadException(String message) {
+ super(message);
+ }
+
+ public LoadException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public LoadException(String message, Object... args) {
+ super(String.format(message, args));
+ }
+
+ public LoadException(String message, Throwable cause, Object... args) {
+ super(String.format(message, args), cause);
+ }
+}
diff --git a/src/main/java/com/baidu/hugegraph/loader/exception/ParseException.java b/src/main/java/com/baidu/hugegraph/loader/exception/ParseException.java
new file mode 100644
index 000000000..cc37ab709
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/exception/ParseException.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.exception;
+
+public class ParseException extends IllegalArgumentException {
+
+ private final String line;
+
+ public ParseException(String line, String message) {
+ super(message);
+ this.line = line;
+ }
+
+ public ParseException(String line, String message, Throwable cause) {
+ super(message, cause);
+ this.line = line;
+ }
+
+ public ParseException(String line, String message, Object... args) {
+ super(String.format(message, args));
+ this.line = line;
+ }
+
+ public ParseException(String line, String message, Throwable cause,
+ Object... args) {
+ super(String.format(message, args), cause);
+ this.line = line;
+ }
+
+ public String line() {
+ return this.line;
+ }
+}
diff --git a/src/main/java/com/baidu/hugegraph/loader/executor/GroovyExecutor.java b/src/main/java/com/baidu/hugegraph/loader/executor/GroovyExecutor.java
new file mode 100644
index 000000000..79e047816
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/executor/GroovyExecutor.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.executor;
+
+import org.codehaus.groovy.control.CompilerConfiguration;
+import org.codehaus.groovy.control.customizers.ImportCustomizer;
+import org.slf4j.Logger;
+
+import com.baidu.hugegraph.driver.HugeClient;
+import com.baidu.hugegraph.driver.SchemaManager;
+import com.baidu.hugegraph.util.Log;
+
+import groovy.lang.Binding;
+import groovy.lang.GroovyShell;
+import groovy.util.DelegatingScript;
+
+public class GroovyExecutor {
+
+ private static final Logger LOG = Log.logger(GroovyExecutor.class);
+
+ private final Binding binding;
+
+ public GroovyExecutor() {
+ this.binding = new Binding();
+ }
+
+ public void bind(String name, Object value) {
+ this.binding.setVariable(name, value);
+ }
+
+ public void execute(String groovyScript, HugeClient client) {
+ CompilerConfiguration config = new CompilerConfiguration();
+ config.setScriptBaseClass(DelegatingScript.class.getName());
+ ImportCustomizer importCustomizer = new ImportCustomizer();
+ importCustomizer.addImports(HugeClient.class.getName());
+ importCustomizer.addImports(SchemaManager.class.getName());
+ config.addCompilationCustomizers(importCustomizer);
+
+ GroovyShell shell = new GroovyShell(getClass().getClassLoader(),
+ this.binding, config);
+
+ // Groovy invoke java through the delegating script.
+ DelegatingScript script = (DelegatingScript) shell.parse(groovyScript);
+ script.setDelegate(client);
+ script.run();
+ }
+}
diff --git a/src/main/java/com/baidu/hugegraph/loader/executor/HugeClients.java b/src/main/java/com/baidu/hugegraph/loader/executor/HugeClients.java
new file mode 100644
index 000000000..5a2cced39
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/executor/HugeClients.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.executor;
+
+import com.baidu.hugegraph.driver.HugeClient;
+import com.baidu.hugegraph.loader.executor.LoadOptions;
+
+public class HugeClients {
+
+ // TODO: seems no need to use ThreadLocal, reuse HugeClient is ok
+ private static final ThreadLocal instance = new ThreadLocal<>();
+
+ public static HugeClient get(LoadOptions options) {
+ HugeClient client = instance.get();
+ if (client == null) {
+ client = newHugeClient(options);
+ instance.set(client);
+ }
+ return client;
+ }
+
+ private HugeClients() {}
+
+ private static HugeClient newHugeClient(LoadOptions options) {
+ String address = options.host + ":" + options.port;
+ return new HugeClient(address, options.graph, options.timeout);
+ }
+}
diff --git a/src/main/java/com/baidu/hugegraph/loader/executor/LoadLogger.java b/src/main/java/com/baidu/hugegraph/loader/executor/LoadLogger.java
new file mode 100644
index 000000000..c3cfc8841
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/executor/LoadLogger.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.executor;
+
+import org.slf4j.Logger;
+
+import com.baidu.hugegraph.loader.exception.InsertException;
+import com.baidu.hugegraph.loader.exception.ParseException;
+import com.baidu.hugegraph.util.JsonUtil;
+import com.baidu.hugegraph.util.Log;
+
+public class LoadLogger {
+
+ private final Logger log;
+
+ public static LoadLogger logger(String name) {
+ return new LoadLogger(name);
+ }
+
+ private LoadLogger(String name) {
+ this.log = Log.logger(name);
+ }
+
+ public void error(ParseException e) {
+ this.log.error(">>>> PARSE ERROR: {}", e.getMessage());
+ this.log.error("{}", e.line());
+ }
+
+ public void error(InsertException e) {
+ this.log.error(">>>> INSERT ERROR: {}", e.getMessage());
+ this.log.error("{}", JsonUtil.toJson(e.element()));
+ }
+}
diff --git a/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java b/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java
new file mode 100644
index 000000000..7e52702f3
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.executor;
+
+import java.io.File;
+
+import org.slf4j.Logger;
+
+import com.baidu.hugegraph.util.Log;
+import com.beust.jcommander.IParameterValidator;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+
+public class LoadOptions {
+
+ private Logger LOG = Log.logger(LoadOptions.class);
+
+ private static final LoadOptions instance = new LoadOptions();
+
+ public static LoadOptions instance() {
+ return instance;
+ }
+
+ private LoadOptions() {}
+
+ @Parameter(names = {"-f", "--file"}, required = true, arity = 1,
+ validateWith = {FileValidator.class},
+ description = "The path of the data source description file")
+ public String file;
+
+ @Parameter(names = {"-g", "--graph"}, required = true, arity = 1,
+ description = "The namespace of the graph to load into")
+ public String graph;
+
+ @Parameter(names = {"-s", "--schema"}, required = true, arity = 1,
+ validateWith = {FileValidator.class},
+ description = "The schema file path which to create manually")
+ public String schema;
+
+ @Parameter(names = {"-h", "--host"}, arity = 1,
+ validateWith = {UrlValidator.class},
+ description = "The host/IP of HugeGraphServer")
+ public String host = "localhost";
+
+ @Parameter(names = {"-p", "--port"}, arity = 1,
+ validateWith = {PositiveValidator.class},
+ description = "The port of HugeGraphServer")
+ public int port = 8080;
+
+ @Parameter(names = {"--num-threads"}, arity = 1,
+ validateWith = {PositiveValidator.class},
+ description = "The number of threads to use")
+ public int numThreads = Runtime.getRuntime().availableProcessors() * 2 - 1;
+
+ @Parameter(names = {"--batch-size"}, arity = 1,
+ validateWith = {PositiveValidator.class},
+ description = "The number of lines in each submit")
+ public int batchSize = 500;
+
+ @Parameter(names = {"--shutdown-timeout"}, arity = 1,
+ validateWith = {PositiveValidator.class},
+ description = "The timeout of awaitTermination in seconds")
+ public int shutdownTimeout = 10;
+
+ @Parameter(names = {"--check-vertex"}, arity = 1,
+ description = "Check vertices exists while inserting edges")
+ public boolean checkVertex = false;
+
+ @Parameter(names = {"--max-parse-errors"}, arity = 1,
+ validateWith = {PositiveValidator.class},
+ description = "The maximum number of rows that parse error " +
+ "before exiting")
+ public int maxParseErrors = 1;
+
+ @Parameter(names = {"--max-insert-errors"}, arity = 1,
+ validateWith = {PositiveValidator.class},
+ description = "The maximum number of rows that insert error " +
+ "before exiting")
+ public int maxInsertErrors = 500;
+
+ @Parameter(names = {"--timeout"}, arity = 1,
+ validateWith = {PositiveValidator.class},
+ description = "The timeout of inserting task in seconds")
+ public int timeout = 100;
+
+ @Parameter(names = {"--retry-times"}, arity = 1,
+ validateWith = {PositiveValidator.class},
+ description = "Setting the max retry times when loading timeout")
+ public int retryTimes = 0;
+
+ @Parameter(names = {"--retry-interval"}, arity = 1,
+ validateWith = {PositiveValidator.class},
+ description = "Setting the interval time before retrying")
+ public int retryInterval = 10;
+
+ @Parameter(names = {"--test-mode"}, arity = 1,
+ description = "Whether the hugegraph-loader work in test mode")
+ public boolean testMode = false;
+
+ public static class UrlValidator implements IParameterValidator {
+
+ @Override
+ public void validate(String name, String value) {
+ String regex = "^((http)?://)"
+ + "?(([0-9a-z_!~*'().&=+$%-]+: )?[0-9a-z_!~*'().&=+$%-]+@)?"
+ + "(([0-9]{1,3}\\.){3}[0-9]{1,3}" // IP URL, like: 10.0.0.1
+ + "|" // Or domain name
+ + "([0-9a-z_!~*'()-]+\\.)*" // Third level, like: www.
+ + "([0-9a-z][0-9a-z-]{0,61})?[0-9a-z]\\." // Second level
+ + "[a-z]{2,6})"; // First level, like: com or museum
+ if (!value.matches(regex)) {
+ throw new ParameterException(String.format(
+ "Invalid value of argument '%s': '%s'", name, value));
+ }
+ }
+ }
+
+ public static class DirectoryValidator implements IParameterValidator {
+
+ @Override
+ public void validate(String name, String value) {
+ File file = new File(value);
+ if (!file.exists() || !file.isDirectory()) {
+ throw new ParameterException(String.format(
+ "Ensure the directory '%s' exists and is indeed a " +
+ "directory instead of a file", value));
+ }
+ }
+ }
+
+ public static class FileValidator implements IParameterValidator {
+
+ @Override
+ public void validate(String name, String value) {
+ File file = new File(value);
+ if (!file.exists() || !file.isFile()) {
+ throw new ParameterException(String.format(
+ "Ensure the file '%s' exists and is indeed a file " +
+ "instead of a directory", value));
+ }
+ }
+ }
+
+ public static class PositiveValidator implements IParameterValidator {
+
+ @Override
+ public void validate(String name, String value) {
+ int retry = Integer.parseInt(value);
+ if (retry <= 0) {
+ throw new ParameterException(String.format(
+ "Parameter '%s' should be positive, but got '%s'",
+ name, value));
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/baidu/hugegraph/loader/executor/LoadSummary.java b/src/main/java/com/baidu/hugegraph/loader/executor/LoadSummary.java
new file mode 100644
index 000000000..e7b76633e
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/executor/LoadSummary.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.executor;
+
+import java.time.Duration;
+import java.util.Formatter;
+
+public class LoadSummary {
+
+ private static Formatter formatter = new Formatter(System.out);
+
+ private long parseFailureVertices;
+ private long insertFailureVertices;
+ private long insertSuccessVertices;
+
+ private long parseFailureEdges;
+ private long insertFailureEdges;
+ private long insertSuccessEdges;
+
+ private Duration vertexLoadTime;
+ private Duration edgeLoadTime;
+
+ public LoadSummary() {
+ this.vertexLoadTime = Duration.ZERO;
+ this.edgeLoadTime = Duration.ZERO;
+ }
+
+ public void print() {
+ System.out.println("-------------------------------------------------");
+ System.out.println("Vertex Results:");
+ printInFormat("Parse failure vertices", this.parseFailureVertices());
+ printInFormat("Insert failure vertices", this.insertFailureVertices());
+ printInFormat("Insert success vertices", this.insertSuccessVertices());
+
+ System.out.println("-------------------------------------------------");
+ System.out.println("Edge Results:");
+ printInFormat("Parse failure edges", this.parseFailureEdges());
+ printInFormat("Insert failure edges", this.insertFailureEdges());
+ printInFormat("Insert success edges", this.insertSuccessEdges());
+
+ System.out.println("-------------------------------------------------");
+ System.out.println("Time Results:");
+ printInFormat("Vertex loading time", this.vertexLoadTime().getSeconds());
+ printInFormat("Edge loading time", this.edgeLoadTime().getSeconds());
+ printInFormat("Total loading time", this.totalTime().getSeconds());
+ }
+
+ private static void printInFormat(String desc, long value) {
+ formatter.format("\t%-25s:\t%-20d%n", desc, value);
+ }
+
+ private Duration totalTime() {
+ return edgeLoadTime().plus(vertexLoadTime());
+ }
+
+ public Duration vertexLoadTime() {
+ return this.vertexLoadTime;
+ }
+
+ public void vertexLoadTime(Duration duration) {
+ this.vertexLoadTime = duration;
+ }
+
+ public Duration edgeLoadTime() {
+ return this.edgeLoadTime;
+ }
+
+ public void edgeLoadTime(Duration duration) {
+ this.edgeLoadTime = duration;
+ }
+
+ public long parseFailureVertices() {
+ return this.parseFailureVertices;
+ }
+
+ public void parseFailureVertices(long count) {
+ this.parseFailureVertices = count;
+ }
+
+ public long insertFailureVertices() {
+ return this.insertFailureVertices;
+ }
+
+ public void insertFailureVertices(long count) {
+ this.insertFailureVertices = count;
+ }
+
+ public long insertSuccessVertices() {
+ return this.insertSuccessVertices;
+ }
+
+ public void insertSuccessVertices(long count) {
+ this.insertSuccessVertices = count;
+ }
+
+ public long parseFailureEdges() {
+ return this.parseFailureEdges;
+ }
+
+ public void parseFailureEdges(long count) {
+ this.parseFailureEdges = count;
+ }
+
+ public long insertFailureEdges() {
+ return this.insertFailureEdges;
+ }
+
+ public void insertFailureEdges(long count) {
+ this.insertFailureEdges = count;
+ }
+
+ public long insertSuccessEdges() {
+ return this.insertSuccessEdges;
+ }
+
+ public void insertSuccessEdges(long count) {
+ this.insertSuccessEdges = count;
+ }
+}
diff --git a/src/main/java/com/baidu/hugegraph/loader/parser/EdgeParser.java b/src/main/java/com/baidu/hugegraph/loader/parser/EdgeParser.java
new file mode 100644
index 000000000..92d8defe3
--- /dev/null
+++ b/src/main/java/com/baidu/hugegraph/loader/parser/EdgeParser.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.parser;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import com.baidu.hugegraph.loader.reader.InputReader;
+import com.baidu.hugegraph.loader.source.EdgeSource;
+import com.baidu.hugegraph.structure.constant.IdStrategy;
+import com.baidu.hugegraph.structure.graph.Edge;
+import com.baidu.hugegraph.structure.schema.EdgeLabel;
+import com.baidu.hugegraph.structure.schema.VertexLabel;
+import com.baidu.hugegraph.util.E;
+
+public class EdgeParser extends ElementParser {
+
+ private final EdgeSource source;
+ private final EdgeLabel edgeLabel;
+ private final VertexLabel sourceLabel;
+ private final VertexLabel targetLabel;
+
+ public EdgeParser(EdgeSource source, InputReader reader) {
+ super(reader);
+ this.source = source;
+ this.edgeLabel = this.getEdgeLabel(source.label());
+ this.sourceLabel = this.getVertexLabel(this.edgeLabel.sourceLabel());
+ this.targetLabel = this.getVertexLabel(this.edgeLabel.targetLabel());
+ // Ensure that the source/target id fileds are matched with id strategy
+ this.checkIdFields(this.sourceLabel, this.source.sourceFields());
+ this.checkIdFields(this.targetLabel, this.source.targetFields());
+ }
+
+ @Override
+ public EdgeSource source() {
+ return this.source;
+ }
+
+ @Override
+ protected Edge parse(Map keyValues) {
+ Edge edge = new Edge(this.source.label());
+ // Must add source/target vertex id
+ edge.source(this.buildVertexId(this.sourceLabel,
+ this.source.sourceFields(), keyValues));
+ edge.target(this.buildVertexId(this.targetLabel,
+ this.source.targetFields(), keyValues));
+ // Must add source/target vertex label
+ edge.sourceLabel(this.sourceLabel.name());
+ edge.targetLabel(this.targetLabel.name());
+ // Add properties
+ this.addProperties(edge, keyValues);
+ return edge;
+ }
+
+ @Override
+ protected boolean isIdField(String fieldName) {
+ return this.source.sourceFields().contains(fieldName) ||
+ this.source.targetFields().contains(fieldName);
+ }
+
+ private Object buildVertexId(VertexLabel vertexLabel,
+ List fieldNames,
+ Map keyValues) {
+ List primaryKeys = vertexLabel.primaryKeys();
+ List