Skip to content

Commit

Permalink
refactor(interactive): Generate Stored Procedure Config Automatically…
Browse files Browse the repository at this point in the history 
… in Compiler (#3140)

<!--
Thanks for your contribution! please review
https://github.com/alibaba/GraphScope/blob/main/CONTRIBUTING.md before
opening an issue.
-->

## What do these changes do?
as titled.

<!-- Please give a short brief about these changes. -->

## Related issue number

<!-- Are there any issues opened that will be resolved by merging this
change? -->

Fixes

---------

Co-authored-by: xiaolei.zl <xiaolei.zl@alibaba-inc.com>
Co-authored-by: Longbin Lai <longbin.lailb@alibaba-inc.com>
  • Loading branch information
@shirly121 @zhanglei1949 @longbinlai
3 people authored Sep 1, 2023
1 parent 8950371 commit 8695655
Show file tree
Hide file tree
Showing 11 changed files with 446 additions and 159 deletions.
46 changes: 31 additions & 15 deletions flex/bin/load_plan_and_gen.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,15 +68,19 @@ fi
#fi

cypher_to_plan() {
if [ $# -ne 5 ]; then
echo "Usage: $0 <input_file> <output_file> <ir_compiler_properties> <graph_schema_path> <gie_home>, but receive: "$#
if [ $# -ne 7 ]; then
echo "Usage: $0 <query_name> <input_file> <output_plan file> <output_yaml_file>"
echo " <ir_compiler_properties> <graph_schema_path> <gie_home>, but receive: "$#
exit 1
fi
input_path=$1
output_path=$2
ir_compiler_properties=$3
graph_schema_path=$4
GIE_HOME=$5
query_name=$1
input_path=$2
output_path=$3
output_yaml_file=$4
ir_compiler_properties=$5
graph_schema_path=$6
GIE_HOME=$7

# find java executable
echo "IR compiler properties = ${ir_compiler_properties}"
#check file exists
Expand Down Expand Up @@ -109,33 +113,45 @@ cypher_to_plan() {
#get abs path of input_path
real_input_path=$(realpath ${input_path})
real_output_path=$(realpath ${output_path})
real_output_yaml=$(realpath ${output_yaml_file})

compiler_jar=${GIE_HOME}/compiler/target/compiler-0.0.1-SNAPSHOT.jar
if [ ! -f ${compiler_jar} ]; then
echo "Compiler jar = ${compiler_jar} not exists."
echo "Fail to find compiler jar."
exit 1
fi
# add extrac_key_value_config
extra_config="name:${query_name}"
extra_config="${extra_config},description:Autogenerated stored procedure configuration yaml for ${query_name}"

cmd="java -cp ${GIE_HOME}/compiler/target/libs/*:${compiler_jar}"
cmd="${cmd} -Dgraph.schema=${graph_schema_path}"
cmd="${cmd} -Djna.library.path=${GIE_HOME}/executor/ir/target/release/"
cmd="${cmd} com.alibaba.graphscope.common.ir.tools.GraphPlanner ${ir_compiler_properties} ${real_input_path} ${real_output_path}"
echo "running physical plan genration with "${cmd}
cmd="${cmd} com.alibaba.graphscope.common.ir.tools.GraphPlanner ${ir_compiler_properties} ${real_input_path} ${real_output_path} ${real_output_yaml} '${extra_config}'"
echo "running physical plan genration with ${cmd}"
eval ${cmd}

echo "---------------------------"
#check output
if [ ! -f ${output_path} ]; then
if [ ! -f ${real_output_path} ]; then
echo "Output file = ${output_path} not exists."
echo "Fail to generate physical plan."
exit 1
fi

#check output yaml file
if [ ! -f ${real_output_yaml} ]; then
echo "Output yaml file = ${output_yaml_file} not exists."
echo "Fail to generate physical plan."
exit 1
fi
}

compile_hqps_so() {
#check input params size eq 2 or 3
if [ $# -ne 5 ] && [ $# -ne 6 ]; then
echo "Usage: $0 <input_file> <work_dir> <ir_compiler_properties_file> <graph_schema_file> <GIE_HOME>[output_dir]"
echo "Usage: $0 <input_file> <work_dir> <ir_compiler_properties_file> <graph_schema_file> <GIE_HOME> [output_dir]"
exit 1
fi
input_path=$1
Expand Down Expand Up @@ -196,7 +212,8 @@ compile_hqps_so() {
echo "Generating code from cypher query"
# first do .cypher to .pb
output_pb_path="${cur_dir}/${query_name}.pb"
cypher_to_plan ${input_path} ${output_pb_path} ${ir_compiler_properties} ${graph_schema_path} ${gie_home}
output_yaml_path="${cur_dir}/${query_name}.yaml"
cypher_to_plan ${query_name} ${input_path} ${output_pb_path} ${output_yaml_path} ${ir_compiler_properties} ${graph_schema_path} ${gie_home}
echo "----------------------------"
echo "Codegen from cypher query done."
echo "----------------------------"
Expand Down Expand Up @@ -461,7 +478,6 @@ run() {
echo "GIE_HOME ="${GIE_HOME}
echo "Output path ="${OUTPUT_DIR}


# check input exist
if [ ! -f ${INPUT} ]; then
echo "Input file ${INPUT} not exists."
Expand All @@ -471,7 +487,7 @@ run() {
# if engine_type equals hqps
if [ ${ENGINE_TYPE} == "hqps" ]; then
echo "Engine type is hqps, generating dynamic library for hqps engine."
compile_hqps_so ${INPUT} ${WORK_DIR} ${IR_CONF} ${GRAPH_SCHEMA_PATH} ${GIE_HOME} ${OUTPUT_DIR}
compile_hqps_so ${INPUT} ${WORK_DIR} ${IR_CONF} ${GRAPH_SCHEMA_PATH} ${GIE_HOME} ${OUTPUT_DIR}

# else if engine_type equals pegasus
elif [ ${ENGINE_TYPE} == "pegasus" ]; then
Expand All @@ -484,7 +500,7 @@ run() {
exit 0
}

if [ $# -lt 6 ]; then
if [ $# -lt 5 ]; then
echo "only receives: $# args"
usage
exit 1
Expand Down
10 changes: 8 additions & 2 deletions interactive_engine/compiler/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ QUIET_OPT := --quiet

query:=
physical:=
procedure:=
extra:=
config.path:=conf/ir.compiler.properties

build:
Expand Down Expand Up @@ -54,13 +56,17 @@ run:
-Dpegasus.hosts=${pegasus.hosts} \
com.alibaba.graphscope.GraphServer ${config.path}

# make physical_plan config.path='<path to the config file>' query='<query in string>' physical='<path to the physical output file>'
# make physical_plan config.path='<path to the config file>'
# query='<query in string>'
# physical='<path to the physical output file>'
# procedure='<path_to_procedure_file>'
# extra='<extra_key_value_config_pairs>'
physical_plan:
cd $(CUR_DIR) && $(java) \
-cp ".:./target/libs/*:./target/compiler-0.0.1-SNAPSHOT.jar" \
-Djna.library.path=../executor/ir/target/release \
-Dgraph.schema=${graph.schema} \
com.alibaba.graphscope.common.ir.tools.GraphPlanner "${config.path}" "$(query)" "$(physical)"
com.alibaba.graphscope.common.ir.tools.GraphPlanner "${config.path}" "$(query)" "$(physical)" "$(procedure)" "$(extra)"

# start rpc server
# make run graph.schema:=../executor/ir/core/resource/ldbc_schema.json
Expand Down
111 changes: 1 addition & 110 deletions .../src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/GraphStoredProcedures.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,32 +17,20 @@
package com.alibaba.graphscope.common.ir.meta.procedure;

import com.alibaba.graphscope.common.ir.meta.reader.MetaDataReader;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
import org.apache.calcite.rel.type.*;
import org.apache.calcite.sql.type.SqlTypeName;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.yaml.snakeyaml.Yaml;

import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

public class GraphStoredProcedures implements StoredProcedures {
private final RelDataTypeFactory typeFactory;
private final Map<String, StoredProcedureMeta> storedProcedureMetaMap;

public GraphStoredProcedures(MetaDataReader reader) throws Exception {
this.typeFactory = new JavaTypeFactoryImpl();
this.storedProcedureMetaMap = Maps.newLinkedHashMap();
for (InputStream inputStream : reader.getStoredProcedures()) {
StoredProcedureMeta createdMeta = createStoredProcedureMeta(inputStream);
StoredProcedureMeta createdMeta = StoredProcedureMeta.Deserializer.perform(inputStream);
this.storedProcedureMetaMap.put(createdMeta.getName(), createdMeta);
}
}
Expand All @@ -51,101 +39,4 @@ public GraphStoredProcedures(MetaDataReader reader) throws Exception {
public @Nullable StoredProcedureMeta getStoredProcedure(String procedureName) {
return this.storedProcedureMetaMap.get(procedureName);
}

private StoredProcedureMeta createStoredProcedureMeta(InputStream inputStream)
throws IOException {
Yaml yaml = new Yaml();
Map<String, Object> config =
yaml.load(new String(inputStream.readAllBytes(), StandardCharsets.UTF_8));
Preconditions.checkArgument(config != null, "stored procedure meta data is null");
String procedureName = (String) config.get("name");
return new StoredProcedureMeta(
procedureName,
createReturnType((List) config.get("returns")),
createParameters((List) config.get("params")));
}

private RelDataType createReturnType(List config) {
List<RelDataTypeField> fields = Lists.newArrayList();
Iterator iterator = config.iterator();
int index = 0;
while (iterator.hasNext()) {
Map<String, Object> field = (Map<String, Object>) iterator.next();
fields.add(
new RelDataTypeFieldImpl(
(String) field.get("name"),
index,
createDataType((String) field.get("type"))));
++index;
}
return new RelRecordType(fields);
}

private List<StoredProcedureMeta.Parameter> createParameters(List config) {
List<StoredProcedureMeta.Parameter> parameters = Lists.newArrayList();
Iterator iterator = config.iterator();
while (iterator.hasNext()) {
Map<String, Object> parameter = (Map<String, Object>) iterator.next();
parameters.add(
new StoredProcedureMeta.Parameter(
(String) parameter.get("name"),
createDataType((String) parameter.get("type"))));
}
return parameters;
}

private RelDataType createDataType(String typeString) {
typeString = typeString.toUpperCase().replaceAll("\\s*", "");
switch (typeString) {
case "STRING":
return typeFactory.createSqlType(SqlTypeName.CHAR);
case "INTEGER":
return typeFactory.createSqlType(SqlTypeName.INTEGER);
case "BOOLEAN":
return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
case "FLOAT":
return typeFactory.createSqlType(SqlTypeName.FLOAT);
case "DOUBLE":
return typeFactory.createSqlType(SqlTypeName.DOUBLE);
case "LONG":
return typeFactory.createSqlType(SqlTypeName.BIGINT);
case "MULTISET(STRING)":
return typeFactory.createMultisetType(
typeFactory.createSqlType(SqlTypeName.CHAR), -1);
case "MULTISET(INTEGER)":
return typeFactory.createMultisetType(
typeFactory.createSqlType(SqlTypeName.INTEGER), -1);
case "MULTISET(BOOLEAN)":
return typeFactory.createMultisetType(
typeFactory.createSqlType(SqlTypeName.BOOLEAN), -1);
case "MULTISET(FLOAT)":
return typeFactory.createMultisetType(
typeFactory.createSqlType(SqlTypeName.FLOAT), -1);
case "MULTISET(DOUBLE)":
return typeFactory.createMultisetType(
typeFactory.createSqlType(SqlTypeName.DOUBLE), -1);
case "MULTISET(LONG)":
return typeFactory.createMultisetType(
typeFactory.createSqlType(SqlTypeName.BIGINT), -1);
case "ARRAY(STRING)":
return typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.CHAR), -1);
case "ARRAY(INTEGER)":
return typeFactory.createArrayType(
typeFactory.createSqlType(SqlTypeName.INTEGER), -1);
case "ARRAY(BOOLEAN)":
return typeFactory.createArrayType(
typeFactory.createSqlType(SqlTypeName.BOOLEAN), -1);
case "ARRAY(FLOAT)":
return typeFactory.createArrayType(
typeFactory.createSqlType(SqlTypeName.FLOAT), -1);
case "ARRAY(DOUBLE)":
return typeFactory.createArrayType(
typeFactory.createSqlType(SqlTypeName.DOUBLE), -1);
case "ARRAY(LONG)":
return typeFactory.createArrayType(
typeFactory.createSqlType(SqlTypeName.BIGINT), -1);
default:
throw new UnsupportedOperationException("unsupported type: " + typeString);
}
}
}
Loading

0 comments on commit 8695655

Please sign in to comment.