From a2540bb2373e15e45aaac2e0710b083f39707880 Mon Sep 17 00:00:00 2001 From: xiyang Date: Thu, 9 Nov 2023 00:18:37 +0800 Subject: [PATCH] Enable rdf test --- src/binder/bind/bind_graph_pattern.cpp | 11 +++- src/binder/bind/copy/bind_copy_rdf_graph.cpp | 9 +-- src/binder/bind/ddl/bind_create_rdf_graph.cpp | 4 +- src/include/binder/copy/bound_copy_from.h | 11 ++-- .../processor/operator/persistent/copy_rel.h | 1 + .../persistent/reader/rdf/rdf_reader.h | 4 -- src/planner/plan/append_extend.cpp | 4 +- .../persistent/reader/rdf/rdf_reader.cpp | 15 ++--- test/test_files/copy/copy_rdf.test | 7 +-- test/test_files/rdf/rdfox_example.test | 59 ++++++++++--------- test/test_files/rdf/tiny.test | 54 ----------------- 11 files changed, 63 insertions(+), 116 deletions(-) delete mode 100644 test/test_files/rdf/tiny.test diff --git a/src/binder/bind/bind_graph_pattern.cpp b/src/binder/bind/bind_graph_pattern.cpp index f81a44bbf2..d6c537acfd 100644 --- a/src/binder/bind/bind_graph_pattern.cpp +++ b/src/binder/bind/bind_graph_pattern.cpp @@ -266,8 +266,15 @@ std::shared_ptr Binder::createNonRecursiveQueryRel(const std::str if (readVersion->getTableSchema(tableIDs[0])->getTableType() == TableType::RDF) { auto predicateID = expressionBinder.bindNodeOrRelPropertyExpression(*queryRel, std::string(rdf::PID)); - auto resourceTableIDs = getNodeTableIDs(tableIDs); - auto resourceTableSchemas = readVersion->getTableSchemas(resourceTableIDs); + std::vector resourceTableIDs; + std::vector resourceTableSchemas; + for (auto& tableID : tableIDs) { + auto rdfGraphSchema = + reinterpret_cast(readVersion->getTableSchema(tableID)); + auto resourceTableID = rdfGraphSchema->getResourceTableID(); + resourceTableIDs.push_back(resourceTableID); + resourceTableSchemas.push_back(readVersion->getTableSchema(resourceTableID)); + } auto predicateIRI = createPropertyExpression(std::string(rdf::IRI), queryRel->getUniqueName(), queryRel->getVariableName(), resourceTableSchemas); auto rdfInfo = diff --git a/src/binder/bind/copy/bind_copy_rdf_graph.cpp b/src/binder/bind/copy/bind_copy_rdf_graph.cpp index a0f757d515..46a17c9e2f 100644 --- a/src/binder/bind/copy/bind_copy_rdf_graph.cpp +++ b/src/binder/bind/copy/bind_copy_rdf_graph.cpp @@ -75,10 +75,11 @@ std::unique_ptr Binder::bindCopyRdfRelFrom(function::TableFuncti copyFunc->bindFunc(clientContext, tableFuncBindInput.get(), catalog.getReadOnlyVersion()); auto boundFileScanInfo = std::make_unique( copyFunc, std::move(bindData), columns, relID, TableType::REL); - auto extraInfo = std::make_unique(columns[0], columns[1], columns[2]); - columns.push_back(std::move(relID)); - auto boundCopyFromInfo = std::make_unique(tableSchema, - std::move(boundFileScanInfo), containsSerial, std::move(columns), std::move(extraInfo)); + auto extraInfo = std::make_unique(columns[0], columns[2]); + expression_vector columnsToCopy = {columns[0], columns[2], relID, columns[1]}; + auto boundCopyFromInfo = + std::make_unique(tableSchema, std::move(boundFileScanInfo), + containsSerial, std::move(columnsToCopy), std::move(extraInfo)); return std::make_unique(std::move(boundCopyFromInfo)); } diff --git a/src/binder/bind/ddl/bind_create_rdf_graph.cpp b/src/binder/bind/ddl/bind_create_rdf_graph.cpp index e928c2765f..3f74803047 100644 --- a/src/binder/bind/ddl/bind_create_rdf_graph.cpp +++ b/src/binder/bind/ddl/bind_create_rdf_graph.cpp @@ -55,7 +55,7 @@ std::unique_ptr Binder::bindCreateRdfGraphInfo(const Creat auto resourceTripleTableName = getRdfResourceTripleTableName(rdfGraphName); std::vector> resourceTripleProperties; resourceTripleProperties.push_back(std::make_unique( - std::string(rdf::PID), std::make_unique(LogicalTypeID::INT64))); + std::string(rdf::PID), std::make_unique(LogicalTypeID::INTERNAL_ID))); auto boundResourceTripleExtraInfo = std::make_unique(RelMultiplicity::MANY_MANY, INVALID_TABLE_ID, INVALID_TABLE_ID, std::move(resourceTripleProperties)); @@ -65,7 +65,7 @@ std::unique_ptr Binder::bindCreateRdfGraphInfo(const Creat auto literalTripleTableName = getRdfLiteralTripleTableName(rdfGraphName); std::vector> literalTripleProperties; literalTripleProperties.push_back(std::make_unique( - std::string(rdf::PID), std::make_unique(LogicalTypeID::INT64))); + std::string(rdf::PID), std::make_unique(LogicalTypeID::INTERNAL_ID))); auto boundLiteralTripleExtraInfo = std::make_unique(RelMultiplicity::MANY_MANY, INVALID_TABLE_ID, INVALID_TABLE_ID, std::move(literalTripleProperties)); diff --git a/src/include/binder/copy/bound_copy_from.h b/src/include/binder/copy/bound_copy_from.h index 2a54dcdd92..d31ef6567d 100644 --- a/src/include/binder/copy/bound_copy_from.h +++ b/src/include/binder/copy/bound_copy_from.h @@ -69,16 +69,13 @@ struct ExtraBoundCopyRelInfo : public ExtraBoundCopyFromInfo { struct ExtraBoundCopyRdfRelInfo : public ExtraBoundCopyFromInfo { std::shared_ptr subjectOffset; - std::shared_ptr predicateOffset; std::shared_ptr objectOffset; - ExtraBoundCopyRdfRelInfo(std::shared_ptr subjectOffset, - std::shared_ptr predicateOffset, std::shared_ptr objectOffset) - : subjectOffset{std::move(subjectOffset)}, predicateOffset{std::move(predicateOffset)}, - objectOffset{std::move(objectOffset)} {} + ExtraBoundCopyRdfRelInfo( + std::shared_ptr subjectOffset, std::shared_ptr objectOffset) + : subjectOffset{std::move(subjectOffset)}, objectOffset{std::move(objectOffset)} {} ExtraBoundCopyRdfRelInfo(const ExtraBoundCopyRdfRelInfo& other) - : subjectOffset{other.subjectOffset}, predicateOffset{other.predicateOffset}, - objectOffset{other.objectOffset} {} + : subjectOffset{other.subjectOffset}, objectOffset{other.objectOffset} {} inline std::unique_ptr copy() final { return std::make_unique(*this); diff --git a/src/include/processor/operator/persistent/copy_rel.h b/src/include/processor/operator/persistent/copy_rel.h index 439f5a1973..d95191a1c6 100644 --- a/src/include/processor/operator/persistent/copy_rel.h +++ b/src/include/processor/operator/persistent/copy_rel.h @@ -18,6 +18,7 @@ struct CopyRelInfo { common::vector_idx_t partitioningIdx; common::RelDataDirection dataDirection; common::ColumnDataFormat dataFormat; + // TODO(Guodong): the following 3 fields are not being used. std::vector dataPoses; DataPos srcOffsetPos; DataPos relIDPos; diff --git a/src/include/processor/operator/persistent/reader/rdf/rdf_reader.h b/src/include/processor/operator/persistent/reader/rdf/rdf_reader.h index c519d37c72..a45ea121cc 100644 --- a/src/include/processor/operator/persistent/reader/rdf/rdf_reader.h +++ b/src/include/processor/operator/persistent/reader/rdf/rdf_reader.h @@ -48,10 +48,6 @@ class RDFReader { common::ValueVector* sVector; // subject common::ValueVector* pVector; // predicate common::ValueVector* oVector; // object - - std::unique_ptr sOffsetVector; - std::unique_ptr pOffsetVector; - std::unique_ptr oOffsetVector; }; struct RdfScanLocalState final : public function::TableFuncLocalState { diff --git a/src/planner/plan/append_extend.cpp b/src/planner/plan/append_extend.cpp index fe54a9461a..e037557002 100644 --- a/src/planner/plan/append_extend.cpp +++ b/src/planner/plan/append_extend.cpp @@ -121,12 +121,12 @@ void QueryPlanner::appendNonRecursiveExtend(std::shared_ptr boun appendNodeLabelFilter(nbrNode->getInternalID(), nbrNode->getTableIDsSet(), plan); } if (iri) { + auto rdfInfo = rel->getRdfPredicateInfo(); + appendFillTableID(rdfInfo->predicateID, rdfInfo->resourceTableIDs[0], plan); // Append hash join for remaining properties auto tmpPlan = std::make_unique(); - auto rdfInfo = rel->getRdfPredicateInfo(); cardinalityEstimator->addNodeIDDom(*rdfInfo->predicateID, rdfInfo->resourceTableIDs); appendScanInternalID(rdfInfo->predicateID, rdfInfo->resourceTableIDs, *tmpPlan); - appendFillTableID(rdfInfo->predicateID, rel->getSingleTableID(), *tmpPlan); appendScanNodeProperties( rdfInfo->predicateID, rdfInfo->resourceTableIDs, expression_vector{iri}, *tmpPlan); appendHashJoin(expression_vector{rdfInfo->predicateID}, JoinType::INNER, plan, *tmpPlan); diff --git a/src/processor/operator/persistent/reader/rdf/rdf_reader.cpp b/src/processor/operator/persistent/reader/rdf/rdf_reader.cpp index 0b9e93a065..5006318c30 100644 --- a/src/processor/operator/persistent/reader/rdf/rdf_reader.cpp +++ b/src/processor/operator/persistent/reader/rdf/rdf_reader.cpp @@ -31,9 +31,6 @@ RDFReader::RDFReader(std::string filePath, std::unique_ptr(fileName.c_str()), true); - sOffsetVector = std::make_unique(LogicalTypeID::INT64); - pOffsetVector = std::make_unique(LogicalTypeID::INT64); - oOffsetVector = std::make_unique(LogicalTypeID::INT64); counter = serd_reader_new( SERD_TURTLE, this, nullptr, nullptr, nullptr, counterStatementSink, nullptr); serd_reader_set_strict(counter, false /* strict */); @@ -180,9 +177,9 @@ SerdStatus RDFReader::readerStatementSink(void* handle, SerdStatementFlags /*fla auto subjectOffset = lookupResourceNode(reader->config->index, subject); auto predicateOffset = lookupResourceNode(reader->config->index, predicate); auto objectOffset = lookupResourceNode(reader->config->index, object); - reader->sOffsetVector->setValue(reader->rowOffset, subjectOffset); - reader->pOffsetVector->setValue(reader->rowOffset, predicateOffset); - reader->oOffsetVector->setValue(reader->rowOffset, objectOffset); + reader->sVector->setValue(reader->rowOffset, subjectOffset); + reader->pVector->setValue(reader->rowOffset, predicateOffset); + reader->oVector->setValue(reader->rowOffset, objectOffset); reader->vectorSize++; reader->rowOffset++; } break; @@ -193,9 +190,9 @@ SerdStatus RDFReader::readerStatementSink(void* handle, SerdStatementFlags /*fla auto subjectOffset = lookupResourceNode(reader->config->index, subject); auto predicateOffset = lookupResourceNode(reader->config->index, predicate); auto objectOffset = reader->rowOffset; - reader->sOffsetVector->setValue(reader->rowOffset, subjectOffset); - reader->pOffsetVector->setValue(reader->rowOffset, predicateOffset); - reader->oOffsetVector->setValue(reader->rowOffset, objectOffset); + reader->sVector->setValue(reader->rowOffset, subjectOffset); + reader->pVector->setValue(reader->rowOffset, predicateOffset); + reader->oVector->setValue(reader->rowOffset, objectOffset); reader->vectorSize++; reader->rowOffset++; } break; diff --git a/test/test_files/copy/copy_rdf.test b/test/test_files/copy/copy_rdf.test index 4add1840d2..40561136e7 100644 --- a/test/test_files/copy/copy_rdf.test +++ b/test/test_files/copy/copy_rdf.test @@ -1,19 +1,18 @@ -GROUP CopyRDFTest -DATASET CSV copy-test/rdf -BUFFER_POOL_SIZE 536870912 --SKIP -- -CASE CopyRDFTest -LOG CountRDFNodeTable --STATEMENT MATCH (s:taxonomy) RETURN COUNT(s.IRI) +-STATEMENT MATCH (s:taxonomy_resource_t) RETURN COUNT(s.iri) ---- 1 1138441 -LOG QueryRDFNodeTable --STATEMENT MATCH (s:taxonomy) RETURN s.IRI ORDER BY s.IRI LIMIT 5 +-STATEMENT MATCH (s:taxonomy_resource_t) RETURN s.iri ORDER BY s.iri LIMIT 5 ---- 5 http://dbpedia.org/class/yago/'hood108641944 http://dbpedia.org/class/yago/14July115200493 @@ -22,7 +21,7 @@ http://dbpedia.org/class/yago/16PF106475933 http://dbpedia.org/class/yago/1750s115149933 -LOG QueryRDFRelTable --STATEMENT MATCH (s:taxonomy)-[p:taxonomy]->(o:taxonomy) RETURN s.IRI, o.IRI ORDER BY s.IRI LIMIT 2 +-STATEMENT MATCH (s:taxonomy_resource_t)-[p:taxonomy]->(o:taxonomy_resource_t) RETURN s.iri, o.iri ORDER BY s.iri LIMIT 2 ---- 2 http://dbpedia.org/class/yago/'hood108641944|http://dbpedia.org/class/yago/Vicinity108641113 http://dbpedia.org/class/yago/14July115200493|http://dbpedia.org/class/yago/LegalHoliday115199592 diff --git a/test/test_files/rdf/rdfox_example.test b/test/test_files/rdf/rdfox_example.test index 17c5096380..1c03937c6b 100644 --- a/test/test_files/rdf/rdfox_example.test +++ b/test/test_files/rdf/rdfox_example.test @@ -78,33 +78,36 @@ Peter||||||||||||||||Peter ||||Monday|1999-08-16a |||||Petera --CASE tmp --SKIP - --STATEMENT MATCH (a:example_resource_t)-[e:example_resource_triples_t]->(b:example_resource_t) RETURN a.iri, b.iri +-STATEMENT MATCH (a:example_resource_t)-[e:example]->(b:example_resource_t) RETURN a.iri, e.iri, b.iri ORDER BY offset(id(e)) +-CHECK_ORDER ---- 10 -:peter|:Person -:peter|:lois -:lois|:Person -:meg|:Person -:meg|:lois -:meg|:peter -:chris|:Person -:chris|:peter -:stewie|:Person -:stewie|:lois +:peter|http://www.w3.org/1999/02/22-rdf-syntax-ns#type|:Person +:peter|:marriedTo|:lois +:lois|http://www.w3.org/1999/02/22-rdf-syntax-ns#type|:Person +:meg|http://www.w3.org/1999/02/22-rdf-syntax-ns#type|:Person +:meg|:hasParent|:lois +:meg|:hasParent|:peter +:chris|http://www.w3.org/1999/02/22-rdf-syntax-ns#type|:Person +:chris|:hasParent|:peter +:stewie|http://www.w3.org/1999/02/22-rdf-syntax-ns#type|:Person +:stewie|:hasParent|:lois --STATEMENT MATCH (a:example_resource_t)-[e:example_literal_triples_t]->(b:example_literal_t) RETURN a.iri, b.iri ----- 12 -:peter|Peter -:peter|male -:lois|Lois -:lois|female -:meg|Meg -:meg|female -:chris|Chris -:chris|male -:stewie|Stewie -:stewie|male -:brian|Brian -:andy|12 +-STATEMENT MATCH (a:example_resource_t)-[e:example]->(b:example_literal_t) RETURN a.iri, e.iri, b.iri ORDER BY offset(id(e)) +-CHECK_ORDER +---- 16 +:andy|:literal|12 +:andy|:literal|-14.900000 +:andy|:literal|True +:andy|:literal|0.016630 +:andy|:literal|1999-08-16 +:peter|:forename|Peter +:peter|:gender|male +:lois|:forename|Lois +:lois|:gender|female +:meg|:forename|Meg +:meg|:gender|female +:chris|:forename|Chris +:chris|:gender|male +:stewie|:forename|Stewie +:stewie|:gender|male +:brian|:forename|Brian diff --git a/test/test_files/rdf/tiny.test b/test/test_files/rdf/tiny.test deleted file mode 100644 index abbdb6c8ef..0000000000 --- a/test/test_files/rdf/tiny.test +++ /dev/null @@ -1,54 +0,0 @@ --GROUP CopyRDFTest --DATASET TTL rdf - --- - --CASE TinyRDFTest --SKIP --STATEMENT MATCH (s:tiny) RETURN id(s), s.IRI ----- 22 -1:0|http://dbpedia.org/class/yago/'hood108641944 -1:10|http://www.w3.org/2000/01/rdf-schema#subClassOf1 -1:11|http://www.w3.org/2000/01/rdf-schema#subClassOf2 -1:12|http://www.w3.org/2000/01/rdf-schema#subClassOf3 -1:13|http://www.w3.org/2000/01/rdf-schema#subClassOf4 -1:14|http://www.w3.org/2000/01/rdf-schema#subClassOf5 -1:15|http://www.w3.org/2000/01/rdf-schema#subClassOf6 -1:16|http://www.w3.org/2000/01/rdf-schema#subClassOf7 -1:17|http://www.w3.org/2000/01/rdf-schema#subClassOf8 -1:18|http://dbpedia.org/class/yago/Vicinity108641113 -1:19|http://dbpedia.org/class/yago/LegalHoliday115199592 -1:1|http://dbpedia.org/class/yago/14July115200493 -1:20|http://dbpedia.org/class/yago/Decade115204983 -1:21|http://dbpedia.org/class/yago/Self-reportPersonalityInventory106474603 -1:2|http://dbpedia.org/class/yago/1530s115148787 -1:3|http://dbpedia.org/class/yago/16PF106475933 -1:4|http://dbpedia.org/class/yago/1750s115149933 -1:5|http://dbpedia.org/class/yago/1760s115150304 -1:6|http://dbpedia.org/class/yago/1770s115150790 -1:7|http://dbpedia.org/class/yago/1780s115151175 -1:8|http://dbpedia.org/class/yago/1790s115151561 -1:9|http://www.w3.org/2000/01/rdf-schema#subClassOf0 - --STATEMENT MATCH (s:tiny)-[p:tiny]->(o:tiny) RETURN s.IRI, p.IRI, o.IRI ----- 9 -http://dbpedia.org/class/yago/'hood108641944|http://www.w3.org/2000/01/rdf-schema#subClassOf0|http://dbpedia.org/class/yago/Vicinity108641113 -http://dbpedia.org/class/yago/14July115200493|http://www.w3.org/2000/01/rdf-schema#subClassOf1|http://dbpedia.org/class/yago/LegalHoliday115199592 -http://dbpedia.org/class/yago/1530s115148787|http://www.w3.org/2000/01/rdf-schema#subClassOf2|http://dbpedia.org/class/yago/Decade115204983 -http://dbpedia.org/class/yago/16PF106475933|http://www.w3.org/2000/01/rdf-schema#subClassOf3|http://dbpedia.org/class/yago/Self-reportPersonalityInventory106474603 -http://dbpedia.org/class/yago/1750s115149933|http://www.w3.org/2000/01/rdf-schema#subClassOf4|http://dbpedia.org/class/yago/Decade115204983 -http://dbpedia.org/class/yago/1760s115150304|http://www.w3.org/2000/01/rdf-schema#subClassOf5|http://dbpedia.org/class/yago/Decade115204983 -http://dbpedia.org/class/yago/1770s115150790|http://www.w3.org/2000/01/rdf-schema#subClassOf6|http://dbpedia.org/class/yago/Decade115204983 -http://dbpedia.org/class/yago/1780s115151175|http://www.w3.org/2000/01/rdf-schema#subClassOf7|http://dbpedia.org/class/yago/Decade115204983 -http://dbpedia.org/class/yago/1790s115151561|http://www.w3.org/2000/01/rdf-schema#subClassOf8|http://dbpedia.org/class/yago/Decade115204983 - --STATEMENT MATCH (s:tiny) RETURN COUNT(s.IRI) ----- 1 -22 --STATEMENT MATCH (s:tiny) RETURN s.IRI ORDER BY s.IRI LIMIT 5 ----- 5 -http://dbpedia.org/class/yago/'hood108641944 -http://dbpedia.org/class/yago/14July115200493 -http://dbpedia.org/class/yago/1530s115148787 -http://dbpedia.org/class/yago/16PF106475933 -http://dbpedia.org/class/yago/1750s115149933