From e3792923afeb731848e29dd7991fc33e16c3be3d Mon Sep 17 00:00:00 2001 From: Sharayu Date: Fri, 14 May 2021 12:01:07 -0700 Subject: [PATCH] Add correct yaml files for real time data(#6787) (#6916) --- .../config/FeatureTest2-schema.json | 94 +++++++++++++++++++ .../data/RealtimeFeatureTest1-data-00.csv | 11 +++ ...eature-test-2-realtime-stream-config.json} | 2 +- ...time.json => feature-test-2-realtime.json} | 8 +- .../feature-test-2-sql-realtime.queries | 62 ++++++++++++ .../feature-test-2-sql-realtime.results | 63 +++++++++++++ .../post-broker-rollback.yaml | 12 +++ .../post-controller-rollback.yaml | 12 +++ .../post-server-rollback.yaml | 12 +++ .../post-server-upgrade.yaml | 12 +++ .../sample-test-suite/pre-broker-upgrade.yaml | 12 +++ .../pre-controller-upgrade.yaml | 22 +++++ .../sample-test-suite/pre-server-upgrade.yaml | 12 +++ 13 files changed, 330 insertions(+), 4 deletions(-) create mode 100644 compatibility-verifier/sample-test-suite/config/FeatureTest2-schema.json create mode 100644 compatibility-verifier/sample-test-suite/config/data/RealtimeFeatureTest1-data-00.csv rename compatibility-verifier/sample-test-suite/config/{feature-test-1-realtime-stream-config.json => feature-test-2-realtime-stream-config.json} (79%) rename compatibility-verifier/sample-test-suite/config/{feature-test-1-realtime.json => feature-test-2-realtime.json} (87%) create mode 100644 compatibility-verifier/sample-test-suite/config/queries/feature-test-2-sql-realtime.queries create mode 100644 compatibility-verifier/sample-test-suite/config/query-results/feature-test-2-sql-realtime.results diff --git a/compatibility-verifier/sample-test-suite/config/FeatureTest2-schema.json b/compatibility-verifier/sample-test-suite/config/FeatureTest2-schema.json new file mode 100644 index 00000000000..f53c5a2c863 --- /dev/null +++ b/compatibility-verifier/sample-test-suite/config/FeatureTest2-schema.json @@ -0,0 +1,94 @@ +{ + "dimensionFieldSpecs": [ + { + "dataType": "INT", + "name": "generationNumber" + }, + { + "dataType": "STRING", + "name": "stringDimSV1" + }, + { + "dataType": "STRING", + "name": "stringDimSV2" + }, + { + "dataType": "LONG", + "name": "longDimSV1" + }, + { + "dataType": "LONG", + "name": "longDimSV2" + }, + { + "dataType": "STRING", + "name": "stringDimMV1", + "singleValueField": false + }, + { + "dataType": "STRING", + "name": "stringDimMV2", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "intDimMV1", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "intDimMV2", + "singleValueField": false + }, + { + "dataType": "STRING", + "maxLength": 1000, + "name": "textDim1" + }, + { + "dataType": "BYTES", + "name": "bytesDimSV1" + }, + { + "dataType": "STRING", + "name": "mapDim1__KEYS", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "mapDim1__VALUES", + "singleValueField": false + }, + { + "dataType": "STRING", + "name": "mapDim2json" + } + ], + "metricFieldSpecs": [ + { + "dataType": "INT", + "name": "intMetric1" + }, + { + "dataType": "LONG", + "name": "longMetric1" + }, + { + "dataType": "FLOAT", + "name": "floatMetric1" + }, + { + "dataType": "DOUBLE", + "name": "doubleMetric1" + } + ], + "dateTimeFieldSpecs" : [ + { + "name" : "HoursSinceEpoch", + "dataType" : "INT", + "format" : "1:HOURS:EPOCH", + "granularity": "1:HOURS" + } + ], + "schemaName": "FeatureTest2" +} diff --git a/compatibility-verifier/sample-test-suite/config/data/RealtimeFeatureTest1-data-00.csv b/compatibility-verifier/sample-test-suite/config/data/RealtimeFeatureTest1-data-00.csv new file mode 100644 index 00000000000..e4e36e47051 --- /dev/null +++ b/compatibility-verifier/sample-test-suite/config/data/RealtimeFeatureTest1-data-00.csv @@ -0,0 +1,11 @@ +# HoursSinceEpoch generationNumber stringDimSV1 stringDimSV2 longDimSV1 longDimSV2 stringDimMV1 stringDimMV2 intDimMV1 intDimMV2 textDim1 mapDim1__KEYS mapDim1__VALUES mapDim2json intMetric1 longMetric1 floatMetric1 doubleMetric1 +123456,__GENERATION_NUMBER__,"s1-0",s2-0,1,2,m1-0-0;m1-0-1,m2-0-0;m2-0-1,3;4,6;7,Java C++ Python,01a0bc,k1;k2;k3;k4;k5,1;1;2;2;2,"{""k1"":1,""k2"":1,""k3"":2,""k4"":2,""k5"":2}",10,11,12.1,13.1 +123456,__GENERATION_NUMBER__,"s1-0",s2-0,1,2,m1-0-0;m1-0-1,m2-0-0;m2-0-1,3;4,6;7,Java C++ Python,4877625602,k1;k2;k3;k4;k5,3;3;3;3;3,"{""k1"":3,""k2"":3,""k3"":3,""k4"":3,""k5"":3}",10,11,12.1,13.1, # Dupliate of row 0 1 +123456,__GENERATION_NUMBER__,s1-2,s2-2,11,21,m1-2-0;m1-2-1,m2-2-0;m2-2-1,32;42,62;72,Java C++ golang,13225573e3f5,k1;k2;k3;k4;k5,4;5;6;7;7,"{""k1"":4,""k2"":5,""k3"":6,""k4"":7,""k5"":7}",10,21,22.1,23.10 +123456,__GENERATION_NUMBER__,s1-2,s2-2,11,21,m1-3-0;m1-3-1,m2-3-0;m2-3-1,32;42,62;72,Java C++ golang,deadbeef,k1;k2;k3;k4;k5,7;7;7;7;7,"{""k1"":7,""k2"":7,""k3"":7,""k4"":7,""k5"":7}",10,21,22.1,23.10, # All sv cols same as prev +123456,__GENERATION_NUMBER__,s1-4,s2-4,41,22,m1-2-0;m1-2-1,m2-2-0;m2-2-1,42;52,72;82,Java C++ golang,deed0507,k1;k2;k3;k4;k5,7;7;8;8;8,"{""k1"":7,""k2"":7,""k3"":8,""k4"":8,""k5"":8}",14,24,24.1,24.10, # All mv cols same as row 2 +123456,__GENERATION_NUMBER__,s1-5,,,32,m1-5-0,m2-2-0,,92;22,golang shell bash,,k1;k2;k3;k4;k5,7;7;7;7;7,"{""k1"":7,""k2"":7,""k3"":7,""k4"":7,""k5"":7}",,24,,24.10, # Default values for some columns +123456,__GENERATION_NUMBER__,s1-6,s2-6,7611,7621,m1-5-0;m1-5-1;m1-5-2,m2-2-0,392;462,6662;782,C++ golang python,deed0507,k1;k2;k3;k4;k5,7;8;9;10;20,"{""k1"":7,""k2"":8,""k3"":9,""k4"":10,""k5"":20}",101,251,262.1,263.10, # 3 values in MV +123456,__GENERATION_NUMBER__,s1-6,s2-6,7611,7621,m1-5-0;m1-5-1;m1-5-2,m2-2-0,392;462,6662;782,C++ golang python,deed0507,k1;k2;k3;k4;k5,7;8;9;10;20,"{""k1"":7,""k2"":8,""k3"":9,""k4"":10,""k5"":20}",2147483647,251,262.1,263.10, # MAX_INT in int metric +123456,__GENERATION_NUMBER__,s1-6,s2-6,7611,7621,m1-5-0;m1-5-1;m1-5-2,m2-2-0,392;462,6662;782,C++ golang python,deed0507,k1;k2;k3;k4;k5,7;8;9;10;20,"{""k1"":7,""k2"":8,""k3"":9,""k4"":10,""k5"":20}",2147483647,251,262.1,263.10, # MAX_INT in int metric +123456,__GENERATION_NUMBER__,s1-7,s2-7,6766,6777,m1-6-0;m1-6-1;m1-6-2;m1-6-3,m2-6-0;m2-6-1,392;462,6662;782,golang Java,d54d0507,k1;k2;k3;k4;k5,31;31;32;32;32,"{""k1"":31,""k2"":31,""k3"":32,""k4"":32,""k5"":32}",87,251,262.10,263.10 diff --git a/compatibility-verifier/sample-test-suite/config/feature-test-1-realtime-stream-config.json b/compatibility-verifier/sample-test-suite/config/feature-test-2-realtime-stream-config.json similarity index 79% rename from compatibility-verifier/sample-test-suite/config/feature-test-1-realtime-stream-config.json rename to compatibility-verifier/sample-test-suite/config/feature-test-2-realtime-stream-config.json index de17987c0d5..441ce201a90 100644 --- a/compatibility-verifier/sample-test-suite/config/feature-test-1-realtime-stream-config.json +++ b/compatibility-verifier/sample-test-suite/config/feature-test-2-realtime-stream-config.json @@ -1,7 +1,7 @@ { "streamType": "kafka", "stream.kafka.consumer.type": "simple", - "topicName": "PinotRealtimeFeatureTest1Event", + "topicName": "PinotRealtimeFeatureTest2Event", "partitionColumn": "longDimSV1", "numPartitions": "1", "stream.kafka.consumer.prop.auto.offset.reset": "smallest" diff --git a/compatibility-verifier/sample-test-suite/config/feature-test-1-realtime.json b/compatibility-verifier/sample-test-suite/config/feature-test-2-realtime.json similarity index 87% rename from compatibility-verifier/sample-test-suite/config/feature-test-1-realtime.json rename to compatibility-verifier/sample-test-suite/config/feature-test-2-realtime.json index 86b56cb43a7..6df335e4884 100644 --- a/compatibility-verifier/sample-test-suite/config/feature-test-1-realtime.json +++ b/compatibility-verifier/sample-test-suite/config/feature-test-2-realtime.json @@ -20,7 +20,7 @@ "replication": "1", "retentionTimeUnit": "", "retentionTimeValue": "", - "schemaName": "FeatureTest1", + "schemaName": "FeatureTest2", "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", "segmentPushFrequency": "daily", "segmentPushType": "APPEND", @@ -40,8 +40,10 @@ "segmentFormatVersion": "v3", "sortedColumn": [], "streamConfigs": { + "realtime.segment.flush.threshold.size": "63", + "realtime.segment.flush.threshold.time": "1h", "streamType": "kafka", - "stream.kafka.topic.name": "PinotRealtimeFeatureTest1Event", + "stream.kafka.topic.name": "PinotRealtimeFeatureTest2Event", "stream.kafka.consumer.type": "simple", "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", @@ -50,7 +52,7 @@ "stream.kafka.consumer.prop.auto.offset.reset": "largest" } }, - "tableName": "FeatureTest1_REALTIME", + "tableName": "FeatureTest2", "tableType": "REALTIME", "tenants": { "broker": "DefaultTenant", diff --git a/compatibility-verifier/sample-test-suite/config/queries/feature-test-2-sql-realtime.queries b/compatibility-verifier/sample-test-suite/config/queries/feature-test-2-sql-realtime.queries new file mode 100644 index 00000000000..b03d86f1c6e --- /dev/null +++ b/compatibility-verifier/sample-test-suite/config/queries/feature-test-2-sql-realtime.queries @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Selection +SELECT longDimSV1, intDimMV1 FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) ORDER BY longDimSV1, intDimMV1 +SELECT longDimSV1, intDimMV1 FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) LIMIT 5 +SELECT longDimSV1, intDimMV1 FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) LIMIT 200 + +# Aggregation +SELECT count(*) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ +SELECT sum(intMetric1), sumMV(intDimMV1), min(intMetric1), minMV(intDimMV2), max(longDimSV1), maxMV(intDimMV1) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ +SELECT count(longDimSV1), countMV(intDimMV1), avg(floatMetric1), avgMV(intDimMV2), minMaxRange(doubleMetric1), minMaxRangeMV(intDimMV2) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ +SELECT percentile(longDimSV1, 80), percentileMV(intDimMV1, 90), percentileEst(longDimSV1, 80), percentileEstMV(intDimMV1, 90), percentileTDigest(longDimSV1, 80), percentileTDigestMV(intDimMV1, 90) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ +SELECT percentile(longDimSV1, 80.01), percentileMV(intDimMV1, 99.99), percentileEst(longDimSV1, 80.01), percentileEstMV(intDimMV1, 99.99), percentileTDigest(longDimSV1, 80.01), percentileTDigestMV(intDimMV1, 99.99) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ +SELECT distinctCount(longDimSV1), distinctCountMV(intDimMV1), distinctCountHLL(longDimSV1), distinctCountHLLMV(intDimMV1) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ + +# Selection & Filtering & Grouping on Aggregation +SELECT longDimSV1, intDimMV1, count(*) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) GROUP BY longDimSV1, intDimMV1 ORDER BY longDimSV1, intDimMV1 LIMIT 20 +SELECT longDimSV1, intDimMV1, sum(intMetric1), sumMV(intDimMV1), min(intMetric1), minMV(intDimMV2), max(longDimSV1), maxMV(intDimMV1) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) GROUP BY longDimSV1, intDimMV1 ORDER BY longDimSV1, intDimMV1 LIMIT 20 +SELECT longDimSV1, intDimMV1, count(longDimSV1), countMV(intDimMV1), avg(floatMetric1), avgMV(intDimMV2), minMaxRange(doubleMetric1), minMaxRangeMV(intDimMV2) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) GROUP BY longDimSV1, intDimMV1 ORDER BY longDimSV1, intDimMV1 LIMIT 20 +SELECT longDimSV1, intDimMV1, percentile(longDimSV1, 80), percentileMV(intDimMV1, 90), percentileEst(longDimSV1, 80), percentileEstMV(intDimMV1, 90), percentileTDigest(longDimSV1, 80), percentileTDigestMV(intDimMV1, 90) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) GROUP BY longDimSV1, intDimMV1 ORDER BY longDimSV1, intDimMV1 LIMIT 20 +SELECT longDimSV1, intDimMV1, percentile(longDimSV1, 80.01), percentileMV(intDimMV1, 99.99), percentileEst(longDimSV1, 80.01), percentileEstMV(intDimMV1, 99.99), percentileTDigest(longDimSV1, 80.01), percentileTDigestMV(intDimMV1, 99.99) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) GROUP BY longDimSV1, intDimMV1 ORDER BY longDimSV1, intDimMV1 LIMIT 20 +SELECT longDimSV1, intDimMV1, distinctCount(longDimSV1), distinctCountMV(intDimMV1), distinctCountHLL(longDimSV1), distinctCountHLLMV(intDimMV1) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) GROUP BY longDimSV1, intDimMV1 ORDER BY longDimSV1, intDimMV1 LIMIT 20 + +# Transformation Functions +SELECT DISTINCT add(longDimSV1, sub(longDimSV2, 3)), mod(intMetric1, 10), div(doubleMetric1, mult(floatMetric1, 5)) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ ORDER BY add(longDimSV1, sub(longDimSV2, 3)) DESC, mod(intMetric1, 10), div(doubleMetric1, mult(floatMetric1, 5)) +SELECT DISTINCT floor(sqrt(doubleMetric1)), ceil(ln(longDimSV1)), exp(mod(abs(longDimSV2), 3)) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ ORDER BY floor(sqrt(doubleMetric1)), ceil(ln(longDimSV1)), exp(mod(abs(longDimSV2), 3)) DESC +SELECT arrayLength(intDimMV1), arrayLength(valueIn(stringDimMV2, 'm2-2-0', 'm2-3-0')), count(*) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ GROUP BY arrayLength(intDimMV1), arrayLength(valueIn(stringDimMV2, 'm2-2-0', 'm2-3-0')) +SELECT valueIn(intDimMV1, 3, 32), count(*) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ GROUP BY valueIn(intDimMV1, 3, 32) +SELECT DISTINCT upper(stringDimSV1), lower(textDim1), reverse(stringDimSV2), ltrim(substr(textDim1, 4, 9)), rtrim(substr(textDim1, 4, 9)) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ ORDER BY upper(stringDimSV1) +SELECT DISTINCT stringDimSV2, replace(stringDimSV2, 'foo', 'bar'), codePoint(stringDimSV2), rpad(stringDimSV2, 11, 'abc'), lpad(stringDimSV2, 11, 'xyz') from FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND(strPos(stringDimSV2, '2', 2) > 1 OR startsWith(stringDimSV2, 'foo') = 'true') ORDER BY codePoint(stringDimSV2), replace(stringDimSV2, 'foo', 'bar') + +# Groovy Scripts +SELECT DISTINCT longDimSV1, longDimSV2, groovy('{"returnType":"LONG","isSingleValue":true}', 'arg0 + arg1', longDimSV1, longDimSV2) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ +SELECT count(*), groovy('{"returnType":"STRING", "isSingleValue":true}', 'def result; if (arg0 < 0) { result = "NEGATIVE"; } else if (arg0 < 10) { result = "SMALL";} else if (arg0 < 50) { result = "MEDIUM";} else{result = "LARGE"}; return result', longDimSV1) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ GROUP BY groovy('{"returnType":"STRING", "isSingleValue":true}', 'def result; if (arg0 < 0) { result = "NEGATIVE"; } else if (arg0 < 10) { result = "SMALL";} else if (arg0 < 50) { result = "MEDIUM";} else{result = "LARGE"}; return result', longDimSV1) ORDER BY count(*) DESC +SELECT DISTINCT groovy('{"returnType":"INT","isSingleValue":true}', 'arg0.toList().max()', intDimMV1) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ ORDER BY groovy('{"returnType":"INT","isSingleValue":true}', 'arg0.toList().max()', intDimMV1) + +# Json Map +SELECT DISTINCT stringDimSV1, mapValue(mapDim1__KEYS, 'k1', mapDim1__VALUES), mapValue(mapDim1__KEYS, 'k4', mapDim1__VALUES) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ LIMIT 20 +SELECT DISTINCT stringDimSV1, jsonExtractScalar(mapDim2json, '$.k1', 'INT'), jsonExtractScalar(mapDim2json, '$.k4', 'INT') FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ ORDER BY stringDimSV1, jsonExtractScalar(mapDim2json, '$.k1', 'INT'), jsonExtractScalar(mapDim2json, '$.k4', 'INT') LIMIT 20 +SELECT jsonExtractScalar(mapDim2json, '$.k1', 'INT'), max(jsonExtractScalar(mapDim2json, '$.k2', 'INT')), min(jsonExtractScalar(mapDim2json, '$.k3', 'INT')) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ GROUP BY jsonExtractScalar(mapDim2json, '$.k1', 'INT') ORDER BY max(jsonExtractScalar(mapDim2json, '$.k2', 'INT')) LIMIT 20 + +# Misc +SELECT longDimSV1, count(*) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND REGEXP_LIKE(textDim1, '^Java.*') GROUP BY longDimSV1 +SELECT stringDimMV2, count(*) FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND REGEXP_LIKE(stringDimMV2, 'm2.*0') GROUP BY stringDimMV2 LIMIT 20 +SELECT stringDimSV1, longDimSV1, intDimMV1, intDimMV2 FROM FeatureTest2 WHERE generationNumber = __GENERATION_NUMBER__ AND (stringDimSV1 != 's1-6' AND longDimSV1 BETWEEN 10 AND 1000 OR (intDimMV1 < 42 AND stringDimMV2 IN ('m2-0-0', 'm2-2-0') AND intDimMV2 NOT IN (6,72))) ORDER BY longDimSV1 DESC, stringDimSV1 LIMIT 100 diff --git a/compatibility-verifier/sample-test-suite/config/query-results/feature-test-2-sql-realtime.results b/compatibility-verifier/sample-test-suite/config/query-results/feature-test-2-sql-realtime.results new file mode 100644 index 00000000000..4d94ca8be45 --- /dev/null +++ b/compatibility-verifier/sample-test-suite/config/query-results/feature-test-2-sql-realtime.results @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Selection +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","intDimMV1"],"columnDataTypes":["LONG","INT_ARRAY"]},"rows":[[-9223372036854775808,[-2147483648]],[-9223372036854775808,[-2147483648]],[-9223372036854775808,[-2147483648]],[-9223372036854775808,[-2147483648]],[-9223372036854775808,[-2147483648]],[-9223372036854775808,[-2147483648]],[-9223372036854775808,[-2147483648]],[11,[32,42]],[11,[32,42]],[11,[32,42]]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":28,"numEntriesScannedInFilter":269,"numEntriesScannedPostFilter":56,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":10,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","intDimMV1"],"columnDataTypes":["LONG","INT_ARRAY"]},"rows":[[11,[32,42]],[11,[32,42]],[41,[42,52]],[-9223372036854775808,[-2147483648]],[11,[32,42]]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":5,"numEntriesScannedInFilter":63,"numEntriesScannedPostFilter":10,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":6,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","intDimMV1"],"columnDataTypes":["LONG","INT_ARRAY"]},"rows":[[11,[32,42]],[11,[32,42]],[41,[42,52]],[-9223372036854775808,[-2147483648]],[11,[32,42]],[11,[32,42]],[41,[42,52]],[-9223372036854775808,[-2147483648]],[11,[32,42]],[11,[32,42]],[41,[42,52]],[-9223372036854775808,[-2147483648]],[11,[32,42]],[11,[32,42]],[41,[42,52]],[-9223372036854775808,[-2147483648]],[11,[32,42]],[11,[32,42]],[41,[42,52]],[-9223372036854775808,[-2147483648]],[11,[32,42]],[11,[32,42]],[41,[42,52]],[-9223372036854775808,[-2147483648]],[11,[32,42]],[11,[32,42]],[41,[42,52]],[-9223372036854775808,[-2147483648]]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":28,"numEntriesScannedInFilter":269,"numEntriesScannedPostFilter":56,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":4,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} + +# Aggregation +{"resultTable":{"dataSchema":{"columnNames":["count(*)"],"columnDataTypes":["LONG"]},"rows":[[66]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":0,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":4,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["sum(intMetric1)","summv(intDimMV1)","min(intMetric1)","minmv(intDimMV2)","max(longDimSV1)","maxmv(intDimMV1)"],"columnDataTypes":["DOUBLE","DOUBLE","DOUBLE","DOUBLE","DOUBLE","DOUBLE"]},"rows":[[2.576980527E10,-1.5032363248E10,0.0,6.0,7611.0,462.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":264,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":4,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["count(*)","countmv(intDimMV1)","avg(floatMetric1)","avgmv(intDimMV2)","minmaxrange(doubleMetric1)","minmaxrangemv(intDimMV2)"],"columnDataTypes":["LONG","LONG","DOUBLE","DOUBLE","DOUBLE","DOUBLE"]},"rows":[[66,125,105.11969939145175,1383.2575757575758,250.00000000000003,6656.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":264,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":5,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["percentile(longDimSV1, 80.0)","percentilemv(intDimMV1, 90.0)","percentileest(longDimSV1, 80.0)","percentileestmv(intDimMV1, 90.0)","percentiletdigest(longDimSV1, 80.0)","percentiletdigestmv(intDimMV1, 90.0)"],"columnDataTypes":["DOUBLE","DOUBLE","LONG","LONG","DOUBLE","DOUBLE"]},"rows":[[7611.0,462.0,7611,462,7611.0,462.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":132,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":9,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["percentile(longDimSV1, 80.01)","percentilemv(intDimMV1, 99.99)","percentileest(longDimSV1, 80.01)","percentileestmv(intDimMV1, 99.99)","percentiletdigest(longDimSV1, 80.01)","percentiletdigestmv(intDimMV1, 99.99)"],"columnDataTypes":["DOUBLE","DOUBLE","LONG","LONG","DOUBLE","DOUBLE"]},"rows":[[7611.0,462.0,7611,462,7611.0,462.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":132,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":9,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["distinctcount(longDimSV1)","distinctcountmv(intDimMV1)","distinctcounthll(longDimSV1)","distinctcounthllmv(intDimMV1)"],"columnDataTypes":["INT","INT","LONG","LONG"]},"rows":[[6,8,6,8]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":132,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":5,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} + +# Selection & Filtering & Grouping on Aggregation +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","intDimMV1","count(*)"],"columnDataTypes":["LONG","INT","LONG"]},"rows":[[-9223372036854775808,-2147483648,7],[11,32,14],[11,42,14],[41,42,7],[41,52,7]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":28,"numEntriesScannedInFilter":269,"numEntriesScannedPostFilter":56,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":6,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","intDimMV1","sum(intMetric1)","summv(intDimMV1)","min(intMetric1)","minmv(intDimMV2)","max(longDimSV1)","maxmv(intDimMV1)"],"columnDataTypes":["LONG","INT","DOUBLE","DOUBLE","DOUBLE","DOUBLE","DOUBLE","DOUBLE"]},"rows":[[-9223372036854775808,-2147483648,0.0,-1.5032385536E10,0.0,22.0,-9.223372036854776E18,-2.147483648E9],[11,32,140.0,1036.0,10.0,62.0,11.0,42.0],[11,42,140.0,1036.0,10.0,62.0,11.0,42.0],[41,42,98.0,658.0,14.0,72.0,41.0,52.0],[41,52,98.0,658.0,14.0,72.0,41.0,52.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":28,"numEntriesScannedInFilter":269,"numEntriesScannedPostFilter":112,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":9,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","intDimMV1","count(*)","countmv(intDimMV1)","avg(floatMetric1)","avgmv(intDimMV2)","minmaxrange(doubleMetric1)","minmaxrangemv(intDimMV2)"],"columnDataTypes":["LONG","INT","LONG","LONG","DOUBLE","DOUBLE","DOUBLE","DOUBLE"]},"rows":[[-9223372036854775808,-2147483648,7,7,0.0,57.0,0.0,70.0],[11,32,14,28,22.100000381469727,67.0,0.0,10.0],[11,42,14,28,22.100000381469727,67.0,0.0,10.0],[41,42,7,14,24.100000381469727,77.0,0.0,10.0],[41,52,7,14,24.100000381469727,77.0,0.0,10.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":28,"numEntriesScannedInFilter":269,"numEntriesScannedPostFilter":140,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":5,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","intDimMV1","percentile(longDimSV1, 80.0)","percentilemv(intDimMV1, 90.0)","percentileest(longDimSV1, 80.0)","percentileestmv(intDimMV1, 90.0)","percentiletdigest(longDimSV1, 80.0)","percentiletdigestmv(intDimMV1, 90.0)"],"columnDataTypes":["LONG","INT","DOUBLE","DOUBLE","LONG","LONG","DOUBLE","DOUBLE"]},"rows":[[-9223372036854775808,-2147483648,-9.223372036854776E18,-2.147483648E9,-9223372036854775808,-2147483648,-9.223372036854776E18,-2.147483648E9],[11,32,11.0,42.0,11,42,11.0,42.0],[11,42,11.0,42.0,11,42,11.0,42.0],[41,42,41.0,52.0,41,52,41.0,52.0],[41,52,41.0,52.0,41,52,41.0,52.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":28,"numEntriesScannedInFilter":269,"numEntriesScannedPostFilter":56,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":8,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","intDimMV1","percentile(longDimSV1, 80.01)","percentilemv(intDimMV1, 99.99)","percentileest(longDimSV1, 80.01)","percentileestmv(intDimMV1, 99.99)","percentiletdigest(longDimSV1, 80.01)","percentiletdigestmv(intDimMV1, 99.99)"],"columnDataTypes":["LONG","INT","DOUBLE","DOUBLE","LONG","LONG","DOUBLE","DOUBLE"]},"rows":[[-9223372036854775808,-2147483648,-9.223372036854776E18,-2.147483648E9,-9223372036854775808,-2147483648,-9.223372036854776E18,-2.147483648E9],[11,32,11.0,42.0,11,42,11.0,42.0],[11,42,11.0,42.0,11,42,11.0,42.0],[41,42,41.0,52.0,41,52,41.0,52.0],[41,52,41.0,52.0,41,52,41.0,52.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":28,"numEntriesScannedInFilter":269,"numEntriesScannedPostFilter":56,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":12,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","intDimMV1","distinctcount(longDimSV1)","distinctcountmv(intDimMV1)","distinctcounthll(longDimSV1)","distinctcounthllmv(intDimMV1)"],"columnDataTypes":["LONG","INT","INT","INT","LONG","LONG"]},"rows":[[-9223372036854775808,-2147483648,1,1,1,1],[11,32,1,2,1,2],[11,42,1,2,1,2],[41,42,1,2,1,2],[41,52,1,2,1,2]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":28,"numEntriesScannedInFilter":269,"numEntriesScannedPostFilter":56,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":7,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} + +# Transformation Functions +{"resultTable":{"dataSchema":{"columnNames":["add(longDimSV1,sub(longDimSV2,'3'))","mod(intMetric1,'10')","div(doubleMetric1,mult(floatMetric1,'5'))"],"columnDataTypes":["DOUBLE","DOUBLE","DOUBLE"]},"rows":[[15229.0,1.0,0.20076306285631254],[15229.0,7.0,0.20076306285631254],[13540.0,7.0,0.20076306285631254],[60.0,4.0,0.1999999968342762],[29.0,0.0,0.20904977014723267],[0.0,0.0,0.21652891879345226],[-9.223372036854776E18,0.0,"Infinity"]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":330,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":31,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["floor(sqrt(doubleMetric1))","ceil(ln(longDimSV1))","exp(mod(abs(longDimSV2),'3'))"],"columnDataTypes":["DOUBLE","DOUBLE","DOUBLE"]},"rows":[[3.0,0.0,7.38905609893065],[4.0,3.0,1.0],[4.0,4.0,2.718281828459045],[4.0,"NaN",7.38905609893065],[16.0,9.0,2.718281828459045],[16.0,9.0,1.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":198,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":7,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["arraylength(intDimMV1)","arraylength(valuein(stringDimMV2,'m2-2-0','m2-3-0'))","count(*)"],"columnDataTypes":["INT","INT","LONG"]},"rows":[[2,1,39],[1,1,7],[2,0,20]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":132,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":9,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["valuein(intDimMV1,'3','32')","count(*)"],"columnDataTypes":["INT","LONG"]},"rows":[[3,14],[32,14]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":66,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":5,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["upper(stringDimSV1)","lower(textDim1)","reverse(stringDimSV2)","ltrim(substr(textDim1,'4','9'))","rtrim(substr(textDim1,'4','9'))"],"columnDataTypes":["STRING","STRING","STRING","STRING","STRING"]},"rows":[["S1-0","java c++ python","0-2s","C++ "," C++"],["S1-2","java c++ golang","2-2s","C++ "," C++"],["S1-4","java c++ golang","4-2s","C++ "," C++"],["S1-5","golang shell bash","llun","ng sh","ng sh"],["S1-6","c++ golang python","6-2s","golan","golan"],["S1-7","golang java","7-2s","ng Ja","ng Ja"]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":198,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":7,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV2","replace(stringDimSV2,'foo','bar')","codepoint(stringDimSV2)","rpad(stringDimSV2,'11','abc')","lpad(stringDimSV2,'11','xyz')"],"columnDataTypes":["STRING","STRING","INT","STRING","STRING"]},"rows":[["s2-2","s2-2",115,"s2-2abcabca","xyzxyzxs2-2"]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":14,"numEntriesScannedInFilter":132,"numEntriesScannedPostFilter":14,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":10,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} + +# Groovy Scripts +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","longDimSV2","groovy('{\"returnType\":\"LONG\",\"isSingleValue\":true}','arg0 + arg1',longDimSV1,longDimSV2)"],"columnDataTypes":["LONG","LONG","LONG"]},"rows":[[7611,7621,15232],[1,2,3],[41,22,63],[6766,6777,13543],[-9223372036854775808,32,-9223372036854775776],[11,21,32]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":132,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":15,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["count(*)","groovy('{\"returnType\":\"STRING\", \"isSingleValue\":true}','def result; if (arg0 < 0) { result = \"NEGATIVE\"; } else if (arg0 < 10) { result = \"SMALL\";} else if (arg0 < 50) { result = \"MEDIUM\";} else{result = \"LARGE\"}; return result',longDimSV1)"],"columnDataTypes":["LONG","STRING"]},"rows":[[24,"LARGE"],[21,"MEDIUM"],[14,"SMALL"],[7,"NEGATIVE"]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":66,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":20,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["groovy('{\"returnType\":\"INT\",\"isSingleValue\":true}','arg0.toList().max()',intDimMV1)"],"columnDataTypes":["INT"]},"rows":[[-2147483648],[4],[42],[52],[462]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":66,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":17,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} + +# Json Map +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV1","mapvalue(mapDim1__KEYS,'k1',mapDim1__VALUES)","mapvalue(mapDim1__KEYS,'k4',mapDim1__VALUES)"],"columnDataTypes":["STRING","INT","INT"]},"rows":[["s1-2",7,7],["s1-7",31,32],["s1-4",7,8],["s1-2",4,7],["s1-6",7,10],["s1-5",7,7],["s1-0",1,2],["s1-0",3,3]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":198,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":7,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV1","jsonextractscalar(mapDim2json,'$.k1','INT')","jsonextractscalar(mapDim2json,'$.k4','INT')"],"columnDataTypes":["STRING","INT","INT"]},"rows":[["s1-0",1,2],["s1-0",3,3],["s1-2",4,7],["s1-2",7,7],["s1-4",7,8],["s1-5",7,7],["s1-6",7,10],["s1-7",31,32]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":132,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":9,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["jsonextractscalar(mapDim2json,'$.k1','INT')","max(jsonextractscalar(mapDim2json,'$.k2','INT'))","min(jsonextractscalar(mapDim2json,'$.k3','INT'))"],"columnDataTypes":["INT","DOUBLE","DOUBLE"]},"rows":[[1,1.0,2.0],[3,3.0,3.0],[4,5.0,6.0],[7,8.0,7.0],[31,31.0,32.0]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":0,"numEntriesScannedPostFilter":66,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":13,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} + +# Misc +{"resultTable":{"dataSchema":{"columnNames":["longDimSV1","count(*)"],"columnDataTypes":["LONG","LONG"]},"rows":[[1,14],[41,7],[11,14]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":35,"numEntriesScannedInFilter":66,"numEntriesScannedPostFilter":35,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":4,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["stringDimMV2","count(*)"],"columnDataTypes":["STRING","LONG"]},"rows":[["m2-6-0",6],["m2-6-1",6],["m2-3-1",7],["m2-2-0",39],["m2-3-0",7],["m2-2-1",14],["m2-0-0",14],["m2-0-1",14]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":66,"numEntriesScannedInFilter":107,"numEntriesScannedPostFilter":66,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":3,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} +{"resultTable":{"dataSchema":{"columnNames":["stringDimSV1","longDimSV1","intDimMV1","intDimMV2"],"columnDataTypes":["STRING","LONG","INT_ARRAY","INT_ARRAY"]},"rows":[["s1-4",41,[42,52],[72,82]],["s1-4",41,[42,52],[72,82]],["s1-4",41,[42,52],[72,82]],["s1-4",41,[42,52],[72,82]],["s1-4",41,[42,52],[72,82]],["s1-4",41,[42,52],[72,82]],["s1-4",41,[42,52],[72,82]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-2",11,[32,42],[62,72]],["s1-5",-9223372036854775808,[-2147483648],[92,22]],["s1-5",-9223372036854775808,[-2147483648],[92,22]],["s1-5",-9223372036854775808,[-2147483648],[92,22]],["s1-5",-9223372036854775808,[-2147483648],[92,22]],["s1-5",-9223372036854775808,[-2147483648],[92,22]],["s1-5",-9223372036854775808,[-2147483648],[92,22]],["s1-5",-9223372036854775808,[-2147483648],[92,22]]]},"exceptions":[],"numServersQueried":1,"numServersResponded":1,"numSegmentsQueried":1,"numSegmentsProcessed":1,"numSegmentsMatched":1,"numConsumingSegmentsQueried":1,"numDocsScanned":28,"numEntriesScannedInFilter":269,"numEntriesScannedPostFilter":140,"numGroupsLimitReached":false,"totalDocs":66,"timeUsedMs":5,"offlineThreadCpuTimeNs":0,"realtimeThreadCpuTimeNs":0,"segmentStatistics":[],"traceInfo":{},"minConsumingFreshnessTimeMs":1620950080428} + diff --git a/compatibility-verifier/sample-test-suite/post-broker-rollback.yaml b/compatibility-verifier/sample-test-suite/post-broker-rollback.yaml index 14b01e436c7..855cd260f56 100644 --- a/compatibility-verifier/sample-test-suite/post-broker-rollback.yaml +++ b/compatibility-verifier/sample-test-suite/post-broker-rollback.yaml @@ -28,7 +28,19 @@ operations: tableConfigFileName: feature-test-1.json recordReaderConfigFileName: data/recordReaderConfig.json segmentName: FeatureTest1_Segment6 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/RealtimeFeatureTest1-data-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json - type: queryOp description: Run query on FeatureTest1 using SQL queryFileName: queries/feature-test-1-sql.queries expectedResultsFileName: query-results/feature-test-1-rest-sql.results + - type: queryOp + description: Run query on FeatureTest2 using SQL + queryFileName: queries/feature-test-2-sql-realtime.queries + expectedResultsFileName: query-results/feature-test-2-sql-realtime.results diff --git a/compatibility-verifier/sample-test-suite/post-controller-rollback.yaml b/compatibility-verifier/sample-test-suite/post-controller-rollback.yaml index 7b39bf06544..3f7be07b23f 100644 --- a/compatibility-verifier/sample-test-suite/post-controller-rollback.yaml +++ b/compatibility-verifier/sample-test-suite/post-controller-rollback.yaml @@ -28,10 +28,22 @@ operations: tableConfigFileName: feature-test-1.json recordReaderConfigFileName: data/recordReaderConfig.json segmentName: FeatureTest1_Segment7 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/RealtimeFeatureTest1-data-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json - type: queryOp description: Run query on FeatureTest1 using SQL queryFileName: queries/feature-test-1-sql.queries expectedResultsFileName: query-results/feature-test-1-rest-sql.results + - type: queryOp + description: Run query on FeatureTest2 using SQL + queryFileName: queries/feature-test-2-sql-realtime.queries + expectedResultsFileName: query-results/feature-test-2-sql-realtime.results - type: segmentOp description: Delete segment FeatureTest1_Segment op: DELETE diff --git a/compatibility-verifier/sample-test-suite/post-server-rollback.yaml b/compatibility-verifier/sample-test-suite/post-server-rollback.yaml index 08f89c7d48d..13e58f3cfad 100644 --- a/compatibility-verifier/sample-test-suite/post-server-rollback.yaml +++ b/compatibility-verifier/sample-test-suite/post-server-rollback.yaml @@ -28,7 +28,19 @@ operations: tableConfigFileName: feature-test-1.json recordReaderConfigFileName: data/recordReaderConfig.json segmentName: FeatureTest1_Segment5 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/RealtimeFeatureTest1-data-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json - type: queryOp description: Run query on FeatureTest1 using SQL queryFileName: queries/feature-test-1-sql.queries expectedResultsFileName: query-results/feature-test-1-rest-sql.results + - type: queryOp + description: Run query on FeatureTest2 using SQL + queryFileName: queries/feature-test-2-sql-realtime.queries + expectedResultsFileName: query-results/feature-test-2-sql-realtime.results diff --git a/compatibility-verifier/sample-test-suite/post-server-upgrade.yaml b/compatibility-verifier/sample-test-suite/post-server-upgrade.yaml index 1efba7fbc73..a581cf6f89a 100644 --- a/compatibility-verifier/sample-test-suite/post-server-upgrade.yaml +++ b/compatibility-verifier/sample-test-suite/post-server-upgrade.yaml @@ -28,7 +28,19 @@ operations: tableConfigFileName: feature-test-1.json recordReaderConfigFileName: data/recordReaderConfig.json segmentName: FeatureTest1_Segment4 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/RealtimeFeatureTest1-data-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json - type: queryOp description: Run query on FeatureTest1 using SQL queryFileName: queries/feature-test-1-sql.queries expectedResultsFileName: query-results/feature-test-1-rest-sql.results + - type: queryOp + description: Run query on FeatureTest2 using SQL + queryFileName: queries/feature-test-2-sql-realtime.queries + expectedResultsFileName: query-results/feature-test-2-sql-realtime.results diff --git a/compatibility-verifier/sample-test-suite/pre-broker-upgrade.yaml b/compatibility-verifier/sample-test-suite/pre-broker-upgrade.yaml index 15a86f5e805..97fd509e40b 100644 --- a/compatibility-verifier/sample-test-suite/pre-broker-upgrade.yaml +++ b/compatibility-verifier/sample-test-suite/pre-broker-upgrade.yaml @@ -28,7 +28,19 @@ operations: tableConfigFileName: feature-test-1.json recordReaderConfigFileName: data/recordReaderConfig.json segmentName: FeatureTest1_Segment2 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/RealtimeFeatureTest1-data-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json - type: queryOp description: Run query on FeatureTest1 using SQL queryFileName: queries/feature-test-1-sql.queries expectedResultsFileName: query-results/feature-test-1-rest-sql.results + - type: queryOp + description: Run query on FeatureTest2 using SQL + queryFileName: queries/feature-test-2-sql-realtime.queries + expectedResultsFileName: query-results/feature-test-2-sql-realtime.results diff --git a/compatibility-verifier/sample-test-suite/pre-controller-upgrade.yaml b/compatibility-verifier/sample-test-suite/pre-controller-upgrade.yaml index e2d10e28d03..bb3b36b1114 100644 --- a/compatibility-verifier/sample-test-suite/pre-controller-upgrade.yaml +++ b/compatibility-verifier/sample-test-suite/pre-controller-upgrade.yaml @@ -20,6 +20,16 @@ # Operations to be done. description: Operations to be run before Controller upgrade operations: + - type: streamOp + description: create Kafka topic PinotRealtimeFeatureTest2Event + op: CREATE + streamConfigFileName: feature-test-2-realtime-stream-config.json + - type: tableOp + description: Create realtime table FeatureTest2 + op: CREATE + schemaFileName: FeatureTest2-schema.json + tableConfigFileName: feature-test-2-realtime.json + recordReaderConfigFileName: data/recordReaderConfig.json - type: tableOp description: Create offline table FeatureTest1 op: CREATE @@ -33,7 +43,19 @@ operations: tableConfigFileName: feature-test-1.json recordReaderConfigFileName: data/recordReaderConfig.json segmentName: FeatureTest1_Segment + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/RealtimeFeatureTest1-data-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json - type: queryOp description: Run query on FeatureTest1 using SQL queryFileName: queries/feature-test-1-sql.queries expectedResultsFileName: query-results/feature-test-1-rest-sql.results + - type: queryOp + description: Run query on FeatureTest2 using SQL + queryFileName: queries/feature-test-2-sql-realtime.queries + expectedResultsFileName: query-results/feature-test-2-sql-realtime.results diff --git a/compatibility-verifier/sample-test-suite/pre-server-upgrade.yaml b/compatibility-verifier/sample-test-suite/pre-server-upgrade.yaml index adee6c815cb..52c7f73da4c 100644 --- a/compatibility-verifier/sample-test-suite/pre-server-upgrade.yaml +++ b/compatibility-verifier/sample-test-suite/pre-server-upgrade.yaml @@ -28,7 +28,19 @@ operations: tableConfigFileName: feature-test-1.json recordReaderConfigFileName: data/recordReaderConfig.json segmentName: FeatureTest1_Segment3 + - type: streamOp + description: publish rows to PinotRealtimeFeatureTest2Event + op: PRODUCE + streamConfigFileName: feature-test-2-realtime-stream-config.json + numRows: 66 + inputDataFileName: data/RealtimeFeatureTest1-data-00.csv + recordReaderConfigFileName: data/recordReaderConfig.json + tableConfigFileName: feature-test-2-realtime.json - type: queryOp description: Run query on FeatureTest1 using SQL queryFileName: queries/feature-test-1-sql.queries expectedResultsFileName: query-results/feature-test-1-rest-sql.results + - type: queryOp + description: Run query on FeatureTest2 using SQL + queryFileName: queries/feature-test-2-sql-realtime.queries + expectedResultsFileName: query-results/feature-test-2-sql-realtime.results