You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2023/07/19 05:40:02 UTC
[pinot] branch master updated: Adding v2 map function supports (#11104)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new df72d1dfb1 Adding v2 map function supports (#11104)
df72d1dfb1 is described below
commit df72d1dfb1663739017212b3ea1ddc73e8455190
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Tue Jul 18 22:39:56 2023 -0700
Adding v2 map function supports (#11104)
---
.../common/function/TransformFunctionType.java | 5 +-
.../tests/MapTypeClusterIntegrationTest.java | 124 ++++++++++++---------
...tiStageEngineMapTypeClusterIntegrationTest.java | 37 ++++++
3 files changed, 110 insertions(+), 56 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
index 6c1c9a388e..2ae2f80603 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
@@ -143,7 +143,10 @@ public enum TransformFunctionType {
ARRAYMAX("arrayMax"),
ARRAYSUM("arraySum"),
VALUEIN("valueIn"),
- MAPVALUE("mapValue"),
+ MAPVALUE("mapValue", ReturnTypes.cascade(opBinding ->
+ opBinding.getOperandType(2).getComponentType(), SqlTypeTransforms.FORCE_NULLABLE),
+ OperandTypes.family(ImmutableList.of(SqlTypeFamily.ANY, SqlTypeFamily.ANY, SqlTypeFamily.ANY)),
+ "map_value"),
// special functions
INIDSET("inIdSet"),
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MapTypeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MapTypeClusterIntegrationTest.java
index 68ec52dc5d..3a4565abad 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MapTypeClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MapTypeClusterIntegrationTest.java
@@ -79,6 +79,7 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
.addMultiValueDimension(INT_KEY_MAP_FIELD_NAME + SchemaUtils.MAP_VALUE_COLUMN_SUFFIX, DataType.INT)
.addSingleValueDimension(STRING_KEY_MAP_STR_FIELD_NAME, DataType.STRING)
.addSingleValueDimension(INT_KEY_MAP_STR_FIELD_NAME, DataType.STRING).build();
+ addSchema(schema);
List<TransformConfig> transformConfigs = Arrays.asList(
new TransformConfig(STRING_KEY_MAP_STR_FIELD_NAME, "toJsonMapStr(" + STRING_KEY_MAP_FIELD_NAME + ")"),
new TransformConfig(INT_KEY_MAP_STR_FIELD_NAME, "toJsonMapStr(" + INT_KEY_MAP_FIELD_NAME + ")"));
@@ -129,6 +130,10 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
return avroFile;
}
+ protected int getSelectionDefaultDocCount() {
+ return 10;
+ }
+
@Test
public void testJsonPathQueries()
throws Exception {
@@ -137,24 +142,24 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
JsonNode pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
JsonNode rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
- assertEquals(rows.get(i).get(0).textValue(), String.format("{\"k1\":%d,\"k2\":100%d}", i, i));
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
+ assertEquals(rows.get(i).get(0).textValue(), String.format("{\"k1\":%d,\"k2\":%d}", i, NUM_DOCS + i));
}
query = "SELECT jsonExtractScalar(stringKeyMapStr, '$.k1', 'INT') FROM " + getTableName();
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), i);
}
query = "SELECT jsonExtractScalar(intKeyMapStr, '$.95', 'INT') FROM " + getTableName();
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), i);
}
@@ -164,8 +169,8 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), NUM_DOCS + i);
}
query = "SELECT jsonExtractScalar(intKeyMapStr, '$.717', 'INT') FROM " + getTableName()
@@ -173,8 +178,8 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), NUM_DOCS + i);
}
@@ -197,8 +202,8 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), i);
assertEquals(rows.get(i).get(1).intValue(), NUM_DOCS + i);
}
@@ -208,8 +213,8 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), i);
assertEquals(rows.get(i).get(1).intValue(), NUM_DOCS + i);
}
@@ -230,20 +235,6 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
assertEquals(rows.size(), 1);
assertEquals(rows.get(0).get(0).intValue(), NUM_DOCS + 25);
- // Filter on non-existing key
- query = "SELECT jsonExtractScalar(stringKeyMapStr, '$.k2', 'INT') FROM " + getTableName()
- + " WHERE jsonExtractScalar(stringKeyMapStr, '$.k3', 'INT_ARRAY') = 25";
- pinotResponse = postQuery(query);
- assertEquals(pinotResponse.get("exceptions").size(), 0);
- rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 0);
- query = "SELECT jsonExtractScalar(intKeyMapStr, '$.717', 'INT') FROM " + getTableName()
- + " WHERE jsonExtractScalar(intKeyMapStr, '$.123', 'INT_ARRAY') = 25";
- pinotResponse = postQuery(query);
- assertEquals(pinotResponse.get("exceptions").size(), 0);
- rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 0);
-
// Select non-existing key (illegal query)
query = "SELECT jsonExtractScalar(stringKeyMapStr, '$.k3', 'INT') FROM " + getTableName();
pinotResponse = postQuery(query);
@@ -259,20 +250,6 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
query = "SELECT jsonExtractScalar(stringKeyMapStr, '$.123', 'INT', '0') FROM " + getTableName();
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
-
- // Select non-existing key with proper filter
- query = "SELECT jsonExtractScalar(intKeyMapStr, '$.123', 'INT') FROM " + getTableName()
- + " WHERE jsonExtractKey(intKeyMapStr, '$.*') = \"$['123']\"";
- pinotResponse = postQuery(query);
- assertEquals(pinotResponse.get("exceptions").size(), 0);
- rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 0);
- query = "SELECT jsonExtractScalar(stringKeyMapStr, '$.k3', 'INT') FROM " + getTableName()
- + " WHERE jsonExtractKey(stringKeyMapStr, '$.*') = \"$['k3']\"";
- pinotResponse = postQuery(query);
- assertEquals(pinotResponse.get("exceptions").size(), 0);
- rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 0);
}
@Test
@@ -283,16 +260,16 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
JsonNode pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
JsonNode rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), i);
}
query = "SELECT mapValue(intKeyMap__KEYS, 95, intKeyMap__VALUES) FROM " + getTableName();
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), i);
}
@@ -302,8 +279,8 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), NUM_DOCS + i);
}
query = "SELECT mapValue(intKeyMap__KEYS, 717, intKeyMap__VALUES) FROM " + getTableName()
@@ -311,8 +288,8 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), NUM_DOCS + i);
}
@@ -335,8 +312,8 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), i);
assertEquals(rows.get(i).get(1).intValue(), NUM_DOCS + i);
}
@@ -346,8 +323,8 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
pinotResponse = postQuery(query);
assertEquals(pinotResponse.get("exceptions").size(), 0);
rows = pinotResponse.get("resultTable").get("rows");
- assertEquals(rows.size(), 10);
- for (int i = 0; i < 10; i++) {
+ assertEquals(rows.size(), getSelectionDefaultDocCount());
+ for (int i = 0; i < getSelectionDefaultDocCount(); i++) {
assertEquals(rows.get(i).get(0).intValue(), i);
assertEquals(rows.get(i).get(1).intValue(), NUM_DOCS + i);
}
@@ -389,6 +366,43 @@ public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
query = "SELECT mapValue(stringKeyMap__KEYS, 123, stringKeyMap__VALUES) FROM " + getTableName();
pinotResponse = postQuery(query);
assertNotEquals(pinotResponse.get("exceptions").size(), 0);
+ }
+
+ @Test
+ public void testMultiValueQueries()
+ throws Exception {
+ String query;
+ JsonNode pinotResponse;
+ JsonNode rows;
+
+ // Filter on non-existing key
+ query
+ = "SELECT jsonExtractScalar(stringKeyMapStr, '$.k2', 'INT') FROM " + getTableName()
+ + " WHERE jsonExtractScalar(stringKeyMapStr, '$.k3', 'INT_ARRAY') = 25";
+ pinotResponse = postQuery(query);
+ assertEquals(pinotResponse.get("exceptions").size(), 0);
+ rows = pinotResponse.get("resultTable").get("rows");
+ assertEquals(rows.size(), 0);
+ query = "SELECT jsonExtractScalar(intKeyMapStr, '$.717', 'INT') FROM " + getTableName()
+ + " WHERE jsonExtractScalar(intKeyMapStr, '$.123', 'INT_ARRAY') = 25";
+ pinotResponse = postQuery(query);
+ assertEquals(pinotResponse.get("exceptions").size(), 0);
+ rows = pinotResponse.get("resultTable").get("rows");
+ assertEquals(rows.size(), 0);
+
+ // Select non-existing key with proper filter
+ query = "SELECT jsonExtractScalar(intKeyMapStr, '$.123', 'INT') FROM " + getTableName()
+ + " WHERE jsonExtractKey(intKeyMapStr, '$.*') = \"$['123']\"";
+ pinotResponse = postQuery(query);
+ assertEquals(pinotResponse.get("exceptions").size(), 0);
+ rows = pinotResponse.get("resultTable").get("rows");
+ assertEquals(rows.size(), 0);
+ query = "SELECT jsonExtractScalar(stringKeyMapStr, '$.k3', 'INT') FROM " + getTableName()
+ + " WHERE jsonExtractKey(stringKeyMapStr, '$.*') = \"$['k3']\"";
+ pinotResponse = postQuery(query);
+ assertEquals(pinotResponse.get("exceptions").size(), 0);
+ rows = pinotResponse.get("resultTable").get("rows");
+ assertEquals(rows.size(), 0);
// Select non-existing key with proper filter
query = "SELECT mapValue(stringKeyMap__KEYS, 'k3', stringKeyMap__VALUES) FROM " + getTableName()
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineMapTypeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineMapTypeClusterIntegrationTest.java
new file mode 100644
index 0000000000..62cfe12e00
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineMapTypeClusterIntegrationTest.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+public class MultiStageEngineMapTypeClusterIntegrationTest extends MapTypeClusterIntegrationTest {
+
+ @Override
+ protected boolean useMultiStageQueryEngine() {
+ return true;
+ }
+
+ @Override
+ protected int getSelectionDefaultDocCount() {
+ return 1000;
+ }
+
+ @Override
+ public void testMultiValueQueries()
+ throws Exception {
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org