You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2019/09/07 01:21:42 UTC
[incubator-pinot] 01/01: Implement all transform function APIs for
MapValueTransformFunction
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch map_transform
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 7068e7d1a11c46415f6dff04ff0d467179ac34ba
Author: Jackie (Xiaotian) Jiang <xa...@linkedin.com>
AuthorDate: Fri Sep 6 18:19:14 2019 -0700
Implement all transform function APIs for MapValueTransformFunction
- Change function name to "mapValue" to follow the convention
- Add documentation for MapValue transform function
- Add more checks on the arguments
- Enhance the test
---
.../function/MapValueTransformFunction.java | 142 +++++++++++---
.../tests/MapTypeClusterIntegrationTest.java | 208 ++++++++-------------
2 files changed, 193 insertions(+), 157 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java
index 8053d3a..b451a6e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java
@@ -18,9 +18,11 @@
*/
package org.apache.pinot.core.operator.transform.function;
+import com.google.common.base.Preconditions;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
+import org.apache.pinot.common.data.FieldSpec.DataType;
import org.apache.pinot.core.common.DataSource;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
@@ -29,20 +31,25 @@ import org.apache.pinot.core.segment.index.readers.Dictionary;
/**
- * map_value(keyColName, 'keyName', valColName)
+ * The MAP_VALUE transform function takes 3 arguments: keyColumn, keyValue, valueColumn, where keyColumn and valueColumn
+ * are dictionary-encoded multi-value columns, and keyValue must be a literal (number or string). In order to make
+ * MAP_VALUE transform function work, the keyValue provided must exist in the keyColumn. To ensure that, the query can
+ * have a filter on the keyColumn, e.g. SELECT mapValue(key, 'myKey', value) FROM myTable WHERE key = 'myKey'.
*/
public class MapValueTransformFunction extends BaseTransformFunction {
+ public static final String FUNCTION_NAME = "mapValue";
- public static final String FUNCTION_NAME = "map_value";
-
- private TransformResultMetadata _resultMetadata;
private TransformFunction _keyColumnFunction;
- private String _keyName;
- private TransformFunction _valueColumnFunction;
- private int[][] _keyDictIds;
- private int[][] _valueDictIds;
private int _inputKeyDictId;
- private int[] _outputValueDictIds;
+ private TransformFunction _valueColumnFunction;
+ private TransformResultMetadata _resultMetadata;
+
+ private int[] _dictIds;
+ private int[] _intValues;
+ private long[] _longValues;
+ private float[] _floatValues;
+ private double[] _doubleValues;
+ private String[] _stringValues;
@Override
public String getName() {
@@ -51,17 +58,27 @@ public class MapValueTransformFunction extends BaseTransformFunction {
@Override
public void init(@Nonnull List<TransformFunction> arguments, @Nonnull Map<String, DataSource> dataSourceMap) {
- int numArguments = arguments.size();
- if (numArguments != 3) {
- throw new IllegalArgumentException("3 arguments are required for MAP_VALUE transform function map_value(keyColName, 'keyName', valColName)");
- }
+ Preconditions.checkArgument(arguments.size() == 3,
+ "3 arguments are required for MAP_VALUE transform function: keyColumn, keyValue, valueColumn, e.g. mapValue(key, 'myKey', value)");
+
_keyColumnFunction = arguments.get(0);
- _keyName = ((LiteralTransformFunction) arguments.get(1)).getLiteral();
+ TransformResultMetadata keyColumnMetadata = _keyColumnFunction.getResultMetadata();
+ Preconditions.checkState(!keyColumnMetadata.isSingleValue() && keyColumnMetadata.hasDictionary(),
+ "Key column must be dictionary-encoded multi-value column");
+
+ TransformFunction keyValueFunction = arguments.get(1);
+ Preconditions.checkState(keyValueFunction instanceof LiteralTransformFunction,
+ "Key value must be a literal (number or string)");
+ String keyValue = ((LiteralTransformFunction) keyValueFunction).getLiteral();
+ _inputKeyDictId = _keyColumnFunction.getDictionary().indexOf(keyValue);
+ Preconditions.checkState(_inputKeyDictId >= 0,
+ "Key value must exist in the key column. Add filter on key column if necessary, e.g. SELECT mapValue(key, 'myKey', value) FROM myTable WHERE key = 'myKey'");
+
_valueColumnFunction = arguments.get(2);
TransformResultMetadata valueColumnMetadata = _valueColumnFunction.getResultMetadata();
- _resultMetadata =
- new TransformResultMetadata(valueColumnMetadata.getDataType(), true, valueColumnMetadata.hasDictionary());
- _inputKeyDictId = _keyColumnFunction.getDictionary().indexOf(_keyName);
+ Preconditions.checkState(!valueColumnMetadata.isSingleValue() && valueColumnMetadata.hasDictionary(),
+ "Value column must be dictionary-encoded multi-value column");
+ _resultMetadata = new TransformResultMetadata(valueColumnMetadata.getDataType(), true, true);
}
@Override
@@ -70,26 +87,97 @@ public class MapValueTransformFunction extends BaseTransformFunction {
}
@Override
- public int[] transformToDictIdsSV(@Nonnull ProjectionBlock projectionBlock) {
- _outputValueDictIds = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ public Dictionary getDictionary() {
+ return _valueColumnFunction.getDictionary();
+ }
- _keyDictIds = _keyColumnFunction.transformToDictIdsMV(projectionBlock);
- _valueDictIds = _valueColumnFunction.transformToDictIdsMV(projectionBlock);
+ @Override
+ public int[] transformToDictIdsSV(@Nonnull ProjectionBlock projectionBlock) {
+ if (_dictIds == null) {
+ _dictIds = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int[][] keyDictIds = _keyColumnFunction.transformToDictIdsMV(projectionBlock);
+ int[][] valueDictIds = _valueColumnFunction.transformToDictIdsMV(projectionBlock);
int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
- int numKeys = _keyDictIds[i].length;
+ int numKeys = keyDictIds[i].length;
for (int j = 0; j < numKeys; j++) {
- if (_keyDictIds[i][j] == _inputKeyDictId) {
- _outputValueDictIds[i] = _valueDictIds[i][j];
+ if (keyDictIds[i][j] == _inputKeyDictId) {
+ _dictIds[i] = valueDictIds[i][j];
break;
}
}
}
- return _outputValueDictIds;
+ return _dictIds;
}
@Override
- public Dictionary getDictionary() {
- return _valueColumnFunction.getDictionary();
+ public int[] transformToIntValuesSV(@Nonnull ProjectionBlock projectionBlock) {
+ if (_resultMetadata.getDataType() != DataType.INT) {
+ return super.transformToIntValuesSV(projectionBlock);
+ }
+ if (_intValues == null) {
+ _intValues = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ Dictionary dictionary = getDictionary();
+ int[] dictIds = transformToDictIdsSV(projectionBlock);
+ dictionary.readIntValues(dictIds, 0, projectionBlock.getNumDocs(), _intValues, 0);
+ return _intValues;
+ }
+
+ @Override
+ public long[] transformToLongValuesSV(@Nonnull ProjectionBlock projectionBlock) {
+ if (_resultMetadata.getDataType() != DataType.LONG) {
+ return super.transformToLongValuesSV(projectionBlock);
+ }
+ if (_longValues == null) {
+ _longValues = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ Dictionary dictionary = getDictionary();
+ int[] dictIds = transformToDictIdsSV(projectionBlock);
+ dictionary.readLongValues(dictIds, 0, projectionBlock.getNumDocs(), _longValues, 0);
+ return _longValues;
+ }
+
+ @Override
+ public float[] transformToFloatValuesSV(@Nonnull ProjectionBlock projectionBlock) {
+ if (_resultMetadata.getDataType() != DataType.FLOAT) {
+ return super.transformToFloatValuesSV(projectionBlock);
+ }
+ if (_floatValues == null) {
+ _floatValues = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ Dictionary dictionary = getDictionary();
+ int[] dictIds = transformToDictIdsSV(projectionBlock);
+ dictionary.readFloatValues(dictIds, 0, projectionBlock.getNumDocs(), _floatValues, 0);
+ return _floatValues;
+ }
+
+ @Override
+ public double[] transformToDoubleValuesSV(@Nonnull ProjectionBlock projectionBlock) {
+ if (_resultMetadata.getDataType() != DataType.DOUBLE) {
+ return super.transformToDoubleValuesSV(projectionBlock);
+ }
+ if (_doubleValues == null) {
+ _doubleValues = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ Dictionary dictionary = getDictionary();
+ int[] dictIds = transformToDictIdsSV(projectionBlock);
+ dictionary.readDoubleValues(dictIds, 0, projectionBlock.getNumDocs(), _doubleValues, 0);
+ return _doubleValues;
+ }
+
+ @Override
+ public String[] transformToStringValuesSV(@Nonnull ProjectionBlock projectionBlock) {
+ if (_resultMetadata.getDataType() != DataType.STRING) {
+ return super.transformToStringValuesSV(projectionBlock);
+ }
+ if (_stringValues == null) {
+ _stringValues = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ Dictionary dictionary = getDictionary();
+ int[] dictIds = transformToDictIdsSV(projectionBlock);
+ dictionary.readStringValues(dictIds, 0, projectionBlock.getNumDocs(), _stringValues, 0);
+ return _stringValues;
}
}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MapTypeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MapTypeClusterIntegrationTest.java
index d89b780..b8ccc04 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MapTypeClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MapTypeClusterIntegrationTest.java
@@ -18,200 +18,148 @@
*/
package org.apache.pinot.integration.tests;
-import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
-import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-
-import javax.annotation.Nonnull;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.common.data.DimensionFieldSpec;
-import org.apache.pinot.common.data.FieldSpec;
import org.apache.pinot.common.data.FieldSpec.DataType;
import org.apache.pinot.common.data.Schema;
-import org.apache.pinot.core.indexsegment.generator.SegmentVersion;
+import org.apache.pinot.core.util.AvroUtils;
import org.apache.pinot.util.TestUtils;
-import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.Lists;
+import static org.testng.Assert.assertEquals;
public class MapTypeClusterIntegrationTest extends BaseClusterIntegrationTest {
+ private static final int NUM_DOCS = 1000;
+ private static final String MAP_FIELD_NAME = "myMap";
- protected static final String DEFAULT_TABLE_NAME = "myTable";
- static final long TOTAL_DOCS = 1_000L;
-
- protected Schema _schema;
-
- private String _currentTable;
-
- @Nonnull
- @Override
- protected String getTableName() {
- return _currentTable;
- }
-
- @Nonnull
@Override
- protected String getSchemaFileName() {
- return "";
+ protected long getCountStarResult() {
+ return NUM_DOCS;
}
@BeforeClass
public void setUp()
throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_tempDir);
+ TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
// Start the Pinot cluster
startZk();
startController();
startBroker();
- startServers(1);
-
- _schema = new Schema();
- FieldSpec keyFieldSpec = new DimensionFieldSpec();
- keyFieldSpec.setDataType(DataType.STRING);
- keyFieldSpec.setDefaultNullValue("");
- keyFieldSpec.setName("myMap__KEYS");
- keyFieldSpec.setSingleValueField(false);
- _schema.addField(keyFieldSpec);
- FieldSpec valueFieldSpec = new DimensionFieldSpec();
- valueFieldSpec.setDataType(DataType.STRING);
- valueFieldSpec.setDefaultNullValue("");
- valueFieldSpec.setName("myMap__VALUES");
- valueFieldSpec.setSingleValueField(false);
- _schema.addField(valueFieldSpec);
+ startServer();
// Create the tables
- ArrayList<String> invertedIndexColumns = Lists.newArrayList();
- addOfflineTable(DEFAULT_TABLE_NAME, null, null, null, null, null, SegmentVersion.v1, invertedIndexColumns, null,
- null, null, null);
+ addOfflineTable(getTableName());
- setUpSegmentsAndQueryGenerator();
+ // Create and upload segments
+ File avroFile = createAvroFile();
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(getTableName())
+ .addMultiValueDimension(MAP_FIELD_NAME + AvroUtils.MAP_KEY_COLUMN_SUFFIX, DataType.STRING)
+ .addMultiValueDimension(MAP_FIELD_NAME + AvroUtils.MAP_VALUE_COLUMN_SUFFIX, DataType.INT).build();
+ ExecutorService executor = Executors.newCachedThreadPool();
+ ClusterIntegrationTestUtils
+ .buildSegmentsFromAvro(Collections.singletonList(avroFile), 0, _segmentDir, _tarDir, getTableName(), false,
+ null, null, schema, executor);
+ executor.shutdown();
+ executor.awaitTermination(10, TimeUnit.MINUTES);
+ uploadSegments(getTableName(), _tarDir);
// Wait for all documents loaded
- _currentTable = DEFAULT_TABLE_NAME;
waitForAllDocsLoaded(60_000);
}
- @Override
- protected long getCountStarResult() {
- return TOTAL_DOCS;
- }
-
- protected void setUpSegmentsAndQueryGenerator()
+ private File createAvroFile()
throws Exception {
-
- org.apache.avro.Schema mapAvroSchema = org.apache.avro.Schema.createMap(org.apache.avro.Schema.create(Type.STRING));
- List<Field> fields = new ArrayList<>();
- fields.add(new Field("myMap", mapAvroSchema, "", null));
- org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", "some desc", null, false);
+ org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
+ org.apache.avro.Schema mapAvroSchema = org.apache.avro.Schema.createMap(org.apache.avro.Schema.create(Type.INT));
+ List<Field> fields = Collections.singletonList(new Field(MAP_FIELD_NAME, mapAvroSchema, null, null));
avroSchema.setFields(fields);
- DataFileWriter<GenericData.Record> recordWriter =
- new DataFileWriter<>(new GenericDatumWriter<GenericData.Record>(avroSchema));
- String parent = "/tmp/mapTest";
- File avroFile = new File(parent, "part-" + 0 + ".avro");
- avroFile.getParentFile().mkdirs();
- recordWriter.create(avroSchema, avroFile);
- for (int i = 0; i < TOTAL_DOCS; i++) {
- Map<String, String> map = new HashMap<>();
- map.put("k1", "value-k1-" + i);
- map.put("k2", "value-k2-" + i);
- GenericData.Record record = new GenericData.Record(avroSchema);
- record.put("myMap", map);
- recordWriter.append(record);
+ File avroFile = new File(_tempDir, "data.avro");
+ try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
+ fileWriter.create(avroSchema, avroFile);
+ for (int i = 0; i < NUM_DOCS; i++) {
+ Map<String, Integer> map = new HashMap<>();
+ map.put("k1", i);
+ map.put("k2", NUM_DOCS + i);
+ GenericData.Record record = new GenericData.Record(avroSchema);
+ record.put(MAP_FIELD_NAME, map);
+ fileWriter.append(record);
+ }
}
- recordWriter.close();
-
- // Unpack the Avro files
- List<File> avroFiles = Lists.newArrayList(avroFile);
-
- // Create and upload segments without star tree indexes from Avro data
- createAndUploadSegments(avroFiles, DEFAULT_TABLE_NAME, false);
- }
-
- private void createAndUploadSegments(List<File> avroFiles, String tableName, boolean createStarTreeIndex)
- throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_segmentDir, _tarDir);
-
- ExecutorService executor = Executors.newCachedThreadPool();
- ClusterIntegrationTestUtils
- .buildSegmentsFromAvro(avroFiles, 0, _segmentDir, _tarDir, tableName, createStarTreeIndex, null, null, _schema,
- executor);
- executor.shutdown();
- executor.awaitTermination(10, TimeUnit.MINUTES);
- uploadSegments(getTableName(), _tarDir);
+ return avroFile;
}
@Test
public void testQueries()
throws Exception {
-
- //Selection Query
- String pqlQuery = "Select map_value(myMap__KEYS, 'k1', myMap__VALUES) from " + DEFAULT_TABLE_NAME;
- JsonNode pinotResponse = postQuery(pqlQuery);
- ArrayNode selectionResults = (ArrayNode) pinotResponse.get("selectionResults").get("results");
- Assert.assertNotNull(selectionResults);
- Assert.assertTrue(selectionResults.size() > 0);
- for (int i = 0; i < selectionResults.size(); i++) {
- String value = selectionResults.get(i).get(0).textValue();
- Assert.assertTrue(value.indexOf("-k1-") > 0);
+ // Selection only
+ String query = "SELECT mapValue(myMap__KEYS, 'k1', myMap__VALUES) FROM " + getTableName();
+ JsonNode pinotResponse = postQuery(query);
+ JsonNode selectionResults = pinotResponse.get("selectionResults").get("results");
+ assertEquals(selectionResults.size(), 10);
+ for (int i = 0; i < 10; i++) {
+ assertEquals(Integer.parseInt(selectionResults.get(i).get(0).textValue()), i);
}
- //Filter Query
- pqlQuery = "Select map_value(myMap__KEYS, 'k1', myMap__VALUES) from " + DEFAULT_TABLE_NAME
- + " where map_value(myMap__KEYS, 'k1', myMap__VALUES) = 'value-k1-0'";
- pinotResponse = postQuery(pqlQuery);
- selectionResults = (ArrayNode) pinotResponse.get("selectionResults").get("results");
- Assert.assertNotNull(selectionResults);
- Assert.assertTrue(selectionResults.size() > 0);
- for (int i = 0; i < selectionResults.size(); i++) {
- String value = selectionResults.get(i).get(0).textValue();
- Assert.assertEquals(value, "value-k1-0");
+ // Selection order-by
+ query = "SELECT mapValue(myMap__KEYS, 'k2', myMap__VALUES) FROM " + getTableName()
+ + " ORDER BY mapValue(myMap__KEYS, 'k1', myMap__VALUES)";
+ pinotResponse = postQuery(query);
+ selectionResults = pinotResponse.get("selectionResults").get("results");
+ assertEquals(selectionResults.size(), 10);
+ for (int i = 0; i < 10; i++) {
+ assertEquals(Integer.parseInt(selectionResults.get(i).get(0).textValue()), NUM_DOCS + i);
}
- //selection order by
- pqlQuery = "Select map_value(myMap__KEYS, 'k1', myMap__VALUES) from " + DEFAULT_TABLE_NAME
- + " order by map_value(myMap__KEYS, 'k1', myMap__VALUES)";
- pinotResponse = postQuery(pqlQuery);
- selectionResults = (ArrayNode) pinotResponse.get("selectionResults").get("results");
- Assert.assertNotNull(selectionResults);
- Assert.assertTrue(selectionResults.size() > 0);
- for (int i = 0; i < selectionResults.size(); i++) {
- String value = selectionResults.get(i).get(0).textValue();
- Assert.assertTrue(value.indexOf("-k1-") > 0);
+ // Aggregation only
+ query = "SELECT MAX(mapValue(myMap__KEYS, 'k1', myMap__VALUES)) FROM " + getTableName();
+ pinotResponse = postQuery(query);
+ JsonNode aggregationResult = pinotResponse.get("aggregationResults").get(0).get("value");
+ assertEquals((int) Double.parseDouble(aggregationResult.textValue()), NUM_DOCS - 1);
+
+ // Aggregation group-by
+ query = "SELECT MIN(mapValue(myMap__KEYS, 'k2', myMap__VALUES)) FROM " + getTableName()
+ + " GROUP BY mapValue(myMap__KEYS, 'k1', myMap__VALUES)";
+ pinotResponse = postQuery(query);
+ JsonNode groupByResults = pinotResponse.get("aggregationResults").get(0).get("groupByResult");
+ assertEquals(groupByResults.size(), 10);
+ for (int i = 0; i < 10; i++) {
+ JsonNode groupByResult = groupByResults.get(i);
+ assertEquals(Integer.parseInt(groupByResult.get("group").get(0).asText()), i);
+ assertEquals((int) Double.parseDouble(groupByResult.get("value").asText()), NUM_DOCS + i);
}
- //Group By Query
- pqlQuery = "Select count(*) from " + DEFAULT_TABLE_NAME + " group by map_value(myMap__KEYS, 'k1', myMap__VALUES)";
- pinotResponse = postQuery(pqlQuery);
- Assert.assertNotNull(pinotResponse.get("aggregationResults"));
- JsonNode groupByResult = pinotResponse.get("aggregationResults").get(0).get("groupByResult");
- Assert.assertNotNull(groupByResult);
- Assert.assertTrue(groupByResult.isArray());
- Assert.assertTrue(groupByResult.size() > 0);
+ // Filter
+ query = "SELECT mapValue(myMap__KEYS, 'k2', myMap__VALUES) FROM " + getTableName()
+ + " WHERE mapValue(myMap__KEYS, 'k1', myMap__VALUES) = 25";
+ pinotResponse = postQuery(query);
+ selectionResults = pinotResponse.get("selectionResults").get("results");
+ assertEquals(selectionResults.size(), 1);
+ assertEquals(Integer.parseInt(selectionResults.get(0).get(0).textValue()), NUM_DOCS + 25);
}
@AfterClass
public void tearDown()
throws Exception {
- dropOfflineTable(DEFAULT_TABLE_NAME);
+ dropOfflineTable(getTableName());
stopServer();
stopBroker();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org