You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2020/07/25 05:51:09 UTC
[incubator-pinot] branch override-methods-in-object-array created
(now 77c3158)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch override-methods-in-object-array
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 77c3158 Transform and extract map values to array
This branch includes the following new commits:
new 77c3158 Transform and extract map values to array
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Transform and extract map values to array
Posted by jl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch override-methods-in-object-array
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 77c3158e5d511737105c8bb124201a95ae7db7df
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Fri Jul 24 22:50:45 2020 -0700
Transform and extract map values to array
---
.../core/data/recordtransformer/PinotDataType.java | 127 ++++++++++++++++++++-
.../data/recordtransformer/PinotDataTypeTest.java | 16 +++
2 files changed, 142 insertions(+), 1 deletion(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java
index d1296b7..5227366 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java
@@ -18,6 +18,8 @@
*/
package org.apache.pinot.core.data.recordtransformer;
+import java.util.Iterator;
+import java.util.Map;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.utils.BytesUtils;
@@ -470,7 +472,112 @@ public enum PinotDataType {
}
},
- OBJECT_ARRAY;
+ OBJECT_ARRAY {
+ @Override
+ public Integer[] toIntegerArray(Object value) {
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Integer[] integerArray = new Integer[length];
+ if (valueArray[0] instanceof Map) {
+ PinotDataType singleValueType = getPrimitiveDataTypeFromMap((Map) valueArray[0]);
+ for (int i = 0; i < length; i++) {
+ for (Object obj : ((Map<Object, Object>) valueArray[i]).values()) {
+ integerArray[i] = singleValueType.toInteger(obj);
+ }
+ }
+ } else {
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ integerArray[i] = singleValueType.toInteger(valueArray[i]);
+ }
+ }
+ return integerArray;
+ }
+
+ @Override
+ public Long[] toLongArray(Object value) {
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Long[] longArray = new Long[length];
+ if (valueArray[0] instanceof Map) {
+ PinotDataType singleValueType = getPrimitiveDataTypeFromMap((Map) valueArray[0]);
+ for (int i = 0; i < length; i++) {
+ for (Object obj : ((Map<Object, Object>) valueArray[i]).values()) {
+ longArray[i] = singleValueType.toLong(obj);
+ }
+ }
+ } else {
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ longArray[i] = singleValueType.toLong(valueArray[i]);
+ }
+ }
+ return longArray;
+ }
+
+ @Override
+ public Float[] toFloatArray(Object value) {
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Float[] floatArray = new Float[length];
+ if (valueArray[0] instanceof Map) {
+ PinotDataType singleValueType = getPrimitiveDataTypeFromMap((Map) valueArray[0]);
+ for (int i = 0; i < length; i++) {
+ for (Object obj : ((Map<Object, Object>) valueArray[i]).values()) {
+ floatArray[i] = singleValueType.toFloat(obj);
+ }
+ }
+ } else {
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ floatArray[i] = singleValueType.toFloat(valueArray[i]);
+ }
+ }
+ return floatArray;
+ }
+
+ @Override
+ public Double[] toDoubleArray(Object value) {
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Double[] doubleArray = new Double[length];
+ if (valueArray[0] instanceof Map) {
+ PinotDataType singleValueType = getPrimitiveDataTypeFromMap((Map) valueArray[0]);
+ for (int i = 0; i < length; i++) {
+ for (Object obj : ((Map<Object, Object>) valueArray[i]).values()) {
+ doubleArray[i] = singleValueType.toDouble(obj);
+ }
+ }
+ } else {
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ doubleArray[i] = singleValueType.toDouble(valueArray[i]);
+ }
+ }
+ return doubleArray;
+ }
+
+ @Override
+ public String[] toStringArray(Object value) {
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ String[] stringArray = new String[length];
+ if (valueArray[0] instanceof Map) {
+ PinotDataType singleValueType = getPrimitiveDataTypeFromMap((Map) valueArray[0]);
+ for (int i = 0; i < length; i++) {
+ for (Object obj : ((Map<Object, Object>) valueArray[i]).values()) {
+ stringArray[i] = singleValueType.toString(obj);
+ }
+ }
+ } else {
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ stringArray[i] = singleValueType.toString(valueArray[i]);
+ }
+ }
+ return stringArray;
+ }
+ };
/**
* NOTE: override toInteger(), toLong(), toFloat(), toDouble(), toString() and toBytes() for single-value types.
@@ -632,4 +739,22 @@ public enum PinotDataType {
"Unsupported data type: " + dataType + " in field: " + fieldSpec.getName());
}
}
+
+ public static PinotDataType getPrimitiveDataTypeFromMap(Map<Object, Object> map) {
+ Iterator<Object> iterator = map.values().iterator();
+ Object obj = iterator.next();
+ if (obj instanceof Integer) {
+ return PinotDataType.INTEGER;
+ } else if (obj instanceof Long) {
+ return PinotDataType.LONG;
+ } else if (obj instanceof Float) {
+ return PinotDataType.FLOAT;
+ } else if (obj instanceof Double) {
+ return PinotDataType.DOUBLE;
+ } else if (obj instanceof String) {
+ return PinotDataType.STRING;
+ } else {
+ throw new IllegalStateException(String.format("'%s' isn't supported in the map.", obj.getClass()));
+ }
+ }
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/PinotDataTypeTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/PinotDataTypeTest.java
index 674bfbd..b1a9b38 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/PinotDataTypeTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/PinotDataTypeTest.java
@@ -18,6 +18,8 @@
*/
package org.apache.pinot.core.data.recordtransformer;
+import java.util.HashMap;
+import java.util.Map;
import org.testng.annotations.Test;
import static org.apache.pinot.core.data.recordtransformer.PinotDataType.*;
@@ -104,6 +106,20 @@ public class PinotDataTypeTest {
}
@Test
+ public void testMap() {
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("item", "10");
+ Map<String, String> map2 = new HashMap<>();
+ map2.put("item", "20");
+ Object[] objectArray = new Object[] {map1, map2};
+ assertEquals(STRING_ARRAY.convert(objectArray, OBJECT_ARRAY), new String[] {"10", "20"});
+ assertEquals(INTEGER_ARRAY.convert(objectArray, OBJECT_ARRAY), new Integer[]{10, 20});
+ assertEquals(LONG_ARRAY.convert(objectArray, OBJECT_ARRAY), new Long[]{10L, 20L});
+ assertEquals(FLOAT_ARRAY.convert(objectArray, OBJECT_ARRAY), new Float[]{10.0f, 20f});
+ assertEquals(DOUBLE_ARRAY.convert(objectArray, OBJECT_ARRAY), new Double[]{10.0d, 20d});
+ }
+
+ @Test
public void testInvalidConversion() {
for (PinotDataType sourceType : values()) {
if (sourceType.isSingleValue() && sourceType != STRING && sourceType != BYTES) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org