You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2019/09/27 01:53:43 UTC
[incubator-pinot] branch master updated: Simplify the logic of List
handling in DataTypeTransformer (#4651)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9cfcd30 Simplify the logic of List handling in DataTypeTransformer (#4651)
9cfcd30 is described below
commit 9cfcd305cb50d10392b4d32d90846f7fe25b9936
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Thu Sep 26 18:53:33 2019 -0700
Simplify the logic of List handling in DataTypeTransformer (#4651)
In DataTypeTransformer, convert List to Object[] for the data type conversion. No need to push down the List handling into PinotDataType.
---
.../recordtransformer/DataTypeTransformer.java | 28 ++---
.../core/data/recordtransformer/PinotDataType.java | 126 ++++++---------------
.../tools/admin/command/CreateSegmentCommand.java | 13 +--
3 files changed, 48 insertions(+), 119 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
index 226a84e..4ed581f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
@@ -60,6 +60,7 @@ public class DataTypeTransformer implements RecordTransformer {
private final Schema _schema;
private final Map<String, PinotDataType> _dataTypes = new HashMap<>();
+
public DataTypeTransformer(Schema schema) {
_schema = schema;
for (Map.Entry<String, FieldSpec> entry : schema.getFieldSpecMap().entrySet()) {
@@ -72,10 +73,8 @@ public class DataTypeTransformer implements RecordTransformer {
for (Map.Entry<String, PinotDataType> entry : _dataTypes.entrySet()) {
String column = entry.getKey();
PinotDataType dest = entry.getValue();
- Object value = record.getValue(column);
- // NOTE: should not need to set default null value in normal case (RecordReader is responsible for filling in the
- // default null value; TimeTransformer is responsible for filling in the outgoing time value if not exists)
+ Object value = record.getValue(column);
if (value == null || (value instanceof Object[] && ((Object[]) value).length == 0) || (value instanceof List
&& ((List) value).isEmpty())) {
// Set default null value
@@ -86,21 +85,15 @@ public class DataTypeTransformer implements RecordTransformer {
} else {
value = new Object[]{defaultNullValue};
}
- record.putField(column, value);
- continue;
} else {
- PinotDataType source;
+ // Convert List value to Object[]
if (value instanceof List) {
- // Multi-valued column
- List values = (List) value;
- source = MULTI_VALUE_TYPE_MAP.get(values.get(0).getClass());
- if (source == null) {
- source = PinotDataType.OBJECT_ARRAY;
- }
- // We need to convert value from list to object[] for further processing.
- record.putField(column, dest.convert(value, source));
- continue;
- } else if (value instanceof Object[]) {
+ value = ((List) value).toArray();
+ }
+
+ // Convert data type if necessary
+ PinotDataType source;
+ if (value instanceof Object[]) {
// Multi-valued column
Object[] values = (Object[]) value;
source = MULTI_VALUE_TYPE_MAP.get(values[0].getClass());
@@ -114,12 +107,11 @@ public class DataTypeTransformer implements RecordTransformer {
source = PinotDataType.OBJECT;
}
}
-
if (source != dest) {
value = dest.convert(value, source);
- record.putField(column, value);
}
}
+ record.putField(column, value);
}
return record;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java
index 392439a..4cc6c43 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.core.data.recordtransformer;
-import java.util.List;
import org.apache.pinot.common.data.FieldSpec;
import org.apache.pinot.common.utils.BytesUtils;
@@ -505,25 +504,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new Integer[]{toInteger(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- Integer[] integerArray = new Integer[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- integerArray[i] = singleValueType.toInteger(valueList.get(i));
- }
- return integerArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- Integer[] integerArray = new Integer[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- integerArray[i] = singleValueType.toInteger(valueArray[i]);
- }
- return integerArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Integer[] integerArray = new Integer[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ integerArray[i] = singleValueType.toInteger(valueArray[i]);
}
+ return integerArray;
}
}
@@ -531,25 +519,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new Long[]{toLong(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- Long[] longArray = new Long[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- longArray[i] = singleValueType.toLong(valueList.get(i));
- }
- return longArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- Long[] longArray = new Long[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- longArray[i] = singleValueType.toLong(valueArray[i]);
- }
- return longArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Long[] longArray = new Long[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ longArray[i] = singleValueType.toLong(valueArray[i]);
}
+ return longArray;
}
}
@@ -557,25 +534,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new Float[]{toFloat(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- Float[] floatArray = new Float[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- floatArray[i] = singleValueType.toFloat(valueList.get(i));
- }
- return floatArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- Float[] floatArray = new Float[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- floatArray[i] = singleValueType.toFloat(valueArray[i]);
- }
- return floatArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Float[] floatArray = new Float[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ floatArray[i] = singleValueType.toFloat(valueArray[i]);
}
+ return floatArray;
}
}
@@ -583,25 +549,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new Double[]{toDouble(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- Double[] doubleArray = new Double[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- doubleArray[i] = singleValueType.toDouble(valueList.get(i));
- }
- return doubleArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- Double[] doubleArray = new Double[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- doubleArray[i] = singleValueType.toDouble(valueArray[i]);
- }
- return doubleArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Double[] doubleArray = new Double[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ doubleArray[i] = singleValueType.toDouble(valueArray[i]);
}
+ return doubleArray;
}
}
@@ -609,25 +564,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new String[]{toString(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- String[] stringArray = new String[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- stringArray[i] = singleValueType.toString(valueList.get(i));
- }
- return stringArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- String[] stringArray = new String[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- stringArray[i] = singleValueType.toString(valueArray[i]);
- }
- return stringArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ String[] stringArray = new String[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ stringArray[i] = singleValueType.toString(valueArray[i]);
}
+ return stringArray;
}
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
index 82c5cbd..1fe5b1c 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
@@ -38,7 +38,6 @@ import org.apache.pinot.common.data.StarTreeIndexSpec;
import org.apache.pinot.common.utils.JsonUtils;
import org.apache.pinot.core.data.readers.FileFormat;
import org.apache.pinot.core.data.readers.RecordReader;
-import org.apache.pinot.core.data.readers.RecordReaderFactory;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.orc.data.readers.ORCRecordReader;
@@ -437,28 +436,22 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
}
protected boolean isDataFile(String fileName) {
- if (_format == null) {
- return fileName.endsWith(".avro") || fileName.endsWith(".csv") || fileName.endsWith(".json") || fileName
- .endsWith(".thrift") || fileName.endsWith(".parquet") || fileName.endsWith(".orc");
- }
switch (_format) {
case AVRO:
case GZIPPED_AVRO:
return fileName.endsWith(".avro");
- case PARQUET:
- return fileName.endsWith(".parquet");
case CSV:
return fileName.endsWith(".csv");
case JSON:
return fileName.endsWith(".json");
case THRIFT:
return fileName.endsWith(".thrift");
+ case PARQUET:
+ return fileName.endsWith(".parquet");
case ORC:
return fileName.endsWith(".orc");
- case OTHER:
- case PINOT:
default:
- throw new RuntimeException("Not supported file format for segment creation");
+ throw new IllegalStateException("Unsupported file format for segment creation: " + _format);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org