You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2019/09/26 21:56:17 UTC
[incubator-pinot] branch pinot_data_type created (now 8ec1d60)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a change to branch pinot_data_type
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 8ec1d60 Simplify the logic of List handling in DataTypeTransformer
This branch includes the following new commits:
new 8ec1d60 Simplify the logic of List handling in DataTypeTransformer
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Simplify the logic of List handling in
DataTypeTransformer
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch pinot_data_type
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 8ec1d60eefffd4fd4b917aa5906cdaa2e6bd355b
Author: Jackie (Xiaotian) Jiang <xa...@linkedin.com>
AuthorDate: Thu Sep 26 14:54:00 2019 -0700
Simplify the logic of List handling in DataTypeTransformer
In DataTypeTransformer, convert List to Object[] for the data type conversion. No need to push down the List handling into PinotDataType.
---
.../recordtransformer/DataTypeTransformer.java | 28 ++---
.../core/data/recordtransformer/PinotDataType.java | 126 ++++++---------------
.../tools/admin/command/CreateSegmentCommand.java | 13 +--
3 files changed, 48 insertions(+), 119 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
index 226a84e..4ed581f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
@@ -60,6 +60,7 @@ public class DataTypeTransformer implements RecordTransformer {
private final Schema _schema;
private final Map<String, PinotDataType> _dataTypes = new HashMap<>();
+
public DataTypeTransformer(Schema schema) {
_schema = schema;
for (Map.Entry<String, FieldSpec> entry : schema.getFieldSpecMap().entrySet()) {
@@ -72,10 +73,8 @@ public class DataTypeTransformer implements RecordTransformer {
for (Map.Entry<String, PinotDataType> entry : _dataTypes.entrySet()) {
String column = entry.getKey();
PinotDataType dest = entry.getValue();
- Object value = record.getValue(column);
- // NOTE: should not need to set default null value in normal case (RecordReader is responsible for filling in the
- // default null value; TimeTransformer is responsible for filling in the outgoing time value if not exists)
+ Object value = record.getValue(column);
if (value == null || (value instanceof Object[] && ((Object[]) value).length == 0) || (value instanceof List
&& ((List) value).isEmpty())) {
// Set default null value
@@ -86,21 +85,15 @@ public class DataTypeTransformer implements RecordTransformer {
} else {
value = new Object[]{defaultNullValue};
}
- record.putField(column, value);
- continue;
} else {
- PinotDataType source;
+ // Convert List value to Object[]
if (value instanceof List) {
- // Multi-valued column
- List values = (List) value;
- source = MULTI_VALUE_TYPE_MAP.get(values.get(0).getClass());
- if (source == null) {
- source = PinotDataType.OBJECT_ARRAY;
- }
- // We need to convert value from list to object[] for further processing.
- record.putField(column, dest.convert(value, source));
- continue;
- } else if (value instanceof Object[]) {
+ value = ((List) value).toArray();
+ }
+
+ // Convert data type if necessary
+ PinotDataType source;
+ if (value instanceof Object[]) {
// Multi-valued column
Object[] values = (Object[]) value;
source = MULTI_VALUE_TYPE_MAP.get(values[0].getClass());
@@ -114,12 +107,11 @@ public class DataTypeTransformer implements RecordTransformer {
source = PinotDataType.OBJECT;
}
}
-
if (source != dest) {
value = dest.convert(value, source);
- record.putField(column, value);
}
}
+ record.putField(column, value);
}
return record;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java
index 392439a..4cc6c43 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/PinotDataType.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.core.data.recordtransformer;
-import java.util.List;
import org.apache.pinot.common.data.FieldSpec;
import org.apache.pinot.common.utils.BytesUtils;
@@ -505,25 +504,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new Integer[]{toInteger(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- Integer[] integerArray = new Integer[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- integerArray[i] = singleValueType.toInteger(valueList.get(i));
- }
- return integerArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- Integer[] integerArray = new Integer[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- integerArray[i] = singleValueType.toInteger(valueArray[i]);
- }
- return integerArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Integer[] integerArray = new Integer[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ integerArray[i] = singleValueType.toInteger(valueArray[i]);
}
+ return integerArray;
}
}
@@ -531,25 +519,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new Long[]{toLong(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- Long[] longArray = new Long[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- longArray[i] = singleValueType.toLong(valueList.get(i));
- }
- return longArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- Long[] longArray = new Long[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- longArray[i] = singleValueType.toLong(valueArray[i]);
- }
- return longArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Long[] longArray = new Long[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ longArray[i] = singleValueType.toLong(valueArray[i]);
}
+ return longArray;
}
}
@@ -557,25 +534,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new Float[]{toFloat(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- Float[] floatArray = new Float[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- floatArray[i] = singleValueType.toFloat(valueList.get(i));
- }
- return floatArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- Float[] floatArray = new Float[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- floatArray[i] = singleValueType.toFloat(valueArray[i]);
- }
- return floatArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Float[] floatArray = new Float[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ floatArray[i] = singleValueType.toFloat(valueArray[i]);
}
+ return floatArray;
}
}
@@ -583,25 +549,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new Double[]{toDouble(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- Double[] doubleArray = new Double[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- doubleArray[i] = singleValueType.toDouble(valueList.get(i));
- }
- return doubleArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- Double[] doubleArray = new Double[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- doubleArray[i] = singleValueType.toDouble(valueArray[i]);
- }
- return doubleArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ Double[] doubleArray = new Double[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ doubleArray[i] = singleValueType.toDouble(valueArray[i]);
}
+ return doubleArray;
}
}
@@ -609,25 +564,14 @@ public enum PinotDataType {
if (isSingleValue()) {
return new String[]{toString(value)};
} else {
- if (value instanceof List) {
- List valueList = (List) value;
- int length = valueList.size();
- String[] stringArray = new String[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- stringArray[i] = singleValueType.toString(valueList.get(i));
- }
- return stringArray;
- } else {
- Object[] valueArray = (Object[]) value;
- int length = valueArray.length;
- String[] stringArray = new String[length];
- PinotDataType singleValueType = getSingleValueType();
- for (int i = 0; i < length; i++) {
- stringArray[i] = singleValueType.toString(valueArray[i]);
- }
- return stringArray;
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ String[] stringArray = new String[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ stringArray[i] = singleValueType.toString(valueArray[i]);
}
+ return stringArray;
}
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
index 82c5cbd..1fe5b1c 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
@@ -38,7 +38,6 @@ import org.apache.pinot.common.data.StarTreeIndexSpec;
import org.apache.pinot.common.utils.JsonUtils;
import org.apache.pinot.core.data.readers.FileFormat;
import org.apache.pinot.core.data.readers.RecordReader;
-import org.apache.pinot.core.data.readers.RecordReaderFactory;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.orc.data.readers.ORCRecordReader;
@@ -437,28 +436,22 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
}
protected boolean isDataFile(String fileName) {
- if (_format == null) {
- return fileName.endsWith(".avro") || fileName.endsWith(".csv") || fileName.endsWith(".json") || fileName
- .endsWith(".thrift") || fileName.endsWith(".parquet") || fileName.endsWith(".orc");
- }
switch (_format) {
case AVRO:
case GZIPPED_AVRO:
return fileName.endsWith(".avro");
- case PARQUET:
- return fileName.endsWith(".parquet");
case CSV:
return fileName.endsWith(".csv");
case JSON:
return fileName.endsWith(".json");
case THRIFT:
return fileName.endsWith(".thrift");
+ case PARQUET:
+ return fileName.endsWith(".parquet");
case ORC:
return fileName.endsWith(".orc");
- case OTHER:
- case PINOT:
default:
- throw new RuntimeException("Not supported file format for segment creation");
+ throw new IllegalStateException("Unsupported file format for segment creation: " + _format);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org