You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by je...@apache.org on 2019/03/21 21:01:43 UTC
[incubator-pinot] 01/01: Add mv support
This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a commit to branch orcMV
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5d9e806959ba10ad7b588d2b56f4329370d681e1
Author: Jennifer Dai <jd...@linkedin.com>
AuthorDate: Thu Mar 21 14:01:27 2019 -0700
Add mv support
---
.../apache/pinot/orc/data/readers/ORCRecordReader.java | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
index 3c4c586..6247b33 100644
--- a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
+++ b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
@@ -20,6 +20,8 @@ package org.apache.pinot.orc.data.readers;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
@@ -63,6 +65,7 @@ public class ORCRecordReader implements RecordReader {
Reader _reader;
org.apache.orc.RecordReader _recordReader;
VectorizedRowBatch _reusableVectorizedRowBatch;
+ OrcMapredRecordReader _orcMapredRecordReader;
public static final String LOCAL_FS_PREFIX = "file://";
@@ -83,6 +86,8 @@ public class ORCRecordReader implements RecordReader {
LOGGER.warn("Pinot schema is not set in segment generator config");
}
_recordReader = _reader.rows(_reader.options().schema(_orcSchema));
+
+ _orcMapredRecordReader = new OrcMapredRecordReader<>(_recordReader, _orcSchema);
} catch (Exception e) {
LOGGER.error("Caught exception initializing record reader at path {}", inputPath);
throw new RuntimeException(e);
@@ -178,8 +183,14 @@ public class ORCRecordReader implements RecordReader {
} else if (Text.class.isAssignableFrom(w.getClass())) {
obj = ((Text) w).toString();
} else if (OrcList.class.isAssignableFrom(w.getClass())) {
- // TODO: This is probably multivalue columns
- LOGGER.info("Skipping unsupported type: list");
+ OrcList orcList = (OrcList) w;
+ LOGGER.info("ORC list is {}", orcList.toString());
+ List<Object> list = new ArrayList();
+ for (Object wc : orcList) {
+ Object objectToAdd = getBaseObject((WritableComparable) wc);
+ list.add(objectToAdd);
+ }
+ return list;
} else {
LOGGER.info("Unknown type found: " + w.getClass().getSimpleName());
throw new IllegalArgumentException("Unknown type: " + w.getClass().getSimpleName());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org