You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by je...@apache.org on 2019/03/21 21:01:42 UTC
[incubator-pinot] branch orcMV created (now 5d9e806)
This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a change to branch orcMV
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 5d9e806 Add mv support
This branch includes the following new commits:
new 5d9e806 Add mv support
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Add mv support
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a commit to branch orcMV
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5d9e806959ba10ad7b588d2b56f4329370d681e1
Author: Jennifer Dai <jd...@linkedin.com>
AuthorDate: Thu Mar 21 14:01:27 2019 -0700
Add mv support
---
.../apache/pinot/orc/data/readers/ORCRecordReader.java | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
index 3c4c586..6247b33 100644
--- a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
+++ b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
@@ -20,6 +20,8 @@ package org.apache.pinot.orc.data.readers;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
@@ -63,6 +65,7 @@ public class ORCRecordReader implements RecordReader {
Reader _reader;
org.apache.orc.RecordReader _recordReader;
VectorizedRowBatch _reusableVectorizedRowBatch;
+ OrcMapredRecordReader _orcMapredRecordReader;
public static final String LOCAL_FS_PREFIX = "file://";
@@ -83,6 +86,8 @@ public class ORCRecordReader implements RecordReader {
LOGGER.warn("Pinot schema is not set in segment generator config");
}
_recordReader = _reader.rows(_reader.options().schema(_orcSchema));
+
+ _orcMapredRecordReader = new OrcMapredRecordReader<>(_recordReader, _orcSchema);
} catch (Exception e) {
LOGGER.error("Caught exception initializing record reader at path {}", inputPath);
throw new RuntimeException(e);
@@ -178,8 +183,14 @@ public class ORCRecordReader implements RecordReader {
} else if (Text.class.isAssignableFrom(w.getClass())) {
obj = ((Text) w).toString();
} else if (OrcList.class.isAssignableFrom(w.getClass())) {
- // TODO: This is probably multivalue columns
- LOGGER.info("Skipping unsupported type: list");
+ OrcList orcList = (OrcList) w;
+ LOGGER.info("ORC list is {}", orcList.toString());
+ List<Object> list = new ArrayList();
+ for (Object wc : orcList) {
+ Object objectToAdd = getBaseObject((WritableComparable) wc);
+ list.add(objectToAdd);
+ }
+ return list;
} else {
LOGGER.info("Unknown type found: " + w.getClass().getSimpleName());
throw new IllegalArgumentException("Unknown type: " + w.getClass().getSimpleName());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org