You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ki...@apache.org on 2019/11/27 06:49:36 UTC
[incubator-pinot] branch refactor-record-reader created (now
5942774)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch refactor-record-reader
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 5942774 Removing segment generation config from RecordReader interface
This branch includes the following new commits:
new 5942774 Removing segment generation config from RecordReader interface
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Removing segment generation config from
RecordReader interface
Posted by ki...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch refactor-record-reader
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5942774220df244dc25a93d5cc97dd78b2e618a2
Author: kishoreg <g....@gmail.com>
AuthorDate: Tue Nov 26 22:49:09 2019 -0800
Removing segment generation config from RecordReader interface
---
.../apache/pinot/core/data/readers/AvroRecordReader.java | 3 ++-
.../apache/pinot/core/data/readers/CSVRecordReader.java | 3 ++-
.../pinot/core/data/readers/GenericRowRecordReader.java | 3 ++-
.../apache/pinot/core/data/readers/JSONRecordReader.java | 4 +++-
.../data/readers/MultiplePinotSegmentRecordReader.java | 11 ++++++-----
.../pinot/core/data/readers/PinotSegmentRecordReader.java | 11 ++++++-----
.../org/apache/pinot/core/data/readers/RecordReader.java | 14 ++++++++++----
.../pinot/core/data/readers/RecordReaderFactory.java | 2 +-
.../apache/pinot/core/data/readers/ThriftRecordReader.java | 11 ++++++-----
.../indexsegment/generator/SegmentGeneratorConfig.java | 2 +-
.../apache/pinot/core/minion/BackfillDateTimeColumn.java | 4 +++-
.../java/org/apache/pinot/core/minion/SegmentPurger.java | 5 ++++-
.../pinot/core/minion/segment/MapperRecordReader.java | 4 +++-
.../pinot/core/minion/segment/ReducerRecordReader.java | 4 +++-
.../realtime/converter/RealtimeSegmentRecordReader.java | 4 +++-
.../org/apache/pinot/orc/data/readers/ORCRecordReader.java | 5 +++--
.../apache/pinot/orc/data/readers/ORCRecordReaderTest.java | 11 ++---------
.../pinot/parquet/data/readers/ParquetRecordReader.java | 7 ++++---
.../parquet/data/readers/ParquetRecordReaderTest.java | 6 +-----
.../pinot/tools/admin/command/CreateSegmentCommand.java | 5 +++--
20 files changed, 68 insertions(+), 51 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java
index bfa225f..9d0fa4e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java
@@ -56,7 +56,8 @@ public class AvroRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java
index 00cce5d..95ac10a 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java
@@ -94,7 +94,8 @@ public class CSVRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java
index 62466e2..e1f4ca2 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java
@@ -41,7 +41,8 @@ public class GenericRowRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java
index cc4bcd7..459a167 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java
@@ -62,7 +62,9 @@ public class JSONRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java
index d924152..dda5fd9 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java
@@ -53,11 +53,6 @@ public class MultiplePinotSegmentRecordReader implements RecordReader {
this(indexDirs, null, null);
}
- @Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
-
- }
-
/**
* Read records using the passed in schema and in the order of sorted column from multiple pinot segments.
* <p>Passed in schema must be a subset of the segment schema.
@@ -106,6 +101,12 @@ public class MultiplePinotSegmentRecordReader implements RecordReader {
}
}
+ @Override
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
+ }
+
/**
* Indicate whether the segment should be sorted or not
*/
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java
index 50c1808..fab4af7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java
@@ -60,11 +60,6 @@ public class PinotSegmentRecordReader implements RecordReader {
this(indexDir, null, null);
}
- @Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
-
- }
-
/**
* Read records using the segment schema with the given schema and sort order
* <p>Passed in schema must be a subset of the segment schema.
@@ -110,6 +105,12 @@ public class PinotSegmentRecordReader implements RecordReader {
}
}
+ @Override
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
+ }
+
/**
* Prepare sorted docIds in order of the given sort order columns
*/
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java
index 836d3d8..7ea744e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java
@@ -20,9 +20,9 @@ package org.apache.pinot.core.data.readers;
import java.io.Closeable;
import java.io.IOException;
+import javax.annotation.Nullable;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
-import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
/**
@@ -34,10 +34,16 @@ import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
public interface RecordReader extends Closeable {
/**
- * Initializes the record reader when needed
+ * initializing recordreader with inputpath, schema and recordreader config. <br/>
+ * The implementation can chose to ignore one or more of these parameters and handle null gracefully <br/>
+ *
+ * @param inputPath absolute path to the file/directory
+ * @param schema Pinot Schema associated with the table
+ * @param recordReaderConfig config for the reader specific to the format. e.g. delimiter for csv format etc
+ * @throws Exception if the arguments are invalid
*/
- void init(SegmentGeneratorConfig segmentGeneratorConfig)
- throws IOException;
+ void init(@Nullable String inputPath, @Nullable Schema schema, @Nullable RecordReaderConfig recordReaderConfig)
+ throws Exception;
/**
* Return <code>true</code> if more records remain to be read.
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java
index 1d63b15..03edcfa 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java
@@ -52,7 +52,7 @@ public class RecordReaderFactory {
.warn("Using class: {} to read segment, ignoring configured file format: {}", recordReaderPath, fileFormat);
}
RecordReader recordReader = (RecordReader) Class.forName(recordReaderPath).newInstance();
- recordReader.init(segmentGeneratorConfig);
+ recordReader.init(dataFile.getAbsolutePath(), schema, segmentGeneratorConfig.getReaderConfig());
return recordReader;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java
index 1f24010..7da8181 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java
@@ -66,6 +66,12 @@ public class ThriftRecordReader implements RecordReader {
init();
}
+ @Override
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
+ }
+
private void init()
throws IOException {
_inputStream = RecordReaderUtils.getBufferedInputStream(_dataFile);
@@ -78,11 +84,6 @@ public class ThriftRecordReader implements RecordReader {
}
}
- @Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
-
- }
-
private boolean hasMoreToRead()
throws IOException {
_inputStream.mark(1);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
index fba03b2..ec51f79 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
@@ -84,7 +84,7 @@ public class SegmentGeneratorConfig {
private String _dataDir = null;
private String _inputFilePath = null;
private FileFormat _format = FileFormat.AVRO;
- private String _recordReaderPath = null;
+ private String _recordReaderPath = null; //TODO: this should be renamed to recordReaderClass or even better removed
private String _outDir = null;
private boolean _overwrite = false;
private String _tableName = null;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java
index 174826f..e485108 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java
@@ -33,6 +33,7 @@ import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.FileFormat;
import org.apache.pinot.core.data.readers.PinotSegmentRecordReader;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.data.recordtransformer.CompositeTransformer;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.segment.creator.RecordReaderSegmentCreationDataSource;
@@ -137,7 +138,8 @@ public class BackfillDateTimeColumn {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java
index 35ae334..82ea567 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java
@@ -32,6 +32,7 @@ import org.apache.pinot.common.segment.StarTreeMetadata;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.PinotSegmentRecordReader;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.core.segment.index.SegmentMetadataImpl;
@@ -170,7 +171,9 @@ public class SegmentPurger {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java
index b27dff8..07ffd51 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java
@@ -26,6 +26,7 @@ import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.MultiplePinotSegmentRecordReader;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
@@ -54,7 +55,8 @@ public class MapperRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java
index ddf7b81..9a2d133 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java
@@ -26,6 +26,7 @@ import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.PinotSegmentRecordReader;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
@@ -51,7 +52,8 @@ public class ReducerRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java
index 235ed00..d8e7c84 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java
@@ -21,6 +21,7 @@ package org.apache.pinot.core.realtime.converter;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.indexsegment.mutable.MutableSegmentImpl;
@@ -51,7 +52,8 @@ public class RealtimeSegmentRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
index 7bfcf60..e9f982c 100644
--- a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
+++ b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
@@ -45,6 +45,7 @@ import org.apache.orc.mapred.OrcMapredRecordReader;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -83,9 +84,9 @@ public class ORCRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig)
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
throws IOException {
- init(segmentGeneratorConfig.getInputFilePath(), segmentGeneratorConfig.getSchema());
+ init(inputPath, schema);
}
@Override
diff --git a/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java b/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java
index b3282ed..bd80ad1 100644
--- a/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java
+++ b/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java
@@ -114,15 +114,12 @@ public class ORCRecordReaderTest {
throws IOException {
ORCRecordReader orcRecordReader = new ORCRecordReader();
- SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig();
- segmentGeneratorConfig.setInputFilePath(ORC_FILE.getAbsolutePath());
Schema schema = new Schema();
FieldSpec xFieldSpec = new DimensionFieldSpec("x", FieldSpec.DataType.LONG, true);
schema.addField(xFieldSpec);
FieldSpec yFieldSpec = new DimensionFieldSpec("y", FieldSpec.DataType.BYTES, true);
schema.addField(yFieldSpec);
- segmentGeneratorConfig.setSchema(schema);
- orcRecordReader.init(segmentGeneratorConfig);
+ orcRecordReader.init(ORC_FILE.getAbsolutePath(), schema, null);
List<GenericRow> genericRows = new ArrayList<>();
while (orcRecordReader.hasNext()) {
@@ -140,16 +137,12 @@ public class ORCRecordReaderTest {
@Test
public void testReadMVData() throws IOException{
ORCRecordReader orcRecordReader = new ORCRecordReader();
-
- SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig();
- segmentGeneratorConfig.setInputFilePath(MULTIVALUE_ORC_FILE.getAbsolutePath());
Schema schema = new Schema();
FieldSpec emailsFieldSpec = new DimensionFieldSpec("emails", FieldSpec.DataType.STRING, false);
schema.addField(emailsFieldSpec);
FieldSpec xFieldSpec = new DimensionFieldSpec("x", FieldSpec.DataType.INT, true);
schema.addField(xFieldSpec);
- segmentGeneratorConfig.setSchema(schema);
- orcRecordReader.init(segmentGeneratorConfig);
+ orcRecordReader.init(MULTIVALUE_ORC_FILE.getAbsolutePath(), schema, null);
List<GenericRow> genericRows = new ArrayList<>();
while (orcRecordReader.hasNext()) {
diff --git a/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java b/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java
index 8e597e2..e80a3f4 100644
--- a/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java
+++ b/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java
@@ -27,6 +27,7 @@ import org.apache.pinot.common.data.FieldSpec;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.data.readers.RecordReaderUtils;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.util.AvroUtils;
@@ -43,10 +44,10 @@ public class ParquetRecordReader implements RecordReader {
private GenericRecord _nextRecord;
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig)
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
throws IOException {
- _dataFilePath = new Path(segmentGeneratorConfig.getInputFilePath());
- _schema = segmentGeneratorConfig.getSchema();
+ _dataFilePath = new Path(inputPath);
+ _schema = schema;
AvroUtils.validateSchema(_schema, ParquetUtils.getParquetSchema(_dataFilePath));
_fieldSpecs = RecordReaderUtils.extractFieldSpecs(_schema);
diff --git a/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java b/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java
index 74822eb..cab095c 100644
--- a/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java
+++ b/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java
@@ -81,12 +81,8 @@ public class ParquetRecordReaderTest extends RecordReaderTest {
@Test
public void testParquetRecordReader()
throws Exception {
- SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig();
- segmentGeneratorConfig.setInputFilePath(DATA_FILE.getAbsolutePath());
- segmentGeneratorConfig.setSchema(SCHEMA);
-
try (ParquetRecordReader recordReader = new ParquetRecordReader()) {
- recordReader.init(segmentGeneratorConfig);
+ recordReader.init(DATA_FILE.getAbsolutePath(), SCHEMA, null);
checkValue(recordReader);
recordReader.rewind();
checkValue(recordReader);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
index 0b5bfed..a1ed11a 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.pinot.common.data.Schema;
import org.apache.pinot.common.data.StarTreeIndexSpec;
import org.apache.pinot.common.segment.ReadMode;
import org.apache.pinot.common.utils.JsonUtils;
@@ -409,12 +410,12 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
switch (config.getFormat()) {
case PARQUET:
RecordReader parquetRecordReader = new ParquetRecordReader();
- parquetRecordReader.init(config);
+ parquetRecordReader.init(localFile, Schema.fromFile(new File(_schemaFile)), null);
driver.init(config, parquetRecordReader);
break;
case ORC:
RecordReader orcRecordReader = new ORCRecordReader();
- orcRecordReader.init(config);
+ orcRecordReader.init(localFile, Schema.fromFile(new File(_schemaFile)), null);
driver.init(config, orcRecordReader);
break;
default:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org