You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ki...@apache.org on 2019/11/27 06:49:37 UTC
[incubator-pinot] 01/01: Removing segment generation config from
RecordReader interface
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch refactor-record-reader
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5942774220df244dc25a93d5cc97dd78b2e618a2
Author: kishoreg <g....@gmail.com>
AuthorDate: Tue Nov 26 22:49:09 2019 -0800
Removing segment generation config from RecordReader interface
---
.../apache/pinot/core/data/readers/AvroRecordReader.java | 3 ++-
.../apache/pinot/core/data/readers/CSVRecordReader.java | 3 ++-
.../pinot/core/data/readers/GenericRowRecordReader.java | 3 ++-
.../apache/pinot/core/data/readers/JSONRecordReader.java | 4 +++-
.../data/readers/MultiplePinotSegmentRecordReader.java | 11 ++++++-----
.../pinot/core/data/readers/PinotSegmentRecordReader.java | 11 ++++++-----
.../org/apache/pinot/core/data/readers/RecordReader.java | 14 ++++++++++----
.../pinot/core/data/readers/RecordReaderFactory.java | 2 +-
.../apache/pinot/core/data/readers/ThriftRecordReader.java | 11 ++++++-----
.../indexsegment/generator/SegmentGeneratorConfig.java | 2 +-
.../apache/pinot/core/minion/BackfillDateTimeColumn.java | 4 +++-
.../java/org/apache/pinot/core/minion/SegmentPurger.java | 5 ++++-
.../pinot/core/minion/segment/MapperRecordReader.java | 4 +++-
.../pinot/core/minion/segment/ReducerRecordReader.java | 4 +++-
.../realtime/converter/RealtimeSegmentRecordReader.java | 4 +++-
.../org/apache/pinot/orc/data/readers/ORCRecordReader.java | 5 +++--
.../apache/pinot/orc/data/readers/ORCRecordReaderTest.java | 11 ++---------
.../pinot/parquet/data/readers/ParquetRecordReader.java | 7 ++++---
.../parquet/data/readers/ParquetRecordReaderTest.java | 6 +-----
.../pinot/tools/admin/command/CreateSegmentCommand.java | 5 +++--
20 files changed, 68 insertions(+), 51 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java
index bfa225f..9d0fa4e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/AvroRecordReader.java
@@ -56,7 +56,8 @@ public class AvroRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java
index 00cce5d..95ac10a 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/CSVRecordReader.java
@@ -94,7 +94,8 @@ public class CSVRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java
index 62466e2..e1f4ca2 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/GenericRowRecordReader.java
@@ -41,7 +41,8 @@ public class GenericRowRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java
index cc4bcd7..459a167 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/JSONRecordReader.java
@@ -62,7 +62,9 @@ public class JSONRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java
index d924152..dda5fd9 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/MultiplePinotSegmentRecordReader.java
@@ -53,11 +53,6 @@ public class MultiplePinotSegmentRecordReader implements RecordReader {
this(indexDirs, null, null);
}
- @Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
-
- }
-
/**
* Read records using the passed in schema and in the order of sorted column from multiple pinot segments.
* <p>Passed in schema must be a subset of the segment schema.
@@ -106,6 +101,12 @@ public class MultiplePinotSegmentRecordReader implements RecordReader {
}
}
+ @Override
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
+ }
+
/**
* Indicate whether the segment should be sorted or not
*/
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java
index 50c1808..fab4af7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/PinotSegmentRecordReader.java
@@ -60,11 +60,6 @@ public class PinotSegmentRecordReader implements RecordReader {
this(indexDir, null, null);
}
- @Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
-
- }
-
/**
* Read records using the segment schema with the given schema and sort order
* <p>Passed in schema must be a subset of the segment schema.
@@ -110,6 +105,12 @@ public class PinotSegmentRecordReader implements RecordReader {
}
}
+ @Override
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
+ }
+
/**
* Prepare sorted docIds in order of the given sort order columns
*/
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java
index 836d3d8..7ea744e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReader.java
@@ -20,9 +20,9 @@ package org.apache.pinot.core.data.readers;
import java.io.Closeable;
import java.io.IOException;
+import javax.annotation.Nullable;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
-import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
/**
@@ -34,10 +34,16 @@ import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
public interface RecordReader extends Closeable {
/**
- * Initializes the record reader when needed
+ * initializing recordreader with inputpath, schema and recordreader config. <br/>
+ * The implementation can chose to ignore one or more of these parameters and handle null gracefully <br/>
+ *
+ * @param inputPath absolute path to the file/directory
+ * @param schema Pinot Schema associated with the table
+ * @param recordReaderConfig config for the reader specific to the format. e.g. delimiter for csv format etc
+ * @throws Exception if the arguments are invalid
*/
- void init(SegmentGeneratorConfig segmentGeneratorConfig)
- throws IOException;
+ void init(@Nullable String inputPath, @Nullable Schema schema, @Nullable RecordReaderConfig recordReaderConfig)
+ throws Exception;
/**
* Return <code>true</code> if more records remain to be read.
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java
index 1d63b15..03edcfa 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/RecordReaderFactory.java
@@ -52,7 +52,7 @@ public class RecordReaderFactory {
.warn("Using class: {} to read segment, ignoring configured file format: {}", recordReaderPath, fileFormat);
}
RecordReader recordReader = (RecordReader) Class.forName(recordReaderPath).newInstance();
- recordReader.init(segmentGeneratorConfig);
+ recordReader.init(dataFile.getAbsolutePath(), schema, segmentGeneratorConfig.getReaderConfig());
return recordReader;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java
index 1f24010..7da8181 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/ThriftRecordReader.java
@@ -66,6 +66,12 @@ public class ThriftRecordReader implements RecordReader {
init();
}
+ @Override
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
+ }
+
private void init()
throws IOException {
_inputStream = RecordReaderUtils.getBufferedInputStream(_dataFile);
@@ -78,11 +84,6 @@ public class ThriftRecordReader implements RecordReader {
}
}
- @Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
-
- }
-
private boolean hasMoreToRead()
throws IOException {
_inputStream.mark(1);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
index fba03b2..ec51f79 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/generator/SegmentGeneratorConfig.java
@@ -84,7 +84,7 @@ public class SegmentGeneratorConfig {
private String _dataDir = null;
private String _inputFilePath = null;
private FileFormat _format = FileFormat.AVRO;
- private String _recordReaderPath = null;
+ private String _recordReaderPath = null; //TODO: this should be renamed to recordReaderClass or even better removed
private String _outDir = null;
private boolean _overwrite = false;
private String _tableName = null;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java
index 174826f..e485108 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/BackfillDateTimeColumn.java
@@ -33,6 +33,7 @@ import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.FileFormat;
import org.apache.pinot.core.data.readers.PinotSegmentRecordReader;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.data.recordtransformer.CompositeTransformer;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.segment.creator.RecordReaderSegmentCreationDataSource;
@@ -137,7 +138,8 @@ public class BackfillDateTimeColumn {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java
index 35ae334..82ea567 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java
@@ -32,6 +32,7 @@ import org.apache.pinot.common.segment.StarTreeMetadata;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.PinotSegmentRecordReader;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.core.segment.index.SegmentMetadataImpl;
@@ -170,7 +171,9 @@ public class SegmentPurger {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
+
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java
index b27dff8..07ffd51 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/MapperRecordReader.java
@@ -26,6 +26,7 @@ import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.MultiplePinotSegmentRecordReader;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
@@ -54,7 +55,8 @@ public class MapperRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java
index ddf7b81..9a2d133 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/segment/ReducerRecordReader.java
@@ -26,6 +26,7 @@ import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.PinotSegmentRecordReader;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
@@ -51,7 +52,8 @@ public class ReducerRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java
index 235ed00..d8e7c84 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/realtime/converter/RealtimeSegmentRecordReader.java
@@ -21,6 +21,7 @@ package org.apache.pinot.core.realtime.converter;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.indexsegment.mutable.MutableSegmentImpl;
@@ -51,7 +52,8 @@ public class RealtimeSegmentRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
+ throws Exception {
}
diff --git a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
index 7bfcf60..e9f982c 100644
--- a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
+++ b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
@@ -45,6 +45,7 @@ import org.apache.orc.mapred.OrcMapredRecordReader;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -83,9 +84,9 @@ public class ORCRecordReader implements RecordReader {
}
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig)
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
throws IOException {
- init(segmentGeneratorConfig.getInputFilePath(), segmentGeneratorConfig.getSchema());
+ init(inputPath, schema);
}
@Override
diff --git a/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java b/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java
index b3282ed..bd80ad1 100644
--- a/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java
+++ b/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java
@@ -114,15 +114,12 @@ public class ORCRecordReaderTest {
throws IOException {
ORCRecordReader orcRecordReader = new ORCRecordReader();
- SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig();
- segmentGeneratorConfig.setInputFilePath(ORC_FILE.getAbsolutePath());
Schema schema = new Schema();
FieldSpec xFieldSpec = new DimensionFieldSpec("x", FieldSpec.DataType.LONG, true);
schema.addField(xFieldSpec);
FieldSpec yFieldSpec = new DimensionFieldSpec("y", FieldSpec.DataType.BYTES, true);
schema.addField(yFieldSpec);
- segmentGeneratorConfig.setSchema(schema);
- orcRecordReader.init(segmentGeneratorConfig);
+ orcRecordReader.init(ORC_FILE.getAbsolutePath(), schema, null);
List<GenericRow> genericRows = new ArrayList<>();
while (orcRecordReader.hasNext()) {
@@ -140,16 +137,12 @@ public class ORCRecordReaderTest {
@Test
public void testReadMVData() throws IOException{
ORCRecordReader orcRecordReader = new ORCRecordReader();
-
- SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig();
- segmentGeneratorConfig.setInputFilePath(MULTIVALUE_ORC_FILE.getAbsolutePath());
Schema schema = new Schema();
FieldSpec emailsFieldSpec = new DimensionFieldSpec("emails", FieldSpec.DataType.STRING, false);
schema.addField(emailsFieldSpec);
FieldSpec xFieldSpec = new DimensionFieldSpec("x", FieldSpec.DataType.INT, true);
schema.addField(xFieldSpec);
- segmentGeneratorConfig.setSchema(schema);
- orcRecordReader.init(segmentGeneratorConfig);
+ orcRecordReader.init(MULTIVALUE_ORC_FILE.getAbsolutePath(), schema, null);
List<GenericRow> genericRows = new ArrayList<>();
while (orcRecordReader.hasNext()) {
diff --git a/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java b/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java
index 8e597e2..e80a3f4 100644
--- a/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java
+++ b/pinot-parquet/src/main/java/org/apache/pinot/parquet/data/readers/ParquetRecordReader.java
@@ -27,6 +27,7 @@ import org.apache.pinot.common.data.FieldSpec;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.core.data.GenericRow;
import org.apache.pinot.core.data.readers.RecordReader;
+import org.apache.pinot.core.data.readers.RecordReaderConfig;
import org.apache.pinot.core.data.readers.RecordReaderUtils;
import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import org.apache.pinot.core.util.AvroUtils;
@@ -43,10 +44,10 @@ public class ParquetRecordReader implements RecordReader {
private GenericRecord _nextRecord;
@Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig)
+ public void init(String inputPath, Schema schema, RecordReaderConfig recordReaderConfig)
throws IOException {
- _dataFilePath = new Path(segmentGeneratorConfig.getInputFilePath());
- _schema = segmentGeneratorConfig.getSchema();
+ _dataFilePath = new Path(inputPath);
+ _schema = schema;
AvroUtils.validateSchema(_schema, ParquetUtils.getParquetSchema(_dataFilePath));
_fieldSpecs = RecordReaderUtils.extractFieldSpecs(_schema);
diff --git a/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java b/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java
index 74822eb..cab095c 100644
--- a/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java
+++ b/pinot-parquet/src/test/java/org/apache/pinot/parquet/data/readers/ParquetRecordReaderTest.java
@@ -81,12 +81,8 @@ public class ParquetRecordReaderTest extends RecordReaderTest {
@Test
public void testParquetRecordReader()
throws Exception {
- SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig();
- segmentGeneratorConfig.setInputFilePath(DATA_FILE.getAbsolutePath());
- segmentGeneratorConfig.setSchema(SCHEMA);
-
try (ParquetRecordReader recordReader = new ParquetRecordReader()) {
- recordReader.init(segmentGeneratorConfig);
+ recordReader.init(DATA_FILE.getAbsolutePath(), SCHEMA, null);
checkValue(recordReader);
recordReader.rewind();
checkValue(recordReader);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
index 0b5bfed..a1ed11a 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.pinot.common.data.Schema;
import org.apache.pinot.common.data.StarTreeIndexSpec;
import org.apache.pinot.common.segment.ReadMode;
import org.apache.pinot.common.utils.JsonUtils;
@@ -409,12 +410,12 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
switch (config.getFormat()) {
case PARQUET:
RecordReader parquetRecordReader = new ParquetRecordReader();
- parquetRecordReader.init(config);
+ parquetRecordReader.init(localFile, Schema.fromFile(new File(_schemaFile)), null);
driver.init(config, parquetRecordReader);
break;
case ORC:
RecordReader orcRecordReader = new ORCRecordReader();
- orcRecordReader.init(config);
+ orcRecordReader.init(localFile, Schema.fromFile(new File(_schemaFile)), null);
driver.init(config, orcRecordReader);
break;
default:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org