You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by mc...@apache.org on 2020/09/09 22:35:16 UTC
[incubator-pinot] branch master updated: Fix CSV reader class path
(#5992)
This is an automated email from the ASF dual-hosted git repository.
mcvsubbu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new feeba9e Fix CSV reader class path (#5992)
feeba9e is described below
commit feeba9e3b4eae3501947429451fa39cc21dd8a0f
Author: Subbu Subramaniam <mc...@users.noreply.github.com>
AuthorDate: Wed Sep 9 15:35:03 2020 -0700
Fix CSV reader class path (#5992)
Fixed the classpath of CSV reader in the code.
Introduced comment marker so that we can have comments
in the csv file as well
---
.../apache/pinot/plugin/inputformat/csv/CSVRecordReader.java | 2 ++
.../pinot/plugin/inputformat/csv/CSVRecordReaderConfig.java | 10 ++++++++++
.../org/apache/pinot/spi/data/readers/RecordReaderFactory.java | 2 +-
3 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java
index 43270a3..2802fb0 100644
--- a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java
+++ b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java
@@ -87,6 +87,8 @@ public class CSVRecordReader implements RecordReader {
} else {
format = format.withHeader(StringUtils.split(csvHeader, delimiter));
}
+ Character commentMarker = config.getCommentMarker();
+ format = format.withCommentMarker(commentMarker);
_format = format;
multiValueDelimiter = config.getMultiValueDelimiter();
}
diff --git a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReaderConfig.java b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReaderConfig.java
index 03c3610..b30c09f 100644
--- a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReaderConfig.java
+++ b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReaderConfig.java
@@ -33,6 +33,8 @@ public class CSVRecordReaderConfig implements RecordReaderConfig {
private char _delimiter = DEFAULT_DELIMITER;
private char _multiValueDelimiter = DEFAULT_MULTI_VALUE_DELIMITER;
+ private Character _commentMarker; // Default is null
+
public String getFileFormat() {
return _fileFormat;
}
@@ -65,6 +67,14 @@ public class CSVRecordReaderConfig implements RecordReaderConfig {
_multiValueDelimiter = multiValueDelimiter;
}
+ public Character getCommentMarker() {
+ return _commentMarker;
+ }
+
+ public void setCommentMarker(Character commentMarker) {
+ _commentMarker = commentMarker;
+ }
+
@Override
public String toString() {
return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReaderFactory.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReaderFactory.java
index 174decf..4bba3f5 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReaderFactory.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReaderFactory.java
@@ -41,7 +41,7 @@ public class RecordReaderFactory {
private static final String DEFAULT_CSV_RECORD_READER_CLASS =
"org.apache.pinot.plugin.inputformat.csv.CSVRecordReader";
private static final String DEFAULT_CSV_RECORD_READER_CONFIG_CLASS =
- "org.apache.pinot.csv.data.readers.CSVRecordReaderConfig";
+ "org.apache.pinot.plugin.inputformat.csv.CSVRecordReaderConfig";
private static final String DEFAULT_JSON_RECORD_READER_CLASS =
"org.apache.pinot.plugin.inputformat.json.JSONRecordReader";
private static final String DEFAULT_THRIFT_RECORD_READER_CLASS =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org