You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/12/18 22:43:13 UTC
[incubator-pinot] 01/01: Fixing the init fields in record extractor,
as the fields might be modified after the init call
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch fixing_init_fields_for_record_Extractor
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 15daa5d3a5d74e578d012fe3d7c0066ef6c1978a
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Fri Dec 18 14:42:44 2020 -0800
Fixing the init fields in record extractor, as the fields might be modified after the init call
---
.../apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java | 6 +++++-
.../org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractor.java | 3 ++-
.../apache/pinot/plugin/inputformat/json/JSONRecordExtractor.java | 6 +++++-
.../pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractor.java | 6 +++++-
.../pinot/plugin/inputformat/thrift/ThriftRecordExtractor.java | 6 +++++-
5 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
index 7ba74af..65ca389 100644
--- a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
+++ b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
@@ -18,6 +18,8 @@
*/
package org.apache.pinot.plugin.inputformat.avro;
+import com.google.common.collect.ImmutableSet;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -39,9 +41,11 @@ public class AvroRecordExtractor extends BaseRecordExtractor<GenericRecord> {
@Override
public void init(@Nullable Set<String> fields, @Nullable RecordExtractorConfig recordExtractorConfig) {
- _fields = fields;
if (fields == null || fields.isEmpty()) {
_extractAll = true;
+ _fields = Collections.emptySet();
+ } else {
+ _fields = ImmutableSet.copyOf(fields);
}
}
diff --git a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractor.java b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractor.java
index 626e229..607fcd0 100644
--- a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractor.java
+++ b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractor.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.plugin.inputformat.csv;
+import com.google.common.collect.ImmutableSet;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.commons.csv.CSVRecord;
@@ -41,7 +42,7 @@ public class CSVRecordExtractor extends BaseRecordExtractor<CSVRecord> {
if (fields == null || fields.isEmpty()) {
_fields = csvRecordExtractorConfig.getColumnNames();
} else {
- _fields = fields;
+ _fields = ImmutableSet.copyOf(fields);
}
_multiValueDelimiter = csvRecordExtractorConfig.getMultiValueDelimiter();
}
diff --git a/pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordExtractor.java b/pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordExtractor.java
index 1eb3a3f..2806edd 100644
--- a/pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordExtractor.java
+++ b/pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordExtractor.java
@@ -18,6 +18,8 @@
*/
package org.apache.pinot.plugin.inputformat.json;
+import com.google.common.collect.ImmutableSet;
+import java.util.Collections;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
@@ -36,9 +38,11 @@ public class JSONRecordExtractor extends BaseRecordExtractor<Map<String, Object>
@Override
public void init(Set<String> fields, @Nullable RecordExtractorConfig recordExtractorConfig) {
- _fields = fields;
if (fields == null || fields.isEmpty()) {
_extractAll = true;
+ _fields = Collections.emptySet();
+ } else {
+ _fields = ImmutableSet.copyOf(fields);
}
}
diff --git a/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractor.java b/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractor.java
index 1379709..2a3a602 100644
--- a/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractor.java
+++ b/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractor.java
@@ -18,11 +18,13 @@
*/
package org.apache.pinot.plugin.inputformat.protobuf;
+import com.google.common.collect.ImmutableSet;
import com.google.protobuf.ByteString;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -43,9 +45,11 @@ public class ProtoBufRecordExtractor extends BaseRecordExtractor<Message> {
@Override
public void init(@Nullable Set<String> fields, RecordExtractorConfig recordExtractorConfig) {
- _fields = fields;
if (fields == null || fields.isEmpty()) {
_extractAll = true;
+ _fields = Collections.emptySet();
+ } else {
+ _fields = ImmutableSet.copyOf(fields);
}
}
diff --git a/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractor.java b/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractor.java
index 678e7aa..3b27f1f 100644
--- a/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractor.java
+++ b/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractor.java
@@ -18,6 +18,8 @@
*/
package org.apache.pinot.plugin.inputformat.thrift;
+import com.google.common.collect.ImmutableSet;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -41,10 +43,12 @@ public class ThriftRecordExtractor extends BaseRecordExtractor<TBase> {
@Override
public void init(@Nullable Set<String> fields, RecordExtractorConfig recordExtractorConfig) {
- _fields = fields;
_fieldIds = ((ThriftRecordExtractorConfig) recordExtractorConfig).getFieldIds();
if (fields == null || fields.isEmpty()) {
_extractAll = true;
+ _fields = Collections.emptySet();
+ } else {
+ _fields = ImmutableSet.copyOf(fields);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org