You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by GitBox <gi...@apache.org> on 2022/03/09 19:18:15 UTC
[GitHub] [pinot] npawar commented on a change in pull request #8312: Refactor Flink Segment Writer to use File-based Segment writer
npawar commented on a change in pull request #8312:
URL: https://github.com/apache/pinot/pull/8312#discussion_r822997957
##########
File path: pinot-connectors/pinot-flink-connector/src/main/java/org/apache/pinot/connector/flink/sink/FlinkSegmentWriter.java
##########
@@ -63,32 +44,15 @@
*/
@SuppressWarnings("NullAway")
@NotThreadSafe
-public class FlinkSegmentWriter implements SegmentWriter {
+public class FlinkSegmentWriter extends FileBasedSegmentWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(FlinkSegmentWriter.class);
- private static final FileFormat BUFFER_FILE_FORMAT = FileFormat.AVRO;
private final int _indexOfSubtask;
-
- private TableConfig _tableConfig;
- private String _tableNameWithType;
- private BatchIngestionConfig _batchIngestionConfig;
- private BatchConfig _batchConfig;
- private String _outputDirURI;
- private Schema _schema;
- private Set<String> _fieldsToRead;
- private RecordTransformer _recordTransformer;
-
- private File _stagingDir;
- private File _bufferFile;
private int _rowCount;
Review comment:
rowId at the very least, in case you dont want to move seqId for backward compatibility reasons
##########
File path: pinot-connectors/pinot-flink-connector/src/main/java/org/apache/pinot/connector/flink/sink/FlinkSegmentWriter.java
##########
@@ -63,32 +44,15 @@
*/
@SuppressWarnings("NullAway")
@NotThreadSafe
-public class FlinkSegmentWriter implements SegmentWriter {
+public class FlinkSegmentWriter extends FileBasedSegmentWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(FlinkSegmentWriter.class);
- private static final FileFormat BUFFER_FILE_FORMAT = FileFormat.AVRO;
private final int _indexOfSubtask;
-
- private TableConfig _tableConfig;
- private String _tableNameWithType;
- private BatchIngestionConfig _batchIngestionConfig;
- private BatchConfig _batchConfig;
- private String _outputDirURI;
- private Schema _schema;
- private Set<String> _fieldsToRead;
- private RecordTransformer _recordTransformer;
-
- private File _stagingDir;
- private File _bufferFile;
private int _rowCount;
Review comment:
thanks for cleaning this up!
can rowId and seqId also move to FileBasedSegmentWriter? I think it will benefit from having these fields.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org