You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2022/01/31 20:26:19 UTC
[pinot] 01/01: Set overwrite to false if exclude.sequence.id is enabled
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch disable-overwrite-if-sequence-id-is-excluded
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 82cd206285d049d63e7900e4c23b5c1ef37225a1
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Mon Jan 31 12:24:17 2022 -0800
Set overwrite to false if exclude.sequence.id is enabled
---
.../apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
index 4291790..6b499b4 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
@@ -308,8 +308,15 @@ public class SegmentCreationMapper extends Mapper<LongWritable, Text, LongWritab
hdfsSegmentTarFile = new Path(relativeOutputPath, segmentTarFileName);
}
_logger.info("Copying segment tar file from: {} to: {}", localSegmentTarFile, hdfsSegmentTarFile);
+ boolean overwriteExistingFile = true;
+ if (_jobConf.getBoolean(JobConfigConstants.EXCLUDE_SEQUENCE_ID, false)) {
+ _logger.info("Set overwrite to 'false' since \"{}\" is set to true in the job properties",
+ JobConfigConstants.EXCLUDE_SEQUENCE_ID);
+ overwriteExistingFile = false;
+ }
FileSystem.get(hdfsSegmentTarFile.toUri(), _jobConf)
- .copyFromLocalFile(true, true, new Path(localSegmentTarFile.getAbsolutePath()), hdfsSegmentTarFile);
+ .copyFromLocalFile(true, overwriteExistingFile, new Path(localSegmentTarFile.getAbsolutePath()),
+ hdfsSegmentTarFile);
context.write(new LongWritable(sequenceId), new Text(segmentTarFileName));
_logger.info("Finish generating segment: {} with HDFS input file: {}, sequence id: {}", segmentName, hdfsInputFile,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org