You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2021/10/27 17:13:47 UTC
[pinot] branch support-segmentNamePostfix-in-segment-name created
(now 70c1d2b)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch support-segmentNamePostfix-in-segment-name
in repository https://gitbox.apache.org/repos/asf/pinot.git.
at 70c1d2b Support segmentNamePostfix in segment name
This branch includes the following new commits:
new 70c1d2b Support segmentNamePostfix in segment name
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[pinot] 01/01: Support segmentNamePostfix in segment name
Posted by jl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch support-segmentNamePostfix-in-segment-name
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 70c1d2b4269fa3ea76aaadcd74ca1ac4ba450de6
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Wed Oct 27 10:13:04 2021 -0700
Support segmentNamePostfix in segment name
---
.../pinot/hadoop/job/mappers/SegmentCreationMapper.java | 3 ++-
.../creator/name/NormalizedDateSegmentNameGenerator.java | 14 +++++++++++++-
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
index 5808b42..4291790 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
@@ -165,7 +165,8 @@ public class SegmentCreationMapper extends Mapper<LongWritable, Text, LongWritab
new NormalizedDateSegmentNameGenerator(_rawTableName, _jobConf.get(JobConfigConstants.SEGMENT_NAME_PREFIX),
_jobConf.getBoolean(JobConfigConstants.EXCLUDE_SEQUENCE_ID, false),
IngestionConfigUtils.getBatchSegmentIngestionType(_tableConfig),
- IngestionConfigUtils.getBatchSegmentIngestionFrequency(_tableConfig), dateTimeFormatSpec);
+ IngestionConfigUtils.getBatchSegmentIngestionFrequency(_tableConfig), dateTimeFormatSpec,
+ _jobConf.get(JobConfigConstants.SEGMENT_NAME_POSTFIX));
break;
default:
throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType);
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
index a1ae0f6..96848de 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
@@ -41,6 +41,7 @@ public class NormalizedDateSegmentNameGenerator implements SegmentNameGenerator
private String _segmentNamePrefix;
private boolean _excludeSequenceId;
private boolean _appendPushType;
+ private String _segmentNamePostfix;
// For APPEND tables
private SimpleDateFormat _outputSDF;
@@ -52,11 +53,18 @@ public class NormalizedDateSegmentNameGenerator implements SegmentNameGenerator
public NormalizedDateSegmentNameGenerator(String tableName, @Nullable String segmentNamePrefix,
boolean excludeSequenceId, @Nullable String pushType, @Nullable String pushFrequency,
@Nullable DateTimeFormatSpec dateTimeFormatSpec) {
+ this(tableName, segmentNamePrefix, excludeSequenceId, pushType, pushFrequency, dateTimeFormatSpec, null);
+ }
+
+ public NormalizedDateSegmentNameGenerator(String tableName, @Nullable String segmentNamePrefix,
+ boolean excludeSequenceId, @Nullable String pushType, @Nullable String pushFrequency,
+ @Nullable DateTimeFormatSpec dateTimeFormatSpec, @Nullable String segmentNamePostfix) {
_segmentNamePrefix = segmentNamePrefix != null ? segmentNamePrefix.trim() : tableName;
Preconditions.checkArgument(
_segmentNamePrefix != null && isValidSegmentName(_segmentNamePrefix));
_excludeSequenceId = excludeSequenceId;
_appendPushType = "APPEND".equalsIgnoreCase(pushType);
+ _segmentNamePostfix = segmentNamePostfix;
// Include time info for APPEND push type
if (_appendPushType) {
@@ -96,7 +104,11 @@ public class NormalizedDateSegmentNameGenerator implements SegmentNameGenerator
return JOINER.join(_segmentNamePrefix, getNormalizedDate(Preconditions.checkNotNull(minTimeValue)),
getNormalizedDate(Preconditions.checkNotNull(maxTimeValue)), sequenceIdInSegmentName);
} else {
- return JOINER.join(_segmentNamePrefix, sequenceIdInSegmentName);
+ if (_segmentNamePostfix != null) {
+ return JOINER.join(_segmentNamePrefix, _segmentNamePostfix, sequenceIdInSegmentName);
+ } else {
+ return JOINER.join(_segmentNamePrefix, sequenceIdInSegmentName);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org