You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2022/03/25 02:52:45 UTC
[pinot] branch master updated: Fixing TIMESTAMP data type usage during segment creation (#8407)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 37fcfb8 Fixing TIMESTAMP data type usage during segment creation (#8407)
37fcfb8 is described below
commit 37fcfb8fd51120026169a0653031c3049b3a5583
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Thu Mar 24 19:52:21 2022 -0700
Fixing TIMESTAMP data type usage during segment creation (#8407)
---
.../transform/transformer/datetime/DateTimeTransformerFactory.java | 6 +++---
.../apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java | 3 ++-
.../spi/creator/name/NormalizedDateSegmentNameGenerator.java | 4 ++--
pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java | 4 ++--
4 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/transformer/datetime/DateTimeTransformerFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/transformer/datetime/DateTimeTransformerFactory.java
index 60552eb..88e28ff 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/transformer/datetime/DateTimeTransformerFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/transformer/datetime/DateTimeTransformerFactory.java
@@ -35,14 +35,14 @@ public class DateTimeTransformerFactory {
TimeFormat inputTimeFormat = inputFormat.getTimeFormat();
TimeFormat outputTimeFormat = outputFormat.getTimeFormat();
- if (inputTimeFormat == TimeFormat.EPOCH) {
- if (outputTimeFormat == TimeFormat.EPOCH) {
+ if (inputTimeFormat == TimeFormat.EPOCH || inputTimeFormat == TimeFormat.TIMESTAMP) {
+ if (outputTimeFormat == TimeFormat.EPOCH || outputTimeFormat == TimeFormat.TIMESTAMP) {
return new EpochToEpochTransformer(inputFormat, outputFormat, outputGranularity);
} else {
return new EpochToSDFTransformer(inputFormat, outputFormat, outputGranularity);
}
} else {
- if (outputTimeFormat == TimeFormat.EPOCH) {
+ if (outputTimeFormat == TimeFormat.EPOCH || outputTimeFormat == TimeFormat.TIMESTAMP) {
return new SDFToEpochTransformer(inputFormat, outputFormat, outputGranularity);
} else {
return new SDFToSDFTransformer(inputFormat, outputFormat, outputGranularity);
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
index 5ae8966..aad0e63 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
@@ -71,7 +71,8 @@ public class SegmentPreprocessingMapper
String timeType = _jobConf.get(InternalConfigConstants.SEGMENT_TIME_TYPE);
String timeFormat = _jobConf.get(InternalConfigConstants.SEGMENT_TIME_FORMAT);
DateTimeFormatSpec dateTimeFormatSpec;
- if (timeFormat.equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString())) {
+ if (timeFormat.equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString()) || timeFormat.equals(
+ DateTimeFieldSpec.TimeFormat.TIMESTAMP.toString())) {
dateTimeFormatSpec = new DateTimeFormatSpec(1, timeType, timeFormat);
} else {
dateTimeFormatSpec = new DateTimeFormatSpec(1, timeType, timeFormat,
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
index a65ef57..decbc50 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
@@ -72,10 +72,10 @@ public class NormalizedDateSegmentNameGenerator implements SegmentNameGenerator
}
_outputSDF.setTimeZone(TimeZone.getTimeZone("UTC"));
- // Parse input time format: 'EPOCH' or 'SIMPLE_DATE_FORMAT' using pattern
+ // Parse input time format: 'EPOCH'/'TIMESTAMP' or 'SIMPLE_DATE_FORMAT' using pattern
Preconditions.checkNotNull(dateTimeFormatSpec);
TimeFormat timeFormat = dateTimeFormatSpec.getTimeFormat();
- if (timeFormat == TimeFormat.EPOCH) {
+ if (timeFormat == TimeFormat.EPOCH || timeFormat == TimeFormat.TIMESTAMP) {
_inputTimeUnit = dateTimeFormatSpec.getColumnUnit();
_inputSDF = null;
} else {
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java
index 0193d31..e445b2e 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java
@@ -752,8 +752,8 @@ public final class Schema implements Serializable {
TimeUnit incomingTimeUnit = incomingGranularitySpec.getTimeType();
String incomingTimeFormat = incomingGranularitySpec.getTimeFormat();
Preconditions.checkState(
- incomingTimeFormat.equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString()) && outgoingTimeFormat
- .equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString()),
+ (incomingTimeFormat.equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString()) || incomingTimeFormat.equals(
+ DateTimeFieldSpec.TimeFormat.TIMESTAMP.toString())) && outgoingTimeFormat.equals(incomingTimeFormat),
"Conversion from incoming to outgoing is not supported for SIMPLE_DATE_FORMAT");
String transformFunction =
constructTransformFunctionString(incomingName, incomingTimeSize, incomingTimeUnit, outgoingTimeSize,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org