You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/11/26 02:57:12 UTC
[incubator-pinot] branch master updated: Rename segmentPushType and
SegmentPushFrequency to segmentIngestionType and segmentIngestionFrequency
(#6289)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 318c107 Rename segmentPushType and SegmentPushFrequency to segmentIngestionType and segmentIngestionFrequency (#6289)
318c107 is described below
commit 318c1077bb4a8aa74a03acad8f236aff8eb9fa0d
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Wed Nov 25 18:56:55 2020 -0800
Rename segmentPushType and SegmentPushFrequency to segmentIngestionType and segmentIngestionFrequency (#6289)
---
.../routing/timeboundary/TimeBoundaryManager.java | 2 +-
.../common/utils/config/TableConfigSerDeTest.java | 4 ++--
.../helix/core/retention/RetentionManager.java | 2 +-
.../controller/util/TableRetentionValidator.java | 2 +-
.../validation/OfflineSegmentIntervalChecker.java | 2 +-
.../apache/pinot/core/util/TableConfigUtils.java | 2 +-
.../batch/common/SegmentGenerationTaskRunner.java | 4 ++--
.../pinot/hadoop/job/HadoopSegmentCreationJob.java | 2 +-
.../hadoop/job/HadoopSegmentPreprocessingJob.java | 4 ++--
.../hadoop/job/mappers/SegmentCreationMapper.java | 4 ++--
.../spark/jobs/SparkSegmentCreationFunction.java | 4 ++--
.../pinot/spark/jobs/SparkSegmentCreationJob.java | 2 +-
.../table/ingestion/BatchIngestionConfig.java | 24 +++++++++----------
.../pinot/spi/utils/IngestionConfigUtils.java | 28 +++++++++++-----------
.../pinot/spi/utils/IngestionConfigUtilsTest.java | 16 ++++++-------
.../segment/converter/SegmentMergeCommand.java | 4 ++--
16 files changed, 53 insertions(+), 53 deletions(-)
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java
index 3097a1b..613b755 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java
@@ -84,7 +84,7 @@ public class TimeBoundaryManager {
// For HOURLY table with time unit other than DAYS, use (maxEndTime - 1 HOUR) as the time boundary; otherwise, use
// (maxEndTime - 1 DAY)
boolean isHourlyTable = CommonConstants.Table.PUSH_FREQUENCY_HOURLY
- .equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig))
+ .equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentIngestionFrequency(tableConfig))
&& _timeFormatSpec.getColumnUnit() != TimeUnit.DAYS;
_timeOffsetMs = isHourlyTable ? TimeUnit.HOURS.toMillis(1) : TimeUnit.DAYS.toMillis(1);
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
index bd7df17..ccf86cd 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
@@ -417,8 +417,8 @@ public class TableConfigSerDeTest {
assertNotNull(ingestionConfig.getBatchIngestionConfig().getBatchConfigMaps());
assertEquals(ingestionConfig.getBatchIngestionConfig().getBatchConfigMaps().size(), 1);
assertEquals(ingestionConfig.getBatchIngestionConfig().getBatchConfigMaps().get(0).get("batchType"), "s3");
- assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentPushType(), "APPEND");
- assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentPushFrequency(), "HOURLY");
+ assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentIngestionType(), "APPEND");
+ assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentIngestionFrequency(), "HOURLY");
assertNotNull(ingestionConfig.getStreamIngestionConfig());
assertNotNull(ingestionConfig.getStreamIngestionConfig().getStreamConfigMaps());
assertEquals(ingestionConfig.getStreamIngestionConfig().getStreamConfigMaps().size(), 1);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
index 2dcd3c4..a7a5f5a 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
@@ -101,7 +101,7 @@ public class RetentionManager extends ControllerPeriodicTask<Void> {
// For offline tables, ensure that the segmentPushType is APPEND.
SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
- String segmentPushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig);
+ String segmentPushType = IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig);
if (tableConfig.getTableType() == TableType.OFFLINE && !"APPEND".equalsIgnoreCase(segmentPushType)) {
LOGGER.info("Segment push type is not APPEND for table: {}, skip", tableNameWithType);
return;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java
index ba44225..bce9260 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java
@@ -109,7 +109,7 @@ public class TableRetentionValidator {
LOGGER.error("Table: {}, \"segmentsConfig\" field is missing in table config", tableName);
continue;
}
- String segmentPushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig);
+ String segmentPushType = IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig);
if (segmentPushType == null) {
LOGGER.error("Table: {}, null push type", tableName);
continue;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java
index e21c72e..7c1f951 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java
@@ -102,7 +102,7 @@ public class OfflineSegmentIntervalChecker extends ControllerPeriodicTask<Void>
.warn("Table: {} has {} segments with invalid interval", offlineTableName, numSegmentsWithInvalidIntervals);
}
Duration frequency =
- SegmentIntervalUtils.convertToDuration(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig));
+ SegmentIntervalUtils.convertToDuration(IngestionConfigUtils.getBatchSegmentIngestionFrequency(tableConfig));
numMissingSegments = computeNumMissingSegments(segmentIntervals, frequency);
}
// Update the gauge that contains the number of missing segments
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java
index e6cafaf..1074cde 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java
@@ -116,7 +116,7 @@ public final class TableConfigUtils {
String.format("Table: %s, \"segmentsConfig\" field is missing in table config", tableName));
}
- String segmentPushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig);
+ String segmentPushType = IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig);
// segmentPushType is not needed for Realtime table
if (tableConfig.getTableType() == TableType.OFFLINE && segmentPushType != null && !segmentPushType.isEmpty()) {
if (!segmentPushType.equalsIgnoreCase("REFRESH") && !segmentPushType.equalsIgnoreCase("APPEND")) {
diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java
index 4d15022..9c80942 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java
@@ -137,8 +137,8 @@ public class SegmentGenerationTaskRunner implements Serializable {
}
return new NormalizedDateSegmentNameGenerator(tableName, segmentNameGeneratorConfigs.get(SEGMENT_NAME_PREFIX),
Boolean.parseBoolean(segmentNameGeneratorConfigs.get(EXCLUDE_SEQUENCE_ID)),
- IngestionConfigUtils.getBatchSegmentPushType(tableConfig),
- IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), dateTimeFormatSpec);
+ IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig),
+ IngestionConfigUtils.getBatchSegmentIngestionFrequency(tableConfig), dateTimeFormatSpec);
default:
throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType);
}
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
index 2b77efc..474d9a4 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
@@ -143,7 +143,7 @@ public class HadoopSegmentCreationJob extends SegmentCreationJob {
SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
// For APPEND use case, timeColumnName and timeType must be set
- if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentPushType(tableConfig))) {
+ if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig))) {
Preconditions.checkState(validationConfig.getTimeColumnName() != null && validationConfig.getTimeType() != null,
"For APPEND use case, time column and type must be set");
}
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
index 9e5e3e1..e8d4b4f 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
@@ -368,7 +368,7 @@ public class HadoopSegmentPreprocessingJob extends SegmentPreprocessingJob {
// If the use case is an append use case, check that one time unit is contained in one file. If there is more than one,
// the job should be disabled, as we should not resize for these use cases. Therefore, setting the time column name
// and value
- if (IngestionConfigUtils.getBatchSegmentPushType(_tableConfig).equalsIgnoreCase("APPEND")) {
+ if (IngestionConfigUtils.getBatchSegmentIngestionType(_tableConfig).equalsIgnoreCase("APPEND")) {
job.getConfiguration().set(InternalConfigConstants.IS_APPEND, "true");
String timeColumnName = validationConfig.getTimeColumnName();
job.getConfiguration().set(InternalConfigConstants.TIME_COLUMN_CONFIG, timeColumnName);
@@ -385,7 +385,7 @@ public class HadoopSegmentPreprocessingJob extends SegmentPreprocessingJob {
}
}
job.getConfiguration().set(InternalConfigConstants.SEGMENT_PUSH_FREQUENCY,
- IngestionConfigUtils.getBatchSegmentPushFrequency(_tableConfig));
+ IngestionConfigUtils.getBatchSegmentIngestionFrequency(_tableConfig));
try (DataFileStream<GenericRecord> dataStreamReader = getAvroReader(path)) {
job.getConfiguration()
.set(InternalConfigConstants.TIME_COLUMN_VALUE, dataStreamReader.next().get(timeColumnName).toString());
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
index 9efb7a3..a54616f 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
@@ -163,8 +163,8 @@ public class SegmentCreationMapper extends Mapper<LongWritable, Text, LongWritab
_segmentNameGenerator =
new NormalizedDateSegmentNameGenerator(_rawTableName, _jobConf.get(JobConfigConstants.SEGMENT_NAME_PREFIX),
_jobConf.getBoolean(JobConfigConstants.EXCLUDE_SEQUENCE_ID, false),
- IngestionConfigUtils.getBatchSegmentPushType(_tableConfig),
- IngestionConfigUtils.getBatchSegmentPushFrequency(_tableConfig), dateTimeFormatSpec);
+ IngestionConfigUtils.getBatchSegmentIngestionType(_tableConfig),
+ IngestionConfigUtils.getBatchSegmentIngestionFrequency(_tableConfig), dateTimeFormatSpec);
break;
default:
throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType);
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java
index 60d3ef9..c3140db 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java
@@ -131,8 +131,8 @@ public class SparkSegmentCreationFunction implements Serializable {
_segmentNameGenerator =
new NormalizedDateSegmentNameGenerator(_rawTableName, _jobConf.get(JobConfigConstants.SEGMENT_NAME_PREFIX),
_jobConf.getBoolean(JobConfigConstants.EXCLUDE_SEQUENCE_ID, false),
- IngestionConfigUtils.getBatchSegmentPushType(_tableConfig),
- IngestionConfigUtils.getBatchSegmentPushFrequency(_tableConfig), dateTimeFormatSpec);
+ IngestionConfigUtils.getBatchSegmentIngestionType(_tableConfig),
+ IngestionConfigUtils.getBatchSegmentIngestionFrequency(_tableConfig), dateTimeFormatSpec);
break;
default:
throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType);
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java
index d737020..d3270b3 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java
@@ -156,7 +156,7 @@ public class SparkSegmentCreationJob extends SegmentCreationJob {
SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
// For APPEND use case, timeColumnName and timeType must be set
- if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentPushType(tableConfig))) {
+ if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig))) {
Preconditions.checkState(validationConfig.getTimeColumnName() != null && validationConfig.getTimeType() != null,
"For APPEND use case, time column and type must be set");
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java
index e9b8718..3cc1f92 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java
@@ -35,19 +35,19 @@ public class BatchIngestionConfig extends BaseJsonConfig {
@JsonPropertyDescription("Configs for all the batch sources to ingest from")
private final List<Map<String, String>> _batchConfigMaps;
- @JsonPropertyDescription("Push type APPEND or REFRESH")
- private final String _segmentPushType;
+ @JsonPropertyDescription("Ingestion type APPEND or REFRESH")
+ private final String _segmentIngestionType;
- @JsonPropertyDescription("Push frequency HOURLY or DAILY")
- private final String _segmentPushFrequency;
+ @JsonPropertyDescription("Ingestion frequency HOURLY or DAILY")
+ private final String _segmentIngestionFrequency;
@JsonCreator
public BatchIngestionConfig(@JsonProperty("batchConfigMaps") @Nullable List<Map<String, String>> batchConfigMaps,
- @JsonProperty("segmentPushType") String segmentPushType,
- @JsonProperty("segmentPushFrequency") String segmentPushFrequency) {
+ @JsonProperty("segmentIngestionType") String segmentIngestionType,
+ @JsonProperty("segmentIngestionFrequency") String segmentIngestionFrequency) {
_batchConfigMaps = batchConfigMaps;
- _segmentPushType = segmentPushType;
- _segmentPushFrequency = segmentPushFrequency;
+ _segmentIngestionType = segmentIngestionType;
+ _segmentIngestionFrequency = segmentIngestionFrequency;
}
@Nullable
@@ -55,11 +55,11 @@ public class BatchIngestionConfig extends BaseJsonConfig {
return _batchConfigMaps;
}
- public String getSegmentPushType() {
- return _segmentPushType;
+ public String getSegmentIngestionType() {
+ return _segmentIngestionType;
}
- public String getSegmentPushFrequency() {
- return _segmentPushFrequency;
+ public String getSegmentIngestionFrequency() {
+ return _segmentIngestionFrequency;
}
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
index 717aab2..62499dd 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
@@ -57,39 +57,39 @@ public final class IngestionConfigUtils {
}
/**
- * Fetches the configured segmentPushType (APPEND/REFRESH) from the table config
+ * Fetches the configured segmentIngestionType (APPEND/REFRESH) from the table config
* First checks in the ingestionConfig. If not found, checks in the segmentsConfig (has been deprecated from here in favor of ingestion config)
*/
- public static String getBatchSegmentPushType(TableConfig tableConfig) {
- String segmentPushType = null;
+ public static String getBatchSegmentIngestionType(TableConfig tableConfig) {
+ String segmentIngestionType = null;
if (tableConfig.getIngestionConfig() != null) {
BatchIngestionConfig batchIngestionConfig = tableConfig.getIngestionConfig().getBatchIngestionConfig();
if (batchIngestionConfig != null) {
- segmentPushType = batchIngestionConfig.getSegmentPushType();
+ segmentIngestionType = batchIngestionConfig.getSegmentIngestionType();
}
}
- if (segmentPushType == null) {
- segmentPushType = tableConfig.getValidationConfig().getSegmentPushType();
+ if (segmentIngestionType == null) {
+ segmentIngestionType = tableConfig.getValidationConfig().getSegmentPushType();
}
- return segmentPushType;
+ return segmentIngestionType;
}
/**
- * Fetches the configured segmentPushFrequency from the table config
+ * Fetches the configured segmentIngestionFrequency from the table config
* First checks in the ingestionConfig. If not found, checks in the segmentsConfig (has been deprecated from here in favor of ingestion config)
*/
- public static String getBatchSegmentPushFrequency(TableConfig tableConfig) {
- String segmentPushFrequency = null;
+ public static String getBatchSegmentIngestionFrequency(TableConfig tableConfig) {
+ String segmentIngestionFrequency = null;
if (tableConfig.getIngestionConfig() != null) {
BatchIngestionConfig batchIngestionConfig = tableConfig.getIngestionConfig().getBatchIngestionConfig();
if (batchIngestionConfig != null) {
- segmentPushFrequency = batchIngestionConfig.getSegmentPushFrequency();
+ segmentIngestionFrequency = batchIngestionConfig.getSegmentIngestionFrequency();
}
}
- if (segmentPushFrequency == null) {
- segmentPushFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency();
+ if (segmentIngestionFrequency == null) {
+ segmentIngestionFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency();
}
- return segmentPushFrequency;
+ return segmentIngestionFrequency;
}
}
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
index dee56bb..ed98a63 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
@@ -103,23 +103,23 @@ public class IngestionConfigUtilsTest {
TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build();
tableConfig
.setIngestionConfig(new IngestionConfig(new BatchIngestionConfig(null, "APPEND", "HOURLY"), null, null, null));
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), "HOURLY");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentIngestionFrequency(tableConfig), "HOURLY");
// get from ingestion config, even if present in segmentsConfig
SegmentsValidationAndRetentionConfig segmentsValidationAndRetentionConfig =
new SegmentsValidationAndRetentionConfig();
segmentsValidationAndRetentionConfig.setSegmentPushFrequency("DAILY");
tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), "HOURLY");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentIngestionFrequency(tableConfig), "HOURLY");
// get from segmentsConfig
tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build();
tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), "DAILY");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentIngestionFrequency(tableConfig), "DAILY");
// present nowhere
segmentsValidationAndRetentionConfig.setSegmentPushFrequency(null);
- Assert.assertNull(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig));
+ Assert.assertNull(IngestionConfigUtils.getBatchSegmentIngestionFrequency(tableConfig));
}
@Test
@@ -128,22 +128,22 @@ public class IngestionConfigUtilsTest {
TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build();
tableConfig
.setIngestionConfig(new IngestionConfig(new BatchIngestionConfig(null, "APPEND", "HOURLY"), null, null, null));
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushType(tableConfig), "APPEND");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig), "APPEND");
// get from ingestion config, even if present in segmentsConfig
SegmentsValidationAndRetentionConfig segmentsValidationAndRetentionConfig =
new SegmentsValidationAndRetentionConfig();
segmentsValidationAndRetentionConfig.setSegmentPushType("REFRESH");
tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushType(tableConfig), "APPEND");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig), "APPEND");
// get from segmentsConfig
tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build();
tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushType(tableConfig), "REFRESH");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig), "REFRESH");
// present nowhere
segmentsValidationAndRetentionConfig.setSegmentPushType(null);
- Assert.assertNull(IngestionConfigUtils.getBatchSegmentPushType(tableConfig));
+ Assert.assertNull(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig));
}
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java
index be2196d..bdf3f00 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java
@@ -239,8 +239,8 @@ public class SegmentMergeCommand extends AbstractBaseAdminCommand implements Com
// Fetch time related configurations from schema and table config.
SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
- String pushFrequency = IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig);
- String pushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig);
+ String pushFrequency = IngestionConfigUtils.getBatchSegmentIngestionFrequency(tableConfig);
+ String pushType = IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig);
String timeColumnName = validationConfig.getTimeColumnName();
DateTimeFormatSpec dateTimeFormatSpec = null;
if (timeColumnName != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org