You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/11/25 09:36:26 UTC
[incubator-pinot] branch update_table_ingestion_batch_config
created (now 49b546c)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch update_table_ingestion_batch_config
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 49b546c Rename segmentPushType and SegmentPushFrequency to segmentLoadType and segmentLoadFrequency
This branch includes the following new commits:
new 49b546c Rename segmentPushType and SegmentPushFrequency to segmentLoadType and segmentLoadFrequency
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Rename segmentPushType and
SegmentPushFrequency to segmentLoadType and segmentLoadFrequency
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch update_table_ingestion_batch_config
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 49b546cc3968af7f79b2d920e2cb6ee930bc3872
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Wed Nov 25 01:33:58 2020 -0800
Rename segmentPushType and SegmentPushFrequency to segmentLoadType and segmentLoadFrequency
---
.../routing/timeboundary/TimeBoundaryManager.java | 2 +-
.../common/utils/config/TableConfigSerDeTest.java | 4 ++--
.../helix/core/retention/RetentionManager.java | 2 +-
.../controller/util/TableRetentionValidator.java | 2 +-
.../validation/OfflineSegmentIntervalChecker.java | 2 +-
.../apache/pinot/core/util/TableConfigUtils.java | 2 +-
.../batch/common/SegmentGenerationTaskRunner.java | 4 ++--
.../pinot/hadoop/job/HadoopSegmentCreationJob.java | 2 +-
.../hadoop/job/HadoopSegmentPreprocessingJob.java | 4 ++--
.../hadoop/job/mappers/SegmentCreationMapper.java | 4 ++--
.../spark/jobs/SparkSegmentCreationFunction.java | 4 ++--
.../pinot/spark/jobs/SparkSegmentCreationJob.java | 2 +-
.../table/ingestion/BatchIngestionConfig.java | 24 +++++++++----------
.../pinot/spi/utils/IngestionConfigUtils.java | 28 +++++++++++-----------
.../pinot/spi/utils/IngestionConfigUtilsTest.java | 16 ++++++-------
.../segment/converter/SegmentMergeCommand.java | 4 ++--
16 files changed, 53 insertions(+), 53 deletions(-)
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java
index 3097a1b..1aa9f52 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java
@@ -84,7 +84,7 @@ public class TimeBoundaryManager {
// For HOURLY table with time unit other than DAYS, use (maxEndTime - 1 HOUR) as the time boundary; otherwise, use
// (maxEndTime - 1 DAY)
boolean isHourlyTable = CommonConstants.Table.PUSH_FREQUENCY_HOURLY
- .equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig))
+ .equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig))
&& _timeFormatSpec.getColumnUnit() != TimeUnit.DAYS;
_timeOffsetMs = isHourlyTable ? TimeUnit.HOURS.toMillis(1) : TimeUnit.DAYS.toMillis(1);
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
index bd7df17..8ad0047 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
@@ -417,8 +417,8 @@ public class TableConfigSerDeTest {
assertNotNull(ingestionConfig.getBatchIngestionConfig().getBatchConfigMaps());
assertEquals(ingestionConfig.getBatchIngestionConfig().getBatchConfigMaps().size(), 1);
assertEquals(ingestionConfig.getBatchIngestionConfig().getBatchConfigMaps().get(0).get("batchType"), "s3");
- assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentPushType(), "APPEND");
- assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentPushFrequency(), "HOURLY");
+ assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentLoadType(), "APPEND");
+ assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentLoadFrequency(), "HOURLY");
assertNotNull(ingestionConfig.getStreamIngestionConfig());
assertNotNull(ingestionConfig.getStreamIngestionConfig().getStreamConfigMaps());
assertEquals(ingestionConfig.getStreamIngestionConfig().getStreamConfigMaps().size(), 1);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
index 2dcd3c4..2e522f1 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
@@ -101,7 +101,7 @@ public class RetentionManager extends ControllerPeriodicTask<Void> {
// For offline tables, ensure that the segmentPushType is APPEND.
SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
- String segmentPushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig);
+ String segmentPushType = IngestionConfigUtils.getBatchSegmentLoadType(tableConfig);
if (tableConfig.getTableType() == TableType.OFFLINE && !"APPEND".equalsIgnoreCase(segmentPushType)) {
LOGGER.info("Segment push type is not APPEND for table: {}, skip", tableNameWithType);
return;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java
index ba44225..f372152 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java
@@ -109,7 +109,7 @@ public class TableRetentionValidator {
LOGGER.error("Table: {}, \"segmentsConfig\" field is missing in table config", tableName);
continue;
}
- String segmentPushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig);
+ String segmentPushType = IngestionConfigUtils.getBatchSegmentLoadType(tableConfig);
if (segmentPushType == null) {
LOGGER.error("Table: {}, null push type", tableName);
continue;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java
index e21c72e..602aebc 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java
@@ -102,7 +102,7 @@ public class OfflineSegmentIntervalChecker extends ControllerPeriodicTask<Void>
.warn("Table: {} has {} segments with invalid interval", offlineTableName, numSegmentsWithInvalidIntervals);
}
Duration frequency =
- SegmentIntervalUtils.convertToDuration(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig));
+ SegmentIntervalUtils.convertToDuration(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig));
numMissingSegments = computeNumMissingSegments(segmentIntervals, frequency);
}
// Update the gauge that contains the number of missing segments
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java
index e6cafaf..75a353c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java
@@ -116,7 +116,7 @@ public final class TableConfigUtils {
String.format("Table: %s, \"segmentsConfig\" field is missing in table config", tableName));
}
- String segmentPushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig);
+ String segmentPushType = IngestionConfigUtils.getBatchSegmentLoadType(tableConfig);
// segmentPushType is not needed for Realtime table
if (tableConfig.getTableType() == TableType.OFFLINE && segmentPushType != null && !segmentPushType.isEmpty()) {
if (!segmentPushType.equalsIgnoreCase("REFRESH") && !segmentPushType.equalsIgnoreCase("APPEND")) {
diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java
index 4d15022..f304701 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java
@@ -137,8 +137,8 @@ public class SegmentGenerationTaskRunner implements Serializable {
}
return new NormalizedDateSegmentNameGenerator(tableName, segmentNameGeneratorConfigs.get(SEGMENT_NAME_PREFIX),
Boolean.parseBoolean(segmentNameGeneratorConfigs.get(EXCLUDE_SEQUENCE_ID)),
- IngestionConfigUtils.getBatchSegmentPushType(tableConfig),
- IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), dateTimeFormatSpec);
+ IngestionConfigUtils.getBatchSegmentLoadType(tableConfig),
+ IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig), dateTimeFormatSpec);
default:
throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType);
}
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
index 2b77efc..66c0ef4 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
@@ -143,7 +143,7 @@ public class HadoopSegmentCreationJob extends SegmentCreationJob {
SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
// For APPEND use case, timeColumnName and timeType must be set
- if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentPushType(tableConfig))) {
+ if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig))) {
Preconditions.checkState(validationConfig.getTimeColumnName() != null && validationConfig.getTimeType() != null,
"For APPEND use case, time column and type must be set");
}
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
index 9e5e3e1..036adb8 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
@@ -368,7 +368,7 @@ public class HadoopSegmentPreprocessingJob extends SegmentPreprocessingJob {
// If the use case is an append use case, check that one time unit is contained in one file. If there is more than one,
// the job should be disabled, as we should not resize for these use cases. Therefore, setting the time column name
// and value
- if (IngestionConfigUtils.getBatchSegmentPushType(_tableConfig).equalsIgnoreCase("APPEND")) {
+ if (IngestionConfigUtils.getBatchSegmentLoadType(_tableConfig).equalsIgnoreCase("APPEND")) {
job.getConfiguration().set(InternalConfigConstants.IS_APPEND, "true");
String timeColumnName = validationConfig.getTimeColumnName();
job.getConfiguration().set(InternalConfigConstants.TIME_COLUMN_CONFIG, timeColumnName);
@@ -385,7 +385,7 @@ public class HadoopSegmentPreprocessingJob extends SegmentPreprocessingJob {
}
}
job.getConfiguration().set(InternalConfigConstants.SEGMENT_PUSH_FREQUENCY,
- IngestionConfigUtils.getBatchSegmentPushFrequency(_tableConfig));
+ IngestionConfigUtils.getBatchSegmentLoadFrequency(_tableConfig));
try (DataFileStream<GenericRecord> dataStreamReader = getAvroReader(path)) {
job.getConfiguration()
.set(InternalConfigConstants.TIME_COLUMN_VALUE, dataStreamReader.next().get(timeColumnName).toString());
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
index 9efb7a3..594b3bd 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
@@ -163,8 +163,8 @@ public class SegmentCreationMapper extends Mapper<LongWritable, Text, LongWritab
_segmentNameGenerator =
new NormalizedDateSegmentNameGenerator(_rawTableName, _jobConf.get(JobConfigConstants.SEGMENT_NAME_PREFIX),
_jobConf.getBoolean(JobConfigConstants.EXCLUDE_SEQUENCE_ID, false),
- IngestionConfigUtils.getBatchSegmentPushType(_tableConfig),
- IngestionConfigUtils.getBatchSegmentPushFrequency(_tableConfig), dateTimeFormatSpec);
+ IngestionConfigUtils.getBatchSegmentLoadType(_tableConfig),
+ IngestionConfigUtils.getBatchSegmentLoadFrequency(_tableConfig), dateTimeFormatSpec);
break;
default:
throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType);
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java
index 60d3ef9..a0c055b 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java
@@ -131,8 +131,8 @@ public class SparkSegmentCreationFunction implements Serializable {
_segmentNameGenerator =
new NormalizedDateSegmentNameGenerator(_rawTableName, _jobConf.get(JobConfigConstants.SEGMENT_NAME_PREFIX),
_jobConf.getBoolean(JobConfigConstants.EXCLUDE_SEQUENCE_ID, false),
- IngestionConfigUtils.getBatchSegmentPushType(_tableConfig),
- IngestionConfigUtils.getBatchSegmentPushFrequency(_tableConfig), dateTimeFormatSpec);
+ IngestionConfigUtils.getBatchSegmentLoadType(_tableConfig),
+ IngestionConfigUtils.getBatchSegmentLoadFrequency(_tableConfig), dateTimeFormatSpec);
break;
default:
throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType);
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java
index d737020..7abf57b 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java
@@ -156,7 +156,7 @@ public class SparkSegmentCreationJob extends SegmentCreationJob {
SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
// For APPEND use case, timeColumnName and timeType must be set
- if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentPushType(tableConfig))) {
+ if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig))) {
Preconditions.checkState(validationConfig.getTimeColumnName() != null && validationConfig.getTimeType() != null,
"For APPEND use case, time column and type must be set");
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java
index e9b8718..90e1b8e 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java
@@ -35,19 +35,19 @@ public class BatchIngestionConfig extends BaseJsonConfig {
@JsonPropertyDescription("Configs for all the batch sources to ingest from")
private final List<Map<String, String>> _batchConfigMaps;
- @JsonPropertyDescription("Push type APPEND or REFRESH")
- private final String _segmentPushType;
+ @JsonPropertyDescription("Load type APPEND or REFRESH")
+ private final String _segmentLoadType;
- @JsonPropertyDescription("Push frequency HOURLY or DAILY")
- private final String _segmentPushFrequency;
+ @JsonPropertyDescription("Load frequency HOURLY or DAILY")
+ private final String _segmentLoadFrequency;
@JsonCreator
public BatchIngestionConfig(@JsonProperty("batchConfigMaps") @Nullable List<Map<String, String>> batchConfigMaps,
- @JsonProperty("segmentPushType") String segmentPushType,
- @JsonProperty("segmentPushFrequency") String segmentPushFrequency) {
+ @JsonProperty("segmentLoadType") String segmentLoadType,
+ @JsonProperty("segmentLoadFrequency") String segmentLoadFrequency) {
_batchConfigMaps = batchConfigMaps;
- _segmentPushType = segmentPushType;
- _segmentPushFrequency = segmentPushFrequency;
+ _segmentLoadType = segmentLoadType;
+ _segmentLoadFrequency = segmentLoadFrequency;
}
@Nullable
@@ -55,11 +55,11 @@ public class BatchIngestionConfig extends BaseJsonConfig {
return _batchConfigMaps;
}
- public String getSegmentPushType() {
- return _segmentPushType;
+ public String getSegmentLoadType() {
+ return _segmentLoadType;
}
- public String getSegmentPushFrequency() {
- return _segmentPushFrequency;
+ public String getSegmentLoadFrequency() {
+ return _segmentLoadFrequency;
}
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
index 717aab2..187ad68 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
@@ -57,39 +57,39 @@ public final class IngestionConfigUtils {
}
/**
- * Fetches the configured segmentPushType (APPEND/REFRESH) from the table config
+ * Fetches the configured segmentLoadType (APPEND/REFRESH) from the table config
* First checks in the ingestionConfig. If not found, checks in the segmentsConfig (has been deprecated from here in favor of ingestion config)
*/
- public static String getBatchSegmentPushType(TableConfig tableConfig) {
- String segmentPushType = null;
+ public static String getBatchSegmentLoadType(TableConfig tableConfig) {
+ String segmentLoadType = null;
if (tableConfig.getIngestionConfig() != null) {
BatchIngestionConfig batchIngestionConfig = tableConfig.getIngestionConfig().getBatchIngestionConfig();
if (batchIngestionConfig != null) {
- segmentPushType = batchIngestionConfig.getSegmentPushType();
+ segmentLoadType = batchIngestionConfig.getSegmentLoadType();
}
}
- if (segmentPushType == null) {
- segmentPushType = tableConfig.getValidationConfig().getSegmentPushType();
+ if (segmentLoadType == null) {
+ segmentLoadType = tableConfig.getValidationConfig().getSegmentPushType();
}
- return segmentPushType;
+ return segmentLoadType;
}
/**
- * Fetches the configured segmentPushFrequency from the table config
+ * Fetches the configured segmentLoadFrequency from the table config
* First checks in the ingestionConfig. If not found, checks in the segmentsConfig (has been deprecated from here in favor of ingestion config)
*/
- public static String getBatchSegmentPushFrequency(TableConfig tableConfig) {
- String segmentPushFrequency = null;
+ public static String getBatchSegmentLoadFrequency(TableConfig tableConfig) {
+ String segmentLoadFrequency = null;
if (tableConfig.getIngestionConfig() != null) {
BatchIngestionConfig batchIngestionConfig = tableConfig.getIngestionConfig().getBatchIngestionConfig();
if (batchIngestionConfig != null) {
- segmentPushFrequency = batchIngestionConfig.getSegmentPushFrequency();
+ segmentLoadFrequency = batchIngestionConfig.getSegmentLoadFrequency();
}
}
- if (segmentPushFrequency == null) {
- segmentPushFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency();
+ if (segmentLoadFrequency == null) {
+ segmentLoadFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency();
}
- return segmentPushFrequency;
+ return segmentLoadFrequency;
}
}
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
index dee56bb..cbd5c4c 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
@@ -103,23 +103,23 @@ public class IngestionConfigUtilsTest {
TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build();
tableConfig
.setIngestionConfig(new IngestionConfig(new BatchIngestionConfig(null, "APPEND", "HOURLY"), null, null, null));
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), "HOURLY");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig), "HOURLY");
// get from ingestion config, even if present in segmentsConfig
SegmentsValidationAndRetentionConfig segmentsValidationAndRetentionConfig =
new SegmentsValidationAndRetentionConfig();
segmentsValidationAndRetentionConfig.setSegmentPushFrequency("DAILY");
tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), "HOURLY");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig), "HOURLY");
// get from segmentsConfig
tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build();
tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), "DAILY");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig), "DAILY");
// present nowhere
segmentsValidationAndRetentionConfig.setSegmentPushFrequency(null);
- Assert.assertNull(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig));
+ Assert.assertNull(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig));
}
@Test
@@ -128,22 +128,22 @@ public class IngestionConfigUtilsTest {
TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build();
tableConfig
.setIngestionConfig(new IngestionConfig(new BatchIngestionConfig(null, "APPEND", "HOURLY"), null, null, null));
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushType(tableConfig), "APPEND");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig), "APPEND");
// get from ingestion config, even if present in segmentsConfig
SegmentsValidationAndRetentionConfig segmentsValidationAndRetentionConfig =
new SegmentsValidationAndRetentionConfig();
segmentsValidationAndRetentionConfig.setSegmentPushType("REFRESH");
tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushType(tableConfig), "APPEND");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig), "APPEND");
// get from segmentsConfig
tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build();
tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig);
- Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushType(tableConfig), "REFRESH");
+ Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig), "REFRESH");
// present nowhere
segmentsValidationAndRetentionConfig.setSegmentPushType(null);
- Assert.assertNull(IngestionConfigUtils.getBatchSegmentPushType(tableConfig));
+ Assert.assertNull(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig));
}
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java
index be2196d..dbbffbe 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java
@@ -239,8 +239,8 @@ public class SegmentMergeCommand extends AbstractBaseAdminCommand implements Com
// Fetch time related configurations from schema and table config.
SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
- String pushFrequency = IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig);
- String pushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig);
+ String pushFrequency = IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig);
+ String pushType = IngestionConfigUtils.getBatchSegmentLoadType(tableConfig);
String timeColumnName = validationConfig.getTimeColumnName();
DateTimeFormatSpec dateTimeFormatSpec = null;
if (timeColumnName != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org