You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2022/04/13 04:03:12 UTC
[pinot] branch master updated: Make getConfigMapWithPrefix auto append tailing dot if missing (#8522)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 900f01f288 Make getConfigMapWithPrefix auto append tailing dot if missing (#8522)
900f01f288 is described below
commit 900f01f288c8d13738b782992b0f29c55d5b1a34
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Tue Apr 12 21:03:04 2022 -0700
Make getConfigMapWithPrefix auto append tailing dot if missing (#8522)
---
.../org/apache/pinot/controller/util/FileIngestionHelper.java | 2 +-
.../SegmentGenerationAndPushTaskExecutor.java | 2 +-
.../java/org/apache/pinot/spi/utils/IngestionConfigUtils.java | 11 +++++++----
.../org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java | 8 ++++++++
4 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
index a180d9cb55..92da871460 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
@@ -176,7 +176,7 @@ public class FileIngestionHelper {
if (!PinotFSFactory.isSchemeSupported(sourceFileURIScheme)) {
PinotFSFactory.register(sourceFileURIScheme, batchConfigMap.get(BatchConfigProperties.INPUT_FS_CLASS),
IngestionConfigUtils.getInputFsProps(IngestionConfigUtils.getConfigMapWithPrefix(
- batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX + IngestionConfigUtils.DOT_SEPARATOR)));
+ batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX)));
}
PinotFSFactory.create(sourceFileURIScheme).copyToLocalFile(sourceFileURI, destFile);
}
diff --git a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
index 25ffea0977..445a37c90c 100644
--- a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
+++ b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
@@ -306,7 +306,7 @@ public class SegmentGenerationAndPushTaskExecutor extends BaseTaskExecutor {
SegmentNameGeneratorSpec segmentNameGeneratorSpec = new SegmentNameGeneratorSpec();
segmentNameGeneratorSpec.setType(taskConfigs.get(BatchConfigProperties.SEGMENT_NAME_GENERATOR_TYPE));
segmentNameGeneratorSpec.setConfigs(IngestionConfigUtils.getConfigMapWithPrefix(taskConfigs,
- BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX + IngestionConfigUtils.DOT_SEPARATOR));
+ BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX));
taskSpec.setSegmentNameGeneratorSpec(segmentNameGeneratorSpec);
taskSpec.setCustomProperty(BatchConfigProperties.INPUT_DATA_FILE_URI_KEY, inputFileURI.toString());
return taskSpec;
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
index 3bd9e3a6ed..e89b28e831 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
@@ -117,7 +117,7 @@ public final class IngestionConfigUtils {
* Fetch the properties which belong to record reader, by removing the identifier prefix
*/
public static Map<String, String> getRecordReaderProps(Map<String, String> batchConfigMap) {
- return getConfigMapWithPrefix(batchConfigMap, BatchConfigProperties.RECORD_READER_PROP_PREFIX + DOT_SEPARATOR);
+ return getConfigMapWithPrefix(batchConfigMap, BatchConfigProperties.RECORD_READER_PROP_PREFIX);
}
/**
@@ -125,17 +125,17 @@ public final class IngestionConfigUtils {
*/
public static Map<String, String> getSegmentNameGeneratorProps(Map<String, String> batchConfigMap) {
return getConfigMapWithPrefix(batchConfigMap,
- BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX + DOT_SEPARATOR);
+ BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX);
}
public static PinotConfiguration getInputFsProps(Map<String, String> batchConfigMap) {
return new PinotConfiguration(
- getPropsWithPrefix(batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX + DOT_SEPARATOR));
+ getPropsWithPrefix(batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX));
}
public static PinotConfiguration getOutputFsProps(Map<String, String> batchConfigMap) {
return new PinotConfiguration(
- getPropsWithPrefix(batchConfigMap, BatchConfigProperties.OUTPUT_FS_PROP_PREFIX + DOT_SEPARATOR));
+ getPropsWithPrefix(batchConfigMap, BatchConfigProperties.OUTPUT_FS_PROP_PREFIX));
}
/**
@@ -160,6 +160,9 @@ public final class IngestionConfigUtils {
public static Map<String, String> getConfigMapWithPrefix(Map<String, String> batchConfigMap, String prefix) {
Map<String, String> props = new HashMap<>();
+ if (!prefix.endsWith(DOT_SEPARATOR)) {
+ prefix = prefix + DOT_SEPARATOR;
+ }
for (String configKey : batchConfigMap.keySet()) {
if (configKey.startsWith(prefix)) {
String[] splits = configKey.split(prefix, 2);
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
index bb6649c934..1121f87f90 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.spi.utils;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.util.HashMap;
import java.util.Map;
@@ -146,4 +147,11 @@ public class IngestionConfigUtilsTest {
segmentsValidationAndRetentionConfig.setSegmentPushType(null);
Assert.assertNull(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig));
}
+
+ @Test
+ public void testGetConfigMapWithPrefix() {
+ Map<String, String> testMap = ImmutableMap.of("k1", "v1", "k1.k2", "v2", "k1.k3", "v3", "k4", "v4");
+ Assert.assertEquals(2, IngestionConfigUtils.getConfigMapWithPrefix(testMap, "k1").size());
+ Assert.assertEquals(2, IngestionConfigUtils.getConfigMapWithPrefix(testMap, "k1.").size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org