You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2023/12/15 19:51:59 UTC
(pinot) branch master updated: Changes for migration to commons-configuration2 (#11985)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b44b3aeb8f Changes for migration to commons-configuration2 (#11985)
b44b3aeb8f is described below
commit b44b3aeb8f166317f5b00a6bf998bdfa217cf5aa
Author: Abhishek Sharma <ab...@spothero.com>
AuthorDate: Fri Dec 15 14:51:54 2023 -0500
Changes for migration to commons-configuration2 (#11985)
---
pinot-common/pom.xml | 14 --
.../segment/ColumnPartitionMetadataTest.java | 4 +-
.../resources/LLCSegmentCompletionHandlers.java | 3 +-
.../data/generator/PatternSeasonalGenerator.java | 2 +-
.../data/generator/PatternSequenceGenerator.java | 2 +-
.../data/generator/PatternSpikeGenerator.java | 2 +-
.../data/generator/PatternStringGenerator.java | 2 +-
pinot-core/pom.xml | 14 --
.../core/data/manager/BaseTableDataManager.java | 4 +-
.../core/data/manager/InstanceDataManager.java | 2 +-
.../BaseTableDataManagerAcquireSegmentTest.java | 2 +-
.../data/manager/BaseTableDataManagerTest.java | 2 +-
.../executor/QueryExecutorExceptionsTest.java | 2 +-
.../core/query/executor/QueryExecutorTest.java | 2 +-
.../pinot/queries/ExplainPlanQueriesTest.java | 2 +-
.../queries/SegmentWithNullValueVectorTest.java | 2 +-
pinot-distribution/pom.xml | 4 +
pinot-integration-test-base/pom.xml | 4 +
pinot-integration-tests/pom.xml | 4 +
.../org/apache/pinot/minion/MinionConfTest.java | 4 +-
pinot-plugins/pinot-file-system/pinot-gcs/pom.xml | 4 -
pinot-plugins/pinot-file-system/pinot-s3/pom.xml | 4 -
.../parquet/ParquetRecordReaderConfig.java | 2 +-
.../pinot-stream-ingestion/pinot-pulsar/pom.xml | 4 +
.../local/data/manager/TableDataManagerConfig.java | 4 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 10 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 6 +-
.../local/segment/index/loader/LoaderUtils.java | 2 +-
.../ColumnMinMaxValueGenerator.java | 4 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 4 +-
.../segment/store/SegmentLocalFSDirectory.java | 10 +-
.../segment/store/SingleFileIndexDirectory.java | 4 +-
.../local/segment/store/StarTreeIndexReader.java | 5 +-
.../local/startree/StarTreeBuilderUtils.java | 4 +-
.../startree/v2/builder/BaseSingleTreeBuilder.java | 2 +-
.../startree/v2/builder/MultipleTreesBuilder.java | 17 +-
.../v2/builder/OffHeapSingleTreeBuilder.java | 2 +-
.../v2/builder/OnHeapSingleTreeBuilder.java | 2 +-
.../v2/builder/StarTreeIndexSeparator.java | 7 +-
.../v2/builder/StarTreeV2BuilderConfig.java | 2 +-
.../startree/v2/store/StarTreeIndexMapUtils.java | 9 +-
.../impl/SegmentColumnarIndexCreatorTest.java | 13 +-
.../segment/index/SegmentMetadataImplTest.java | 3 +-
.../index/loader/ForwardIndexHandlerTest.java | 9 +-
.../index/loader/SegmentPreProcessorTest.java | 14 +-
.../store/SingleFileIndexDirectoryTest.java | 2 +-
.../segment/store/StarTreeIndexReaderTest.java | 3 +-
.../v2/builder/StarTreeIndexSeparatorTest.java | 5 +-
.../pinot/segment/spi/creator/SegmentCreator.java | 2 +-
.../spi/index/metadata/ColumnMetadataImpl.java | 4 +-
.../spi/index/metadata/SegmentMetadataImpl.java | 12 +-
.../spi/index/startree/StarTreeV2Metadata.java | 2 +-
.../pinot/segment/spi/store/SegmentDirectory.java | 2 +-
.../segment/spi/utils/SegmentMetadataUtils.java | 17 +-
pinot-server/pom.xml | 14 --
.../starter/helix/HelixInstanceDataManager.java | 2 +-
.../helix/HelixInstanceDataManagerConfig.java | 2 +-
.../pinot/server/api/TablesResourceTest.java | 3 +-
pinot-spi/pom.xml | 18 +-
.../pinot/spi/env/CommonsConfigurationUtils.java | 206 ++++++++-------------
.../apache/pinot/spi/env/PinotConfiguration.java | 8 +-
.../spi/env/CommonsConfigurationUtilsTest.java | 15 +-
.../pinot/spi/env/PinotConfigurationTest.java | 2 +-
.../tools/filesystem/PinotFSBenchmarkDriver.java | 2 +-
.../converter/DictionaryToRawIndexConverter.java | 13 +-
pom.xml | 13 --
66 files changed, 245 insertions(+), 326 deletions(-)
diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml
index 8fc3260666..3fb09c7340 100644
--- a/pinot-common/pom.xml
+++ b/pinot-common/pom.xml
@@ -271,20 +271,6 @@
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadataTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadataTest.java
index eacbc68fcd..b939367f7c 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadataTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadataTest.java
@@ -21,7 +21,8 @@ package org.apache.pinot.common.metadata.segment;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.convert.LegacyListDelimiterHandler;
import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.utils.JsonUtils;
import org.testng.annotations.Test;
@@ -65,6 +66,7 @@ public class ColumnPartitionMetadataTest {
@Test
public void testLegacyPartitionRangesConfig() {
PropertiesConfiguration config = new PropertiesConfiguration();
+ config.setListDelimiterHandler(new LegacyListDelimiterHandler(','));
config.setProperty("partitionRanges", LEGACY_PARTITION_RANGES_STRING);
Set<Integer> actual = ColumnPartitionMetadata.extractPartitions(config.getList("partitionRanges"));
assertEquals(actual, PARTITIONS);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
index 9ffd0a5416..cef0de67b4 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
@@ -36,6 +36,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.metrics.ControllerGauge;
import org.apache.pinot.common.protocols.SegmentCompletionProtocol;
@@ -458,7 +459,7 @@ public class LLCSegmentCompletionHandlers {
* temporarily.
*/
private static SegmentMetadataImpl extractSegmentMetadataFromForm(FormDataMultiPart form, String segmentName)
- throws IOException {
+ throws IOException, ConfigurationException {
File tempIndexDir = org.apache.pinot.common.utils.FileUtils.concatAndValidateFile(
ControllerFilePathProvider.getInstance().getUntarredFileTempDir(), getTempSegmentFileName(segmentName),
"Invalid segment name: %s", segmentName);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSeasonalGenerator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSeasonalGenerator.java
index 6dc1f75529..c15dad7dcc 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSeasonalGenerator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSeasonalGenerator.java
@@ -21,7 +21,7 @@ package org.apache.pinot.controller.recommender.data.generator;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import org.apache.commons.configuration.PropertyConverter;
+import org.apache.commons.configuration2.convert.PropertyConverter;
import org.apache.commons.math3.distribution.NormalDistribution;
import org.apache.commons.math3.random.Well19937c;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSequenceGenerator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSequenceGenerator.java
index 2469571a80..2dfd493961 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSequenceGenerator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSequenceGenerator.java
@@ -19,7 +19,7 @@
package org.apache.pinot.controller.recommender.data.generator;
import java.util.Map;
-import org.apache.commons.configuration.PropertyConverter;
+import org.apache.commons.configuration2.convert.PropertyConverter;
/**
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSpikeGenerator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSpikeGenerator.java
index 7dede83164..545ca2a696 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSpikeGenerator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternSpikeGenerator.java
@@ -19,7 +19,7 @@
package org.apache.pinot.controller.recommender.data.generator;
import java.util.Map;
-import org.apache.commons.configuration.PropertyConverter;
+import org.apache.commons.configuration2.convert.PropertyConverter;
import org.apache.commons.math3.distribution.AbstractRealDistribution;
import org.apache.commons.math3.distribution.ExponentialDistribution;
import org.apache.commons.math3.distribution.LogNormalDistribution;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternStringGenerator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternStringGenerator.java
index 0f67a2963e..8fc3c40bc7 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternStringGenerator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/PatternStringGenerator.java
@@ -20,7 +20,7 @@ package org.apache.pinot.controller.recommender.data.generator;
import java.util.List;
import java.util.Map;
-import org.apache.commons.configuration.PropertyConverter;
+import org.apache.commons.configuration2.convert.PropertyConverter;
/**
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index 815bf75291..dcacac4c36 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -72,20 +72,6 @@
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
index 2335a0fc33..f13f04c3b2 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
@@ -39,8 +39,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationConverter;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.ConfigurationConverter;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
index ffb5923411..aab7c08b0b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
@@ -24,7 +24,7 @@ import java.util.Set;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;
-import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.helix.HelixManager;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
index 9f1dfbbe0a..c5a38c4f94 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
@@ -31,7 +31,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.configuration.MapConfiguration;
+import org.apache.commons.configuration2.MapConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.helix.HelixManager;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java
index 7eb2ef0acd..9113a327ad 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import org.apache.commons.configuration.MapConfiguration;
+import org.apache.commons.configuration2.MapConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.helix.HelixManager;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java
index 65bb9a60a0..27d4893403 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorExceptionsTest.java
@@ -156,7 +156,7 @@ public class QueryExecutorExceptionsTest {
resourceUrl = getClass().getClassLoader().getResource(QUERY_EXECUTOR_CONFIG_PATH);
assertNotNull(resourceUrl);
PropertiesConfiguration queryExecutorConfig =
- CommonsConfigurationUtils.loadFromFile(new File(resourceUrl.getFile()));
+ CommonsConfigurationUtils.fromFile(new File(resourceUrl.getFile()));
_queryExecutor = new ServerQueryExecutorV1Impl();
_queryExecutor.init(new PinotConfiguration(queryExecutorConfig), instanceDataManager, _serverMetrics);
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorTest.java
index 4398e796bf..d68c6ab6f4 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/query/executor/QueryExecutorTest.java
@@ -155,7 +155,7 @@ public class QueryExecutorTest {
resourceUrl = getClass().getClassLoader().getResource(QUERY_EXECUTOR_CONFIG_PATH);
Assert.assertNotNull(resourceUrl);
PropertiesConfiguration queryExecutorConfig =
- CommonsConfigurationUtils.loadFromFile(new File(resourceUrl.getFile()));
+ CommonsConfigurationUtils.fromFile(new File(resourceUrl.getFile()));
_queryExecutor = new ServerQueryExecutorV1Impl();
_queryExecutor.init(new PinotConfiguration(queryExecutorConfig), instanceDataManager, _serverMetrics);
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java
index 73f6f7dcd1..a5c2cc3f5c 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java
@@ -316,7 +316,7 @@ public class ExplainPlanQueriesTest extends BaseQueriesTest {
URL resourceUrl = getClass().getClassLoader().getResource(QUERY_EXECUTOR_CONFIG_PATH);
Assert.assertNotNull(resourceUrl);
PropertiesConfiguration queryExecutorConfig =
- CommonsConfigurationUtils.loadFromFile(new File(resourceUrl.getFile()));
+ CommonsConfigurationUtils.fromFile(new File(resourceUrl.getFile()));
_queryExecutor = new ServerQueryExecutorV1Impl();
_queryExecutor.init(new PinotConfiguration(queryExecutorConfig), instanceDataManager, _serverMetrics);
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/SegmentWithNullValueVectorTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/SegmentWithNullValueVectorTest.java
index 1506a44426..ddf46340be 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/SegmentWithNullValueVectorTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/SegmentWithNullValueVectorTest.java
@@ -171,7 +171,7 @@ public class SegmentWithNullValueVectorTest {
URL resourceUrl = getClass().getClassLoader().getResource(QUERY_EXECUTOR_CONFIG_PATH);
Assert.assertNotNull(resourceUrl);
PropertiesConfiguration queryExecutorConfig =
- CommonsConfigurationUtils.loadFromFile(new File(resourceUrl.getFile()));
+ CommonsConfigurationUtils.fromFile(new File(resourceUrl.getFile()));
_queryExecutor = new ServerQueryExecutorV1Impl();
_queryExecutor.init(new PinotConfiguration(queryExecutorConfig), _instanceDataManager, _serverMetrics);
}
diff --git a/pinot-distribution/pom.xml b/pinot-distribution/pom.xml
index 4164ec80d7..ce171f0006 100644
--- a/pinot-distribution/pom.xml
+++ b/pinot-distribution/pom.xml
@@ -112,6 +112,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/pinot-integration-test-base/pom.xml b/pinot-integration-test-base/pom.xml
index 5394fc089e..d734ca2eed 100644
--- a/pinot-integration-test-base/pom.xml
+++ b/pinot-integration-test-base/pom.xml
@@ -64,6 +64,10 @@
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/pinot-integration-tests/pom.xml b/pinot-integration-tests/pom.xml
index 6e20646d4e..2328b3a628 100644
--- a/pinot-integration-tests/pom.xml
+++ b/pinot-integration-tests/pom.xml
@@ -196,6 +196,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/pinot-minion/src/test/java/org/apache/pinot/minion/MinionConfTest.java b/pinot-minion/src/test/java/org/apache/pinot/minion/MinionConfTest.java
index 6ec33a5d59..841cd5c24a 100644
--- a/pinot-minion/src/test/java/org/apache/pinot/minion/MinionConfTest.java
+++ b/pinot-minion/src/test/java/org/apache/pinot/minion/MinionConfTest.java
@@ -39,7 +39,7 @@ public class MinionConfTest {
CommonConstants.Minion.DEPRECATED_PREFIX_OF_CONFIG_OF_SEGMENT_UPLOADER,
CommonConstants.Minion.DEPRECATED_PREFIX_OF_CONFIG_OF_PINOT_CRYPTER
};
- PropertiesConfiguration config = CommonsConfigurationUtils.loadFromPath(
+ PropertiesConfiguration config = CommonsConfigurationUtils.fromPath(
PropertiesConfiguration.class.getClassLoader().getResource("pinot-configuration-old-minion.properties")
.getFile());
PinotConfiguration rawCfg = new PinotConfiguration(config);
@@ -51,7 +51,7 @@ public class MinionConfTest {
}
// Check configs with new names that have the pinot.minion prefix.
- config = CommonsConfigurationUtils.loadFromPath(
+ config = CommonsConfigurationUtils.fromPath(
PropertiesConfiguration.class.getClassLoader().getResource("pinot-configuration-new-minion.properties")
.getFile());
rawCfg = new PinotConfiguration(config);
diff --git a/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml b/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml
index 999c6619ef..e34797c1cb 100644
--- a/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml
+++ b/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml
@@ -38,10 +38,6 @@
<phase.prop>package</phase.prop>
</properties>
<dependencies>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- </dependency>
<!-- google cloud storage -->
<dependency>
<groupId>com.google.cloud</groupId>
diff --git a/pinot-plugins/pinot-file-system/pinot-s3/pom.xml b/pinot-plugins/pinot-file-system/pinot-s3/pom.xml
index 3b79fbfe75..c0fdfe5030 100644
--- a/pinot-plugins/pinot-file-system/pinot-s3/pom.xml
+++ b/pinot-plugins/pinot-file-system/pinot-s3/pom.xml
@@ -43,10 +43,6 @@
</properties>
<dependencies>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-spi</artifactId>
diff --git a/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderConfig.java b/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderConfig.java
index e6da878589..e9893bd369 100644
--- a/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderConfig.java
+++ b/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderConfig.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.plugin.inputformat.parquet;
-import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration2.Configuration;
import org.apache.pinot.spi.data.readers.RecordReaderConfig;
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml
index 495a71d483..58009972a7 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml
@@ -239,6 +239,10 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManagerConfig.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManagerConfig.java
index fd8fe29627..b17c24e564 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManagerConfig.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/data/manager/TableDataManagerConfig.java
@@ -22,8 +22,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.pinot.spi.config.instance.InstanceDataManagerConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
index 3a7a96415a..15ae19a9f3 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
@@ -32,8 +32,8 @@ import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.pinot.common.utils.FileUtils;
import org.apache.pinot.segment.local.io.util.PinotDataBitSet;
import org.apache.pinot.segment.local.segment.creator.impl.nullvalue.NullValueVectorCreator;
@@ -458,8 +458,8 @@ public class SegmentColumnarIndexCreator implements SegmentCreator {
private void writeMetadata()
throws ConfigurationException {
- PropertiesConfiguration properties =
- new PropertiesConfiguration(new File(_indexDir, V1Constants.MetadataKeys.METADATA_FILE_NAME));
+ File metadataFile = new File(_indexDir, V1Constants.MetadataKeys.METADATA_FILE_NAME);
+ PropertiesConfiguration properties = CommonsConfigurationUtils.fromFile(metadataFile);
properties.setProperty(SEGMENT_CREATOR_VERSION, _config.getCreatorVersion());
properties.setProperty(SEGMENT_PADDING_CHARACTER, String.valueOf(V1Constants.Str.DEFAULT_STRING_PAD_CHAR));
@@ -554,7 +554,7 @@ public class SegmentColumnarIndexCreator implements SegmentCreator {
properties.setProperty(Realtime.END_OFFSET, segmentZKPropsConfig.getEndOffset());
}
- properties.save();
+ CommonsConfigurationUtils.saveToFile(properties, metadataFile);
}
public static void addColumnMetadataInfo(PropertiesConfiguration properties, String column,
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
index 3ee1b3912a..a61a3acc8a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
@@ -30,8 +30,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.spi.V1Constants;
@@ -211,7 +211,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
final PropertiesConfiguration properties = CommonsConfigurationUtils.fromFile(v2MetadataFile);
// update the segment version
properties.setProperty(V1Constants.MetadataKeys.Segment.SEGMENT_VERSION, SegmentVersion.v3.toString());
- properties.save(v3MetadataFile);
+ CommonsConfigurationUtils.saveToFile(properties, v3MetadataFile);
}
private void copyCreationMetadataIfExists(File currentDir, File v3Dir)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/LoaderUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/LoaderUtils.java
index d417c223ca..8ab7b5678f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/LoaderUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/LoaderUtils.java
@@ -23,7 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.spi.index.IndexType;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
index 6d3cc83c10..7b627c5692 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
@@ -21,7 +21,7 @@ package org.apache.pinot.segment.local.segment.index.loader.columnminmaxvalue;
import com.google.common.base.Preconditions;
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentColumnarIndexCreator;
import org.apache.pinot.segment.local.segment.index.forward.ForwardIndexReaderFactory;
@@ -82,7 +82,7 @@ public class ColumnMinMaxValueGenerator {
addColumnMinMaxValueForColumn(column);
}
if (_minMaxValueAdded) {
- SegmentMetadataUtils.savePropertiesConfiguration(_segmentProperties);
+ SegmentMetadataUtils.savePropertiesConfiguration(_segmentProperties, _segmentMetadata.getIndexDir());
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
index 09643966c2..bb55d27f33 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.pinot.common.function.FunctionUtils;
import org.apache.pinot.common.utils.PinotDataType;
import org.apache.pinot.segment.local.function.FunctionEvaluator;
@@ -211,7 +211,7 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
_segmentProperties.setProperty(V1Constants.MetadataKeys.Segment.DATETIME_COLUMNS, dateTimeColumns);
// Save the new metadata
- SegmentMetadataUtils.savePropertiesConfiguration(_segmentProperties);
+ SegmentMetadataUtils.savePropertiesConfiguration(_segmentProperties, _segmentMetadata.getIndexDir());
}
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
index b5efaa215a..e158219a08 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
@@ -29,7 +29,7 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.Nullable;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.IndexType;
@@ -77,7 +77,7 @@ public class SegmentLocalFSDirectory extends SegmentDirectory {
}
public SegmentLocalFSDirectory(File directory, ReadMode readMode)
- throws IOException {
+ throws IOException, ConfigurationException {
this(directory, new SegmentMetadataImpl(directory), readMode);
}
@@ -269,7 +269,11 @@ public class SegmentLocalFSDirectory extends SegmentDirectory {
break;
}
if (CollectionUtils.isNotEmpty(_segmentMetadata.getStarTreeV2MetadataList())) {
- _starTreeIndexReader = new StarTreeIndexReader(_segmentDirectory, _segmentMetadata, _readMode);
+ try {
+ _starTreeIndexReader = new StarTreeIndexReader(_segmentDirectory, _segmentMetadata, _readMode);
+ } catch (ConfigurationException e) {
+ throw new RuntimeException(e);
+ }
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectory.java
index 3d79e00de8..78968106e8 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectory.java
@@ -35,8 +35,8 @@ import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.IndexType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/StarTreeIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/StarTreeIndexReader.java
index 524fab0d23..770af1b58b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/StarTreeIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/StarTreeIndexReader.java
@@ -28,6 +28,7 @@ import java.nio.ByteOrder;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.pinot.segment.local.startree.v2.store.StarTreeIndexMapUtils;
import org.apache.pinot.segment.spi.index.IndexType;
import org.apache.pinot.segment.spi.index.StandardIndexes;
@@ -64,7 +65,7 @@ public class StarTreeIndexReader implements Closeable {
* @param readMode mmap vs heap mode
*/
public StarTreeIndexReader(File segmentDirectory, SegmentMetadataImpl segmentMetadata, ReadMode readMode)
- throws IOException {
+ throws IOException, ConfigurationException {
Preconditions.checkNotNull(segmentDirectory);
Preconditions.checkArgument(segmentDirectory.exists(), "SegmentDirectory: " + segmentDirectory + " does not exist");
Preconditions.checkArgument(segmentDirectory.isDirectory(),
@@ -82,7 +83,7 @@ public class StarTreeIndexReader implements Closeable {
}
private void load()
- throws IOException {
+ throws IOException, ConfigurationException {
List<Map<StarTreeIndexMapUtils.IndexKey, StarTreeIndexMapUtils.IndexValue>> indexMapList;
try (InputStream inputStream = new FileInputStream(
new File(_segmentDirectory, StarTreeV2Constants.INDEX_MAP_FILE_NAME))) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
index bc678cc58a..59625678eb 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
@@ -27,7 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Queue;
import javax.annotation.Nullable;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.startree.v2.builder.StarTreeV2BuilderConfig;
import org.apache.pinot.segment.spi.SegmentMetadata;
@@ -262,7 +262,7 @@ public class StarTreeBuilderUtils {
// Remove the star-tree metadata
PropertiesConfiguration metadataProperties = SegmentMetadataUtils.getPropertiesConfiguration(indexDir);
metadataProperties.subset(StarTreeV2Constants.MetadataKey.STAR_TREE_SUBSET).clear();
- SegmentMetadataUtils.savePropertiesConfiguration(metadataProperties);
+ SegmentMetadataUtils.savePropertiesConfiguration(metadataProperties, indexDir);
// Remove the index file and index map file
File segmentDirectory = SegmentDirectoryPaths.findSegmentDirectory(indexDir);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/BaseSingleTreeBuilder.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/BaseSingleTreeBuilder.java
index 8fde477e18..c2e508d366 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/BaseSingleTreeBuilder.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/BaseSingleTreeBuilder.java
@@ -31,7 +31,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import javax.annotation.Nullable;
-import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration2.Configuration;
import org.apache.pinot.segment.local.aggregator.ValueAggregator;
import org.apache.pinot.segment.local.aggregator.ValueAggregatorFactory;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.SingleValueFixedByteRawIndexCreator;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
index 4941843e80..ee060d946b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
@@ -27,8 +27,8 @@ import java.util.ArrayList;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
@@ -41,7 +41,6 @@ import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants.MetadataKey;
import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
-import org.apache.pinot.segment.spi.utils.SegmentMetadataUtils;
import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
import org.apache.pinot.spi.utils.ReadMode;
@@ -85,7 +84,8 @@ public class MultipleTreesBuilder implements Closeable {
_buildMode = buildMode;
_segmentDirectory = SegmentDirectoryPaths.findSegmentDirectory(indexDir);
_metadataProperties =
- CommonsConfigurationUtils.fromFile(new File(_segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME));
+ CommonsConfigurationUtils.fromFile(
+ new File(_segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME));
_separator = getSeparator();
_segment = ImmutableSegmentLoader.load(indexDir, ReadMode.mmap);
}
@@ -106,7 +106,8 @@ public class MultipleTreesBuilder implements Closeable {
_buildMode = buildMode;
_segmentDirectory = SegmentDirectoryPaths.findSegmentDirectory(indexDir);
_metadataProperties =
- CommonsConfigurationUtils.fromFile(new File(_segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME));
+ CommonsConfigurationUtils.fromFile(
+ new File(_segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME));
Preconditions.checkState(!_metadataProperties.containsKey(MetadataKey.STAR_TREE_COUNT), "Star-tree already exists");
_segment = ImmutableSegmentLoader.load(indexDir, ReadMode.mmap);
try {
@@ -135,7 +136,8 @@ public class MultipleTreesBuilder implements Closeable {
new StarTreeIndexSeparator(new File(_separatorTempDir, StarTreeV2Constants.INDEX_MAP_FILE_NAME),
new File(_separatorTempDir, StarTreeV2Constants.INDEX_FILE_NAME), _metadataProperties);
_metadataProperties.subset(StarTreeV2Constants.MetadataKey.STAR_TREE_SUBSET).clear();
- SegmentMetadataUtils.savePropertiesConfiguration(_metadataProperties);
+ CommonsConfigurationUtils.saveToFile(_metadataProperties,
+ new File(_segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME));
return separator;
} catch (Exception e) {
try {
@@ -184,7 +186,8 @@ public class MultipleTreesBuilder implements Closeable {
}
// Save the metadata and index maps to the disk
- SegmentMetadataUtils.savePropertiesConfiguration(_metadataProperties);
+ CommonsConfigurationUtils.saveToFile(_metadataProperties,
+ new File(_segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME));
StarTreeIndexMapUtils.storeToFile(indexMaps,
new File(_segmentDirectory, StarTreeV2Constants.INDEX_MAP_FILE_NAME));
FileUtils.forceDelete(starTreeIndexDir);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
index 39ecf6f93b..49a72eedf6 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
@@ -29,7 +29,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
-import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration2.Configuration;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OnHeapSingleTreeBuilder.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OnHeapSingleTreeBuilder.java
index a36a1f2cd0..e3d093eef0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OnHeapSingleTreeBuilder.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OnHeapSingleTreeBuilder.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
-import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration2.Configuration;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparator.java
index 85fc038064..c017ea4b6a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparator.java
@@ -29,8 +29,9 @@ import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.startree.v2.store.StarTreeIndexMapUtils;
import org.apache.pinot.segment.spi.V1Constants;
@@ -62,7 +63,7 @@ public class StarTreeIndexSeparator implements Closeable {
int numStarTrees) {
try (InputStream inputStream = new FileInputStream(indexMapFile)) {
return StarTreeIndexMapUtils.loadFromInputStream(inputStream, numStarTrees);
- } catch (IOException e) {
+ } catch (IOException | ConfigurationException e) {
throw new RuntimeException(e);
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfig.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfig.java
index d1c327d668..cd02bba462 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfig.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfig.java
@@ -27,7 +27,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
-import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration2.Configuration;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.apache.pinot.segment.spi.AggregationFunctionType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
index 3b989d3c79..8cbf7d4958 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexMapUtils.java
@@ -29,7 +29,8 @@ import java.util.Map;
import java.util.Objects;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
@@ -147,7 +148,8 @@ public class StarTreeIndexMapUtils {
/**
* Stores the index maps for multiple star-trees into a file.
*/
- public static void storeToFile(List<List<Pair<IndexKey, IndexValue>>> indexMaps, File indexMapFile) {
+ public static void storeToFile(List<List<Pair<IndexKey, IndexValue>>> indexMaps, File indexMapFile)
+ throws ConfigurationException {
Preconditions.checkState(!indexMapFile.exists(), "Star-tree index map file already exists");
PropertiesConfiguration configuration = CommonsConfigurationUtils.fromFile(indexMapFile);
@@ -167,7 +169,8 @@ public class StarTreeIndexMapUtils {
/**
* Loads the index maps for multiple star-trees from an input stream.
*/
- public static List<Map<IndexKey, IndexValue>> loadFromInputStream(InputStream indexMapInputStream, int numStarTrees) {
+ public static List<Map<IndexKey, IndexValue>> loadFromInputStream(InputStream indexMapInputStream, int numStarTrees)
+ throws ConfigurationException {
List<Map<IndexKey, IndexValue>> indexMaps = new ArrayList<>(numStarTrees);
for (int i = 0; i < numStarTrees; i++) {
indexMaps.add(new HashMap<>());
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
index af8418ea47..1b7479ff6f 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
@@ -38,6 +38,7 @@ import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.env.CommonsConfigurationUtils;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.ReadMode;
@@ -66,13 +67,13 @@ public class SegmentColumnarIndexCreatorTest {
@Test
public void testRemoveColumnMetadataInfo()
throws Exception {
- PropertiesConfiguration configuration = new PropertiesConfiguration(CONFIG_FILE);
+ PropertiesConfiguration configuration = CommonsConfigurationUtils.fromFile(CONFIG_FILE);
configuration.setProperty(COLUMN_PROPERTY_KEY_PREFIX + "a", "foo");
configuration.setProperty(COLUMN_PROPERTY_KEY_PREFIX + "b", "bar");
configuration.setProperty(COLUMN_PROPERTY_KEY_PREFIX + "c", "foobar");
- configuration.save();
+ CommonsConfigurationUtils.saveToFile(configuration, CONFIG_FILE);
- configuration = new PropertiesConfiguration(CONFIG_FILE);
+ configuration = CommonsConfigurationUtils.fromFile(CONFIG_FILE);
assertTrue(configuration.containsKey(COLUMN_PROPERTY_KEY_PREFIX + "a"));
assertTrue(configuration.containsKey(COLUMN_PROPERTY_KEY_PREFIX + "b"));
assertTrue(configuration.containsKey(COLUMN_PROPERTY_KEY_PREFIX + "c"));
@@ -80,9 +81,9 @@ public class SegmentColumnarIndexCreatorTest {
assertFalse(configuration.containsKey(COLUMN_PROPERTY_KEY_PREFIX + "a"));
assertFalse(configuration.containsKey(COLUMN_PROPERTY_KEY_PREFIX + "b"));
assertFalse(configuration.containsKey(COLUMN_PROPERTY_KEY_PREFIX + "c"));
- configuration.save();
+ CommonsConfigurationUtils.saveToFile(configuration, CONFIG_FILE);
- configuration = new PropertiesConfiguration(CONFIG_FILE);
+ configuration = CommonsConfigurationUtils.fromFile(CONFIG_FILE);
assertFalse(configuration.containsKey(COLUMN_PROPERTY_KEY_PREFIX + "a"));
assertFalse(configuration.containsKey(COLUMN_PROPERTY_KEY_PREFIX + "b"));
assertFalse(configuration.containsKey(COLUMN_PROPERTY_KEY_PREFIX + "c"));
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/SegmentMetadataImplTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/SegmentMetadataImplTest.java
index 447476acc4..b8a02dc7ed 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/SegmentMetadataImplTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/SegmentMetadataImplTest.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
@@ -69,7 +70,7 @@ public class SegmentMetadataImplTest {
@Test
public void testToJson()
- throws IOException {
+ throws IOException, ConfigurationException {
SegmentMetadataImpl metadata = new SegmentMetadataImpl(_segmentDirectory);
Assert.assertNotNull(metadata);
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
index 6077ca1625..53f6995a57 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
@@ -33,6 +33,7 @@ import java.util.Random;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
@@ -2363,7 +2364,7 @@ public class ForwardIndexHandlerTest {
}
private void validateIndexesForForwardIndexDisabledColumns(String columnName)
- throws IOException {
+ throws IOException, ConfigurationException {
// Setup
SegmentMetadataImpl existingSegmentMetadata = new SegmentMetadataImpl(_segmentDirectory);
SegmentDirectory segmentLocalFSDirectory =
@@ -2387,7 +2388,7 @@ public class ForwardIndexHandlerTest {
private void validateForwardIndex(String columnName, @Nullable CompressionCodec expectedCompressionType,
boolean isSorted)
- throws IOException {
+ throws IOException, ConfigurationException {
// Setup
SegmentMetadataImpl existingSegmentMetadata = new SegmentMetadataImpl(_segmentDirectory);
SegmentDirectory segmentLocalFSDirectory =
@@ -2567,7 +2568,7 @@ public class ForwardIndexHandlerTest {
}
private void validateIndexMap(String columnName, boolean dictionaryEnabled, boolean forwardIndexDisabled)
- throws IOException {
+ throws IOException, ConfigurationException {
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_segmentDirectory);
// Panic validation to make sure all columns have only one forward index entry in index map.
@@ -2597,7 +2598,7 @@ public class ForwardIndexHandlerTest {
int cardinality, int totalDocs, FieldSpec.DataType dataType, FieldSpec.FieldType fieldType, boolean isSorted,
boolean isSingleValue, int maxNumberOfMVEntries, int totalNumberOfEntries, boolean isAutoGenerated,
Comparable minValue, Comparable maxValue, boolean isRegeneratedMVColumnWithDuplicates)
- throws IOException {
+ throws IOException, ConfigurationException {
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_segmentDirectory);
ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column);
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
index 3cc14682bc..ef25c785e0 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
@@ -34,7 +34,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
@@ -1895,18 +1896,23 @@ public class SegmentPreProcessorTest {
return driver.getOutputDirectory();
}
- private static void removeMinMaxValuesFromMetadataFile(File indexDir) {
+ private static void removeMinMaxValuesFromMetadataFile(File indexDir)
+ throws ConfigurationException {
PropertiesConfiguration configuration = SegmentMetadataUtils.getPropertiesConfiguration(indexDir);
Iterator<String> keys = configuration.getKeys();
+ List<String> keysToClear = new ArrayList<>();
while (keys.hasNext()) {
String key = keys.next();
if (key.endsWith(V1Constants.MetadataKeys.Column.MIN_VALUE) || key.endsWith(
V1Constants.MetadataKeys.Column.MAX_VALUE) || key.endsWith(
V1Constants.MetadataKeys.Column.MIN_MAX_VALUE_INVALID)) {
- configuration.clearProperty(key);
+ keysToClear.add(key);
}
}
- SegmentMetadataUtils.savePropertiesConfiguration(configuration);
+ for (String key: keysToClear) {
+ configuration.clearProperty(key);
+ }
+ SegmentMetadataUtils.savePropertiesConfiguration(configuration, indexDir);
}
private static Map<String, Consumer<IndexLoadingConfig>> createConfigPrepFunctions() {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
index 7943edcde7..2a03044abe 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
@@ -32,7 +32,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
import java.util.stream.Collectors;
-import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
import org.apache.pinot.segment.local.segment.index.readers.text.LuceneTextIndexReader;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/StarTreeIndexReaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/StarTreeIndexReaderTest.java
index 93a8e7428e..02278354e2 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/StarTreeIndexReaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/StarTreeIndexReaderTest.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.segment.local.startree.v2.store.StarTreeIndexMapUtils;
@@ -75,7 +76,7 @@ public class StarTreeIndexReaderTest {
@Test
public void testLoadStarTreeIndexBuffers()
- throws IOException {
+ throws IOException, ConfigurationException {
// Test with 2 index trees.
StarTreeV2Metadata stMeta1 = mock(StarTreeV2Metadata.class);
when(stMeta1.getDimensionsSplitOrder()).thenReturn(Arrays.asList("dim0", "dim1"));
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparatorTest.java
index b222c9273a..b830493b8d 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexSeparatorTest.java
@@ -25,7 +25,8 @@ import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
@@ -59,7 +60,7 @@ public class StarTreeIndexSeparatorTest {
@BeforeClass
public void setup()
- throws IOException {
+ throws IOException, ConfigurationException {
ClassLoader classLoader = getClass().getClassLoader();
URL segmentUrl = classLoader.getResource(SEGMENT_PATH);
File segmentDir = new File(segmentUrl.getFile());
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/SegmentCreator.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/SegmentCreator.java
index ecc3f68ab3..9adda03b72 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/SegmentCreator.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/SegmentCreator.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.io.Serializable;
import java.util.TreeMap;
import javax.annotation.Nullable;
-import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.index.creator.SegmentIndexCreationInfo;
import org.apache.pinot.spi.data.Schema;
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
index 35b19c9d96..51c1d11642 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
@@ -28,8 +28,8 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.V1Constants;
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
index 9117bb9d01..b0bb5d04f0 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
@@ -42,8 +42,9 @@ import java.util.TimeZone;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.V1Constants;
@@ -102,7 +103,7 @@ public class SegmentMetadataImpl implements SegmentMetadata {
* For segments that can only provide the inputstream to the metadata
*/
public SegmentMetadataImpl(InputStream metadataPropertiesInputStream, InputStream creationMetaInputStream)
- throws IOException {
+ throws IOException, ConfigurationException {
_indexDir = null;
_columnMetadataMap = new TreeMap<>();
_schema = new Schema();
@@ -122,7 +123,7 @@ public class SegmentMetadataImpl implements SegmentMetadata {
* <p>If segment metadata file exists in multiple segment version, load the one in highest segment version.
*/
public SegmentMetadataImpl(File indexDir)
- throws IOException {
+ throws IOException, ConfigurationException {
_indexDir = indexDir;
_columnMetadataMap = new TreeMap<>();
_schema = new Schema();
@@ -199,7 +200,8 @@ public class SegmentMetadataImpl implements SegmentMetadata {
}
}
- private void init(PropertiesConfiguration segmentMetadataPropertiesConfiguration) {
+ private void init(PropertiesConfiguration segmentMetadataPropertiesConfiguration)
+ throws ConfigurationException {
if (segmentMetadataPropertiesConfiguration.containsKey(Segment.SEGMENT_CREATOR_VERSION)) {
_creatorName = segmentMetadataPropertiesConfiguration.getString(Segment.SEGMENT_CREATOR_VERSION);
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/StarTreeV2Metadata.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/StarTreeV2Metadata.java
index 81ba74cb9a..4e53ad3763 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/StarTreeV2Metadata.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/StarTreeV2Metadata.java
@@ -23,7 +23,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeMap;
-import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration2.Configuration;
import org.apache.pinot.segment.spi.AggregationFunctionType;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants.MetadataKey;
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectory.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectory.java
index 5f95f5c094..331ca1c59d 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectory.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectory.java
@@ -25,7 +25,7 @@ import java.net.URI;
import java.nio.file.Path;
import java.util.Set;
import javax.annotation.Nullable;
-import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.pinot.segment.spi.FetchContext;
import org.apache.pinot.segment.spi.index.IndexType;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/utils/SegmentMetadataUtils.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/utils/SegmentMetadataUtils.java
index 6d81588bb3..6658895537 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/utils/SegmentMetadataUtils.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/utils/SegmentMetadataUtils.java
@@ -21,7 +21,8 @@ package org.apache.pinot.segment.spi.utils;
import com.google.common.base.Preconditions;
import java.io.File;
import java.util.Map;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
@@ -32,22 +33,24 @@ public class SegmentMetadataUtils {
private SegmentMetadataUtils() {
}
- public static PropertiesConfiguration getPropertiesConfiguration(File indexDir) {
+ public static PropertiesConfiguration getPropertiesConfiguration(File indexDir)
+ throws ConfigurationException {
File metadataFile = SegmentDirectoryPaths.findMetadataFile(indexDir);
Preconditions.checkNotNull(metadataFile, "Cannot find segment metadata file under directory: %s", indexDir);
return CommonsConfigurationUtils.fromFile(metadataFile);
}
- public static PropertiesConfiguration getPropertiesConfiguration(SegmentMetadata segmentMetadata) {
+ public static PropertiesConfiguration getPropertiesConfiguration(SegmentMetadata segmentMetadata)
+ throws ConfigurationException {
File indexDir = segmentMetadata.getIndexDir();
Preconditions.checkState(indexDir != null, "Cannot get PropertiesConfiguration from in-memory segment: %s",
segmentMetadata.getName());
return getPropertiesConfiguration(indexDir);
}
- public static void savePropertiesConfiguration(PropertiesConfiguration propertiesConfiguration) {
- File metadataFile = propertiesConfiguration.getFile();
- Preconditions.checkState(metadataFile != null, "Cannot save PropertiesConfiguration not loaded from file");
+ public static void savePropertiesConfiguration(PropertiesConfiguration propertiesConfiguration, File indexDir) {
+ File metadataFile = SegmentDirectoryPaths.findMetadataFile(indexDir);
+ Preconditions.checkState(metadataFile != null, "Cannot find segment metadata file under directory: %s", indexDir);
CommonsConfigurationUtils.saveToFile(propertiesConfiguration, metadataFile);
}
@@ -59,7 +62,7 @@ public class SegmentMetadataUtils {
for (Map.Entry<String, String> entry : metadataProperties.entrySet()) {
propertiesConfiguration.setProperty(entry.getKey(), entry.getValue());
}
- SegmentMetadataUtils.savePropertiesConfiguration(propertiesConfiguration);
+ savePropertiesConfiguration(propertiesConfiguration, segmentMetadata.getIndexDir());
segmentDirectory.reloadMetadata();
return segmentDirectory.getSegmentMetadata();
}
diff --git a/pinot-server/pom.xml b/pinot-server/pom.xml
index a0254362c9..eb275ccd2c 100644
--- a/pinot-server/pom.xml
+++ b/pinot-server/pom.xml
@@ -69,20 +69,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
index 480c392db8..0d194c51ae 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
@@ -38,7 +38,7 @@ import java.util.concurrent.locks.Lock;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;
-import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.helix.HelixDataAccessor;
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
index 84d72ad8b1..9c69e8afd4 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
@@ -19,7 +19,7 @@
package org.apache.pinot.server.starter.helix;
import java.util.Optional;
-import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.spi.config.instance.InstanceDataManagerConfig;
diff --git a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
index 488350aa17..0469d805d2 100644
--- a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
+++ b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.ws.rs.core.Response;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.response.server.TableIndexMetadataResponse;
import org.apache.pinot.common.restlet.resources.TableMetadataInfo;
@@ -312,7 +313,7 @@ public class TablesResourceTest extends BaseResourceTest {
// Verify metadata file from segments.
private void downLoadAndVerifySegmentContent(String tableNameWithType, IndexSegment segment)
- throws IOException {
+ throws IOException, ConfigurationException {
String segmentPath = "/segments/" + tableNameWithType + "/" + segment.getSegmentName();
// Download the segment and save to a temp local file.
diff --git a/pinot-spi/pom.xml b/pinot-spi/pom.xml
index 5c8b23fe1f..8fbad57d1f 100644
--- a/pinot-spi/pom.xml
+++ b/pinot-spi/pom.xml
@@ -48,20 +48,6 @@
</plugins>
</build>
<dependencies>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
@@ -144,6 +130,10 @@
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ </dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/env/CommonsConfigurationUtils.java b/pinot-spi/src/main/java/org/apache/pinot/spi/env/CommonsConfigurationUtils.java
index a7da9f9dd7..ae79fe9d90 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/env/CommonsConfigurationUtils.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/env/CommonsConfigurationUtils.java
@@ -18,10 +18,8 @@
*/
package org.apache.pinot.spi.env;
+import com.google.common.base.Preconditions;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Iterator;
@@ -33,9 +31,10 @@ import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.apache.commons.configuration2.Configuration;
import org.apache.commons.configuration2.PropertiesConfiguration;
-import org.apache.commons.configuration2.convert.DefaultListDelimiterHandler;
+import org.apache.commons.configuration2.convert.LegacyListDelimiterHandler;
import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.configuration2.io.FileHandler;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
@@ -47,19 +46,44 @@ public class CommonsConfigurationUtils {
private CommonsConfigurationUtils() {
}
- public static void setListDelimiterHandler(PropertiesConfiguration configuration, Character delimiter) {
- configuration.setListDelimiterHandler(new DefaultListDelimiterHandler(delimiter));
+ /**
+ * Instantiate a {@link PropertiesConfiguration} from a {@link File}.
+ * @param file containing properties
+ * @return a {@link PropertiesConfiguration} instance. Empty if file does not exist.
+ */
+ public static PropertiesConfiguration fromFile(File file)
+ throws ConfigurationException {
+ return fromFile(file, false, true);
}
- public static void setDefaultListDelimiterHandler(PropertiesConfiguration configuration) {
- setListDelimiterHandler(configuration, DEFAULT_LIST_DELIMITER);
+ /**
+ * Instantiate a {@link PropertiesConfiguration} from an {@link InputStream}.
+ * @param stream containing properties
+ * @return a {@link PropertiesConfiguration} instance.
+ */
+ public static PropertiesConfiguration fromInputStream(InputStream stream)
+ throws ConfigurationException {
+ return fromInputStream(stream, false, true);
}
- public static PropertiesConfiguration loadFromPath(String path) throws ConfigurationException {
- return loadFromPath(path, false, false);
+ /**
+ * Instantiate a {@link PropertiesConfiguration} from an {@link String}.
+ * @param path representing the path of file
+ * @return a {@link PropertiesConfiguration} instance.
+ */
+ public static PropertiesConfiguration fromPath(String path)
+ throws ConfigurationException {
+ return fromPath(path, false, true);
}
- public static PropertiesConfiguration loadFromPath(String path, boolean setIOFactory, boolean setDefaultDelimiter)
+ /**
+ * Instantiate a {@link PropertiesConfiguration} from an {@link String}.
+ * @param path representing the path of file
+ * @param setIOFactory representing to set the IOFactory or not
+ * @param setDefaultDelimiter representing to set the default list delimiter.
+ * @return a {@link PropertiesConfiguration} instance.
+ */
+ public static PropertiesConfiguration fromPath(String path, boolean setIOFactory, boolean setDefaultDelimiter)
throws ConfigurationException {
PropertiesConfiguration config = createPropertiesConfiguration(setIOFactory, setDefaultDelimiter);
FileHandler fileHandler = new FileHandler(config);
@@ -67,11 +91,14 @@ public class CommonsConfigurationUtils {
return config;
}
- public static PropertiesConfiguration loadFromInputStream(InputStream stream) throws ConfigurationException {
- return loadFromInputStream(stream, false, false);
- }
-
- public static PropertiesConfiguration loadFromInputStream(InputStream stream, boolean setIOFactory,
+ /**
+ * Instantiate a {@link PropertiesConfiguration} from an {@link InputStream}.
+ * @param stream containing properties
+ * @param setIOFactory representing to set the IOFactory or not
+ * @param setDefaultDelimiter representing to set the default list delimiter.
+ * @return a {@link PropertiesConfiguration} instance.
+ */
+ public static PropertiesConfiguration fromInputStream(InputStream stream, boolean setIOFactory,
boolean setDefaultDelimiter) throws ConfigurationException {
PropertiesConfiguration config = createPropertiesConfiguration(setIOFactory, setDefaultDelimiter);
FileHandler fileHandler = new FileHandler(config);
@@ -79,68 +106,38 @@ public class CommonsConfigurationUtils {
return config;
}
- public static PropertiesConfiguration loadFromFile(File file) throws ConfigurationException {
- return loadFromFile(file, false, false);
- }
-
- public static PropertiesConfiguration loadFromFile(File file, boolean setIOFactory,
- boolean setDefaultDelimiter) throws ConfigurationException {
- PropertiesConfiguration config = createPropertiesConfiguration(setIOFactory, setDefaultDelimiter);
- FileHandler fileHandler = new FileHandler(config);
- fileHandler.load(file);
- return config;
- }
-
/**
- * Instantiate a {@link org.apache.commons.configuration.PropertiesConfiguration} from a {@link File}.
+ * Instantiate a {@link PropertiesConfiguration} from a {@link File}.
* @param file containing properties
- * @return a {@link org.apache.commons.configuration.PropertiesConfiguration} instance. Empty if file does not exist.
+ * @param setIOFactory representing to set the IOFactory or not
+ * @param setDefaultDelimiter representing to set the default list delimiter.
+ * @return a {@link PropertiesConfiguration} instance.
*/
- @Deprecated
- public static org.apache.commons.configuration.PropertiesConfiguration fromFile(File file) {
- try {
- org.apache.commons.configuration.PropertiesConfiguration propertiesConfiguration =
- new org.apache.commons.configuration.PropertiesConfiguration();
-
- // Commons Configuration 1.10 does not support file path containing '%'.
- // Explicitly providing the input stream on load bypasses the problem.
- propertiesConfiguration.setFile(file);
- if (file.exists()) {
- try (InputStream in = new FileInputStream(file)) {
- propertiesConfiguration.load(in);
- }
- }
- return propertiesConfiguration;
- } catch (org.apache.commons.configuration.ConfigurationException | IOException e) {
- throw new RuntimeException(e);
+ public static PropertiesConfiguration fromFile(File file, boolean setIOFactory,
+ boolean setDefaultDelimiter) throws ConfigurationException {
+ PropertiesConfiguration config = createPropertiesConfiguration(setIOFactory, setDefaultDelimiter);
+ FileHandler fileHandler = new FileHandler(config);
+ // check if file exists, load the properties otherwise set the file.
+ if (file.exists()) {
+ fileHandler.load(file);
+ } else {
+ fileHandler.setFile(file);
}
+ return config;
}
/**
- * Instantiate a {@link org.apache.commons.configuration.PropertiesConfiguration} from an inputstream.
- * @param inputStream containing properties
- * @return a {@link org.apache.commons.configuration.PropertiesConfiguration} instance.
+ * Save the propertiesConfiguration content into the provided file.
+ * @param propertiesConfiguration a {@link PropertiesConfiguration} instance.
+ * @param file a {@link File} instance.
*/
- @Deprecated
- public static org.apache.commons.configuration.PropertiesConfiguration fromInputStream(InputStream inputStream) {
+ public static void saveToFile(PropertiesConfiguration propertiesConfiguration, File file) {
+ Preconditions.checkNotNull(file, "File object can not be null for saving configurations");
+ FileHandler fileHandler = new FileHandler(propertiesConfiguration);
+ fileHandler.setFile(file);
try {
- org.apache.commons.configuration.PropertiesConfiguration propertiesConfiguration =
- new org.apache.commons.configuration.PropertiesConfiguration();
- propertiesConfiguration.load(inputStream);
- return propertiesConfiguration;
- } catch (org.apache.commons.configuration.ConfigurationException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Deprecated
- public static void saveToFile(org.apache.commons.configuration.PropertiesConfiguration propertiesConfiguration,
- File file) {
- // Commons Configuration 1.10 does not support file path containing '%'.
- // Explicitly providing the output stream for save bypasses the problem.
- try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
- propertiesConfiguration.save(fileOutputStream);
- } catch (org.apache.commons.configuration.ConfigurationException | IOException e) {
+ fileHandler.save();
+ } catch (ConfigurationException e) {
throw new RuntimeException(e);
}
}
@@ -149,62 +146,18 @@ public class CommonsConfigurationUtils {
return () -> keys;
}
- /**
- * Provides a stream of all the keys found in a {@link Configuration}.
- * @param configuration to iterate on keys
- * @return a stream of keys
- */
- @Deprecated
- public static Stream<String> getKeysStream(org.apache.commons.configuration.Configuration configuration) {
- return StreamSupport.stream(getIterable(configuration.getKeys()).spliterator(), false);
- }
-
public static Stream<String> getKeysStream(Configuration configuration) {
return StreamSupport.stream(getIterable(configuration.getKeys()).spliterator(), false);
}
-
- /**
- * Provides a list of all the keys found in a {@link Configuration}.
- * @param configuration to iterate on keys
- * @return a list of keys
- */
- @Deprecated
- public static List<String> getKeys(org.apache.commons.configuration.Configuration configuration) {
- return getKeysStream(configuration).collect(Collectors.toList());
- }
-
public static List<String> getKeys(Configuration configuration) {
return getKeysStream(configuration).collect(Collectors.toList());
}
- /**
- * @return a key-value {@link Map} found in the provided {@link Configuration}
- */
- @Deprecated
- public static Map<String, Object> toMap(org.apache.commons.configuration.Configuration configuration) {
- return getKeysStream(configuration).collect(Collectors.toMap(key -> key, key -> mapValue(key, configuration)));
- }
-
public static Map<String, Object> toMap(Configuration configuration) {
return getKeysStream(configuration).collect(Collectors.toMap(key -> key, key -> mapValue(key, configuration)));
}
- @Deprecated
- private static Object mapValue(String key, org.apache.commons.configuration.Configuration configuration) {
- // For multi-value config, convert it to a single comma connected string value.
- // For single-value config, return its raw property, unless it needs interpolation.
- return Optional.of(configuration.getStringArray(key)).filter(values -> values.length > 1)
- .<Object>map(values -> Arrays.stream(values).collect(Collectors.joining(","))).orElseGet(() -> {
- Object rawProperty = configuration.getProperty(key);
- if (!needInterpolate(rawProperty)) {
- return rawProperty;
- }
- // The string value is converted to the requested type when accessing it via PinotConfiguration.
- return configuration.getString(key);
- });
- }
-
private static Object mapValue(String key, Configuration configuration) {
// For multi-value config, convert it to a single comma connected string value.
// For single-value config, return its raw property, unless it needs interpolation.
@@ -229,24 +182,6 @@ public class CommonsConfigurationUtils {
return false;
}
- @SuppressWarnings("unchecked")
- @Deprecated
- public static <T> T interpolate(org.apache.commons.configuration.Configuration configuration,
- String key, T defaultValue, Class<T> returnType) {
- // Different from the generic getProperty() method, those type specific getters do config interpolation.
- if (Integer.class.equals(returnType)) {
- return (T) configuration.getInteger(key, (Integer) defaultValue);
- } else if (Boolean.class.equals(returnType)) {
- return (T) configuration.getBoolean(key, (Boolean) defaultValue);
- } else if (Long.class.equals(returnType)) {
- return (T) configuration.getLong(key, (Long) defaultValue);
- } else if (Double.class.equals(returnType)) {
- return (T) configuration.getDouble(key, (Double) defaultValue);
- } else {
- throw new IllegalArgumentException(returnType + " is not a supported type for conversion.");
- }
- }
-
@SuppressWarnings("unchecked")
public static <T> T interpolate(Configuration configuration, String key, T defaultValue, Class<T> returnType) {
// Different from the generic getProperty() method, those type specific getters do config interpolation.
@@ -288,6 +223,7 @@ public class CommonsConfigurationUtils {
* - Escaping comma with backslash doesn't work when comma is preceded by a backslash
*/
public static String replaceSpecialCharacterInPropertyValue(String value) {
+ value = StringEscapeUtils.escapeJava(value);
if (value.isEmpty()) {
return value;
}
@@ -305,6 +241,7 @@ public class CommonsConfigurationUtils {
* {@link #replaceSpecialCharacterInPropertyValue(String)}.
*/
public static String recoverSpecialCharacterInPropertyValue(String value) {
+ value = StringEscapeUtils.unescapeJava(value);
if (value.isEmpty()) {
return value;
}
@@ -328,8 +265,13 @@ public class CommonsConfigurationUtils {
// setting DEFAULT_LIST_DELIMITER
if (setDefaultDelimiter) {
- CommonsConfigurationUtils.setDefaultListDelimiterHandler(config);
+ setListDelimiterHandler(config);
}
+
return config;
}
+
+ private static void setListDelimiterHandler(PropertiesConfiguration configuration) {
+ configuration.setListDelimiterHandler(new LegacyListDelimiterHandler(DEFAULT_LIST_DELIMITER));
+ }
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
index 49e8b381e8..6b7e9089eb 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
@@ -30,7 +30,7 @@ import org.apache.commons.configuration2.CompositeConfiguration;
import org.apache.commons.configuration2.Configuration;
import org.apache.commons.configuration2.MapConfiguration;
import org.apache.commons.configuration2.PropertiesConfiguration;
-import org.apache.commons.configuration2.convert.DefaultListDelimiterHandler;
+import org.apache.commons.configuration2.convert.LegacyListDelimiterHandler;
import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.pinot.spi.ingestion.batch.spec.PinotFSSpec;
import org.apache.pinot.spi.utils.Obfuscator;
@@ -165,7 +165,7 @@ public class PinotConfiguration {
return Stream.concat(Stream.of(relaxedBaseProperties, relaxedEnvVariables).map(e -> {
MapConfiguration mapConfiguration = new MapConfiguration(e);
- mapConfiguration.setListDelimiterHandler(new DefaultListDelimiterHandler(','));
+ mapConfiguration.setListDelimiterHandler(new LegacyListDelimiterHandler(','));
return mapConfiguration;
}), propertiesFromConfigPaths).collect(Collectors.toList());
}
@@ -186,11 +186,11 @@ public class PinotConfiguration {
try {
PropertiesConfiguration propertiesConfiguration;
if (configPath.startsWith("classpath:")) {
- propertiesConfiguration = CommonsConfigurationUtils.loadFromInputStream(
+ propertiesConfiguration = CommonsConfigurationUtils.fromInputStream(
PinotConfiguration.class.getResourceAsStream(configPath.substring("classpath:".length())),
true, true);
} else {
- propertiesConfiguration = CommonsConfigurationUtils.loadFromPath(configPath, true, true);
+ propertiesConfiguration = CommonsConfigurationUtils.fromPath(configPath, true, true);
}
return propertiesConfiguration;
} catch (ConfigurationException e) {
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/env/CommonsConfigurationUtilsTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/env/CommonsConfigurationUtilsTest.java
index 83bfd71e41..9d31a7926f 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/env/CommonsConfigurationUtilsTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/env/CommonsConfigurationUtilsTest.java
@@ -20,8 +20,8 @@ package org.apache.pinot.spi.env;
import java.io.File;
import java.io.IOException;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
@@ -36,7 +36,8 @@ public class CommonsConfigurationUtilsTest {
private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(), "CommonsConfigurationUtilsTest");
private static final File CONFIG_FILE = new File(TEMP_DIR, "config");
private static final String PROPERTY_KEY = "testKey";
- private static final int NUM_ROUNDS = 1000;
+ private static final String LIST_PROPERTY_KEY = "listTestKey";
+ private static final int NUM_ROUNDS = 10000;
@BeforeClass
public void setUp()
@@ -94,13 +95,15 @@ public class CommonsConfigurationUtilsTest {
private void testPropertyValueWithSpecialCharacters(String value)
throws ConfigurationException {
String replacedValue = CommonsConfigurationUtils.replaceSpecialCharacterInPropertyValue(value);
- PropertiesConfiguration configuration = new PropertiesConfiguration(CONFIG_FILE);
+
+ PropertiesConfiguration configuration = CommonsConfigurationUtils.fromFile(CONFIG_FILE, false, true);
configuration.setProperty(PROPERTY_KEY, replacedValue);
String recoveredValue = CommonsConfigurationUtils.recoverSpecialCharacterInPropertyValue(
(String) configuration.getProperty(PROPERTY_KEY));
assertEquals(recoveredValue, value);
- configuration.save();
- configuration = new PropertiesConfiguration(CONFIG_FILE);
+ CommonsConfigurationUtils.saveToFile(configuration, CONFIG_FILE);
+ configuration = CommonsConfigurationUtils.fromFile(CONFIG_FILE, false, true);
+
recoveredValue = CommonsConfigurationUtils.recoverSpecialCharacterInPropertyValue(
(String) configuration.getProperty(PROPERTY_KEY));
assertEquals(recoveredValue, value);
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/env/PinotConfigurationTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/env/PinotConfigurationTest.java
index eb686e9e67..8ae846099c 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/env/PinotConfigurationTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/env/PinotConfigurationTest.java
@@ -179,7 +179,7 @@ public class PinotConfigurationTest {
@Test
public void assertPropertiesFromBaseConfiguration()
throws ConfigurationException {
- PropertiesConfiguration propertiesConfiguration = CommonsConfigurationUtils.loadFromPath(
+ PropertiesConfiguration propertiesConfiguration = CommonsConfigurationUtils.fromPath(
PropertiesConfiguration.class.getClassLoader().getResource("pinot-configuration-1.properties").getFile(),
true, true);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/filesystem/PinotFSBenchmarkDriver.java b/pinot-tools/src/main/java/org/apache/pinot/tools/filesystem/PinotFSBenchmarkDriver.java
index 14f998a798..e444b6d0d0 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/filesystem/PinotFSBenchmarkDriver.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/filesystem/PinotFSBenchmarkDriver.java
@@ -54,7 +54,7 @@ public class PinotFSBenchmarkDriver {
Integer numSegmentsForListFilesTest, Integer dataSizeInMBsForCopyTest, Integer numOps)
throws ConfigurationException {
PropertiesConfiguration configuration =
- CommonsConfigurationUtils.loadFromFile(new File(configFilePath));
+ CommonsConfigurationUtils.fromFile(new File(configFilePath));
PinotFSFactory.init(new PinotConfiguration(configuration));
_mode = mode;
_baseDirectoryUri = URI.create(baseDirectoryUri);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
index d29c62cd34..7be99a0392 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
@@ -26,8 +26,8 @@ import java.util.Arrays;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
@@ -43,6 +43,7 @@ import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.env.CommonsConfigurationUtils;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.slf4j.Logger;
@@ -248,13 +249,11 @@ public class DictionaryToRawIndexConverter {
* @param segmentDir Segment directory
* @param columns Converted columns
* @param tableName New table name to be written in the meta-data. Skipped if null.
- * @throws IOException
- * @throws ConfigurationException
*/
private void updateMetadata(File segmentDir, String[] columns, String tableName)
- throws IOException, ConfigurationException {
+ throws ConfigurationException {
File metadataFile = new File(segmentDir, V1Constants.MetadataKeys.METADATA_FILE_NAME);
- PropertiesConfiguration properties = new PropertiesConfiguration(metadataFile);
+ PropertiesConfiguration properties = CommonsConfigurationUtils.fromFile(metadataFile);
if (tableName != null) {
properties
@@ -267,7 +266,7 @@ public class DictionaryToRawIndexConverter {
properties.setProperty(
V1Constants.MetadataKeys.Column.getKeyFor(column, V1Constants.MetadataKeys.Column.BITS_PER_ELEMENT), -1);
}
- properties.save();
+ CommonsConfigurationUtils.saveToFile(properties, metadataFile);
}
/**
diff --git a/pom.xml b/pom.xml
index ea2bf105e3..c750249b33 100644
--- a/pom.xml
+++ b/pom.xml
@@ -473,19 +473,6 @@
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
</dependency>
-
- <!-- commons-configuration1.10, will be deleted after the upgrade. -->
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <version>1.10</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org