You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2018/06/08 11:40:50 UTC
[25/50] [abbrv] carbondata git commit: [CARBONDATA-2507]
enable.offheap.sort not validate in CarbonData
[CARBONDATA-2507] enable.offheap.sort not validate in CarbonData
In #2274, the value of enable.offheap.sort will transform to false when args[0] not equal to true, including false and other string, like f,any and so on.
So we should validate it.
This closes #2331
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d8bafa34
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d8bafa34
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d8bafa34
Branch: refs/heads/spark-2.3
Commit: d8bafa34dd4ce31b0596cabe139ea2022ba9d78b
Parents: 33b825d
Author: xubo245 <xu...@huawei.com>
Authored: Wed May 23 16:08:25 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Mon May 28 17:25:16 2018 +0800
----------------------------------------------------------------------
.../carbondata/core/util/CarbonProperties.java | 17 +++++++++++++
.../carbondata/core/util/SessionParams.java | 2 ++
.../core/CarbonPropertiesValidationTest.java | 25 ++++++++++++++++++++
3 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8bafa34/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 391096d..4ee5199 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -56,6 +56,7 @@ import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_
import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.CSV_READ_BUFFER_SIZE;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_AUTO_HANDOFF;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_OFFHEAP_SORT;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_UNSAFE_SORT;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_VECTOR_READER;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.HANDOFF_SIZE;
@@ -149,6 +150,9 @@ public final class CarbonProperties {
case ENABLE_UNSAFE_SORT:
validateEnableUnsafeSort();
break;
+ case ENABLE_OFFHEAP_SORT:
+ validateEnableOffHeapSort();
+ break;
case CARBON_CUSTOM_BLOCK_DISTRIBUTION:
validateCustomBlockDistribution();
break;
@@ -235,6 +239,7 @@ public final class CarbonProperties {
validateBlockletGroupSizeInMB();
validateNumberOfColumnPerIORead();
validateEnableUnsafeSort();
+ validateEnableOffHeapSort();
validateCustomBlockDistribution();
validateEnableVectorReader();
validateLockType();
@@ -471,6 +476,18 @@ public final class CarbonProperties {
}
}
+ private void validateEnableOffHeapSort() {
+ String value = carbonProperties.getProperty(ENABLE_OFFHEAP_SORT);
+ boolean isValidBooleanValue = CarbonUtil.validateBoolean(value);
+ if (!isValidBooleanValue) {
+ LOGGER.warn("The enable off heap sort value \"" + value
+ + "\" is invalid. Using the default value \""
+ + CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT);
+ carbonProperties.setProperty(ENABLE_OFFHEAP_SORT,
+ CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT);
+ }
+ }
+
private void initPropertySet() throws IllegalAccessException {
Field[] declaredFields = CarbonCommonConstants.class.getDeclaredFields();
for (Field field : declaredFields) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8bafa34/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
index 3823aef..169c003 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.exception.InvalidConfigurationException;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_SEARCH_MODE_ENABLE;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_OFFHEAP_SORT;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_UNSAFE_SORT;
import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION;
import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE;
@@ -145,6 +146,7 @@ public class SessionParams implements Serializable, Cloneable {
boolean isValid = false;
switch (key) {
case ENABLE_UNSAFE_SORT:
+ case ENABLE_OFFHEAP_SORT:
case CARBON_CUSTOM_BLOCK_DISTRIBUTION:
case CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE:
case CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8bafa34/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
index bbfe26c..7cc665e 100644
--- a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import junit.framework.TestCase;
+import org.junit.Assert;
import org.junit.Test;
/**
@@ -65,6 +66,30 @@ public class CarbonPropertiesValidationTest extends TestCase {
CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT.equalsIgnoreCase(valueAfterValidation));
}
+ @Test public void testValidateEnableOffHeapSort()
+ throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ Method validateMethodType =
+ carbonProperties.getClass().getDeclaredMethod("validateEnableOffHeapSort");
+ validateMethodType.setAccessible(true);
+ carbonProperties.addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "True");
+ assert (carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT)
+ .equalsIgnoreCase("true"));
+ carbonProperties.addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false");
+ assert (carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT)
+ .equalsIgnoreCase("false"));
+ carbonProperties.addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "xyz");
+ assert (carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT)
+ .equalsIgnoreCase("true"));
+ String valueBeforeValidation =
+ carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT);
+ validateMethodType.invoke(carbonProperties);
+ String valueAfterValidation =
+ carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT);
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(
+ CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT.equalsIgnoreCase(valueAfterValidation));
+ }
+
@Test public void testValidateCustomBlockDistribution()
throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
Method validateMethodType =