You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2018/01/05 05:03:34 UTC
carbondata git commit: [CARBONDATA-1929][Validation]carbon property
configuration validation + Fixed test case
Repository: carbondata
Updated Branches:
refs/heads/master 7a4bd2298 -> bcf3ca3fe
[CARBONDATA-1929][Validation]carbon property configuration validation + Fixed test case
Added validation for below parameter:
carbon.timestamp.format
carbon.date.format
carbon.sort.file.write.buffer.size (minValue = 10 KB, maxValue=10MB, defaultValue =16 KB )
carbon.sort.intermediate.files.limit (minValue = 2, maxValue=50, defaultValue =20 )
This closes #1718
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bcf3ca3f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bcf3ca3f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bcf3ca3f
Branch: refs/heads/master
Commit: bcf3ca3feda544dcbc1b5c096b98369c8a27f4e3
Parents: 7a4bd22
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Fri Dec 22 16:52:44 2017 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Fri Jan 5 10:36:32 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 6 +
.../carbondata/core/util/CarbonProperties.java | 318 ++++++++++++++-----
.../core/CarbonPropertiesValidationTest.java | 66 +++-
.../testsuite/datamap/DataMapWriterSuite.scala | 20 +-
4 files changed, 318 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bcf3ca3f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 2d1e4f9..b58f42a 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -297,6 +297,10 @@ public final class CarbonCommonConstants {
* SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE
*/
public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
+
+ public static final int SORT_INTERMEDIATE_FILES_LIMIT_MIN = 2;
+
+ public static final int SORT_INTERMEDIATE_FILES_LIMIT_MAX = 50;
/**
* BAD_RECORD_KEY_VALUE
*/
@@ -325,6 +329,8 @@ public final class CarbonCommonConstants {
@CarbonProperty
public static final String CARBON_SORT_FILE_WRITE_BUFFER_SIZE =
"carbon.sort.file.write.buffer.size";
+ public static final int CARBON_SORT_FILE_WRITE_BUFFER_SIZE_MIN = 10240;
+ public static final int CARBON_SORT_FILE_WRITE_BUFFER_SIZE_MAX = 10485760;
/**
* SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bcf3ca3f/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 8042cfa..a918611 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -22,6 +22,7 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -34,6 +35,27 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.BLOCKLET_SIZE;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_DATA_FILE_VERSION;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_DATE_FORMAT;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_EXECUTOR_STARTUP_TIMEOUT;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CSV_READ_BUFFER_SIZE;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_AUTO_HANDOFF;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_UNSAFE_SORT;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_VECTOR_READER;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.HANDOFF_SIZE;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.LOCK_TYPE;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.NUM_CORES;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.NUM_CORES_BLOCK_SORT;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.SORT_SIZE;
+import static org.apache.carbondata.core.constants.CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB;
+import static org.apache.carbondata.core.constants.CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO;
import org.apache.hadoop.conf.Configuration;
@@ -81,16 +103,87 @@ public final class CarbonProperties {
}
/**
+ * This method is to validate only a specific key added to carbonProperties using addProperty
+ *
+ * @param key
+ */
+ private void validateAndLoadDefaultProperties(String key) {
+ switch (key) {
+ case BLOCKLET_SIZE:
+ validateBlockletSize();
+ break;
+ case NUM_CORES:
+ validateNumCores();
+ break;
+ case NUM_CORES_BLOCK_SORT:
+ validateNumCoresBlockSort();
+ break;
+ case SORT_SIZE:
+ validateSortSize();
+ break;
+ case CARBON_DATA_FILE_VERSION:
+ validateCarbonDataFileVersion();
+ break;
+ case CARBON_EXECUTOR_STARTUP_TIMEOUT:
+ validateExecutorStartUpTime();
+ break;
+ case CARBON_PREFETCH_BUFFERSIZE:
+ validatePrefetchBufferSize();
+ break;
+ case BLOCKLET_SIZE_IN_MB:
+ validateBlockletGroupSizeInMB();
+ break;
+ case NUMBER_OF_COLUMN_TO_READ_IN_IO:
+ validateNumberOfColumnPerIORead();
+ break;
+ case ENABLE_UNSAFE_SORT:
+ validateEnableUnsafeSort();
+ break;
+ case CARBON_CUSTOM_BLOCK_DISTRIBUTION:
+ validateCustomBlockDistribution();
+ break;
+ case ENABLE_VECTOR_READER:
+ validateEnableVectorReader();
+ break;
+ case CSV_READ_BUFFER_SIZE:
+ validateCarbonCSVReadBufferSizeByte();
+ break;
+ case HANDOFF_SIZE:
+ validateHandoffSize();
+ break;
+ case CARBON_COMBINE_SMALL_INPUT_FILES:
+ validateCombineSmallInputFiles();
+ break;
+ // The method validate the validity of configured carbon.timestamp.format value
+ // and reset to default value if validation fail
+ case CARBON_TIMESTAMP_FORMAT:
+ validateTimeFormatKey(CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ break;
+ // The method validate the validity of configured carbon.date.format value
+ // and reset to default value if validation fail
+ case CARBON_DATE_FORMAT:
+ validateTimeFormatKey(CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
+ break;
+ case CARBON_SORT_FILE_WRITE_BUFFER_SIZE:
+ validateSortFileWriteBufferSize();
+ break;
+ case SORT_INTERMEDIATE_FILES_LIMIT:
+ validateSortIntermediateFilesLimit();
+ break;
+ case ENABLE_AUTO_HANDOFF:
+ validateHandoffSize();
+ break;
+ // TODO : Validation for carbon.lock.type should be handled for addProperty flow
+ default:
+ // none
+ }
+ }
+ /**
* This method validates the loaded properties and loads default
* values in case of wrong values.
*/
private void validateAndLoadDefaultProperties() {
- try {
- initPropertySet();
- } catch (IllegalAccessException e) {
- LOGGER.error("Illelagal access to declared field" + e.getMessage());
- }
-
validateBlockletSize();
validateNumCores();
validateNumCoresBlockSort();
@@ -107,40 +200,107 @@ public final class CarbonProperties {
validateCarbonCSVReadBufferSizeByte();
validateHandoffSize();
validateCombineSmallInputFiles();
+ // The method validate the validity of configured carbon.timestamp.format value
+ // and reset to default value if validation fail
+ validateTimeFormatKey(CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ // The method validate the validity of configured carbon.date.format value
+ // and reset to default value if validation fail
+ validateTimeFormatKey(CARBON_DATE_FORMAT,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
+ validateSortFileWriteBufferSize();
+ validateSortIntermediateFilesLimit();
validateEnableAutoHandoff();
}
- private void validateCarbonCSVReadBufferSizeByte() {
- String csvReadBufferSizeStr =
- carbonProperties.getProperty(CarbonCommonConstants.CSV_READ_BUFFER_SIZE);
- if (null != csvReadBufferSizeStr) {
+ /**
+ * Sort intermediate file size validation and if not valid then reset to the default value
+ */
+ private void validateSortIntermediateFilesLimit() {
+ validateRange(SORT_INTERMEDIATE_FILES_LIMIT,
+ CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE,
+ CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_MIN,
+ CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_MAX);
+ }
+
+ /**
+ *
+ * @param key
+ * @param defaultValue default value for the given key
+ * @param minValue Minimum value for the given key
+ * @param maxValue Max value for the given key
+ */
+ private void validateRange(String key, String defaultValue, int minValue, int maxValue) {
+ String fileBufferSize = carbonProperties
+ .getProperty(key, defaultValue);
+ if (null != fileBufferSize) {
try {
- int bufferSize = Integer.parseInt(csvReadBufferSizeStr);
- if (bufferSize < CarbonCommonConstants.CSV_READ_BUFFER_SIZE_MIN
- || bufferSize > CarbonCommonConstants.CSV_READ_BUFFER_SIZE_MAX) {
- LOGGER.warn("The value \"" + csvReadBufferSizeStr + "\" configured for key "
- + CarbonCommonConstants.CSV_READ_BUFFER_SIZE
+ int bufferSize = Integer.parseInt(fileBufferSize);
+
+ if (bufferSize < minValue
+ || bufferSize > maxValue) {
+ LOGGER.warn("The value \"" + fileBufferSize + "\" configured for key "
+ + key
+ "\" is not in range. Valid range is (byte) \""
- + CarbonCommonConstants.CSV_READ_BUFFER_SIZE_MIN + " to \""
- + CarbonCommonConstants.CSV_READ_BUFFER_SIZE_MAX + ". Using the default value \""
- + CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.CSV_READ_BUFFER_SIZE,
- CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT);
+ + minValue + " to \""
+ + maxValue +
+ ". Using the default value \""
+ + defaultValue);
+ carbonProperties.setProperty(key,
+ defaultValue);
}
} catch (NumberFormatException nfe) {
- LOGGER.warn("The value \"" + csvReadBufferSizeStr + "\" configured for key "
- + CarbonCommonConstants.CSV_READ_BUFFER_SIZE
+ LOGGER.warn("The value \"" + fileBufferSize + "\" configured for key "
+ + key
+ "\" is invalid. Using the default value \""
- + CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.CSV_READ_BUFFER_SIZE,
- CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT);
+ + defaultValue);
+ carbonProperties.setProperty(key,
+ defaultValue);
}
}
}
+ /**
+ * validate carbon.sort.file.write.buffer.size and if not valid then reset to the default value
+ */
+ private void validateSortFileWriteBufferSize() {
+ validateRange(CARBON_SORT_FILE_WRITE_BUFFER_SIZE,
+ CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE,
+ CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE_MIN,
+ CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE_MAX);
+ }
+
+ /**
+ * The method validate the validity of configured carbon.date.format value
+ * and reset to default value if validation fail
+ */
+ private void validateTimeFormatKey(String key, String defaultValue) {
+ String dateFormat = carbonProperties
+ .getProperty(key, defaultValue);
+ try {
+ new SimpleDateFormat(dateFormat);
+ } catch (Exception e) {
+ LOGGER.warn("The value \"" + dateFormat + "\" configured for key "
+ + key
+ + "\" is invalid. Using the default value \""
+ + key);
+ carbonProperties.setProperty(key, defaultValue);
+ }
+ }
+
+ /**
+ * The method value csv read buffer size and if not valid then reset to the default value
+ */
+ private void validateCarbonCSVReadBufferSizeByte() {
+ validateRange(CSV_READ_BUFFER_SIZE,
+ CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT,
+ CarbonCommonConstants.CSV_READ_BUFFER_SIZE_MIN,
+ CarbonCommonConstants.CSV_READ_BUFFER_SIZE_MAX);
+ }
+
private void validateLockType() {
String lockTypeConfigured = carbonProperties
- .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
+ .getProperty(LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
switch (lockTypeConfigured.toUpperCase()) {
// if user is setting the lock type as CARBON_LOCK_TYPE_ZOOKEEPER then no need to validate
// else validate based on the file system type for LOCAL file system lock will be
@@ -167,67 +327,67 @@ public final class CarbonProperties {
.startsWith(CarbonCommonConstants.ALLUXIOURL_PREFIX))
&& !CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS.equalsIgnoreCase(lockTypeConfigured)) {
LOGGER.warn("The value \"" + lockTypeConfigured + "\" configured for key "
- + CarbonCommonConstants.LOCK_TYPE + " is invalid for current file system. "
+ + LOCK_TYPE + " is invalid for current file system. "
+ "Use the default value " + CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS + " instead.");
- carbonProperties.setProperty(CarbonCommonConstants.LOCK_TYPE,
+ carbonProperties.setProperty(LOCK_TYPE,
CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS);
} else if (null != defaultFs && defaultFs.startsWith(CarbonCommonConstants.LOCAL_FILE_PREFIX)
&& !CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL.equalsIgnoreCase(lockTypeConfigured)) {
- carbonProperties.setProperty(CarbonCommonConstants.LOCK_TYPE,
+ carbonProperties.setProperty(LOCK_TYPE,
CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL);
LOGGER.warn("The value \"" + lockTypeConfigured + "\" configured for key "
- + CarbonCommonConstants.LOCK_TYPE + " is invalid for current file system. "
+ + LOCK_TYPE + " is invalid for current file system. "
+ "Use the default value " + CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL + " instead.");
}
}
private void validateEnableVectorReader() {
String vectorReaderStr =
- carbonProperties.getProperty(CarbonCommonConstants.ENABLE_VECTOR_READER);
+ carbonProperties.getProperty(ENABLE_VECTOR_READER);
boolean isValidBooleanValue = CarbonUtil.validateBoolean(vectorReaderStr);
if (!isValidBooleanValue) {
LOGGER.warn("The enable vector reader value \"" + vectorReaderStr
+ "\" is invalid. Using the default value \""
+ CarbonCommonConstants.ENABLE_VECTOR_READER_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.ENABLE_VECTOR_READER,
+ carbonProperties.setProperty(ENABLE_VECTOR_READER,
CarbonCommonConstants.ENABLE_VECTOR_READER_DEFAULT);
}
}
private void validateCustomBlockDistribution() {
String customBlockDistributionStr =
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION);
+ carbonProperties.getProperty(CARBON_CUSTOM_BLOCK_DISTRIBUTION);
boolean isValidBooleanValue = CarbonUtil.validateBoolean(customBlockDistributionStr);
if (!isValidBooleanValue) {
LOGGER.warn("The custom block distribution value \"" + customBlockDistributionStr
+ "\" is invalid. Using the default value \""
+ CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
+ carbonProperties.setProperty(CARBON_CUSTOM_BLOCK_DISTRIBUTION,
CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT);
}
}
private void validateCombineSmallInputFiles() {
String combineSmallInputFilesStr =
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES);
+ carbonProperties.getProperty(CARBON_COMBINE_SMALL_INPUT_FILES);
boolean isValidBooleanValue = CarbonUtil.validateBoolean(combineSmallInputFilesStr);
if (!isValidBooleanValue) {
LOGGER.warn("The combine small files value \"" + combineSmallInputFilesStr
+ "\" is invalid. Using the default value \""
+ CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES,
+ carbonProperties.setProperty(CARBON_COMBINE_SMALL_INPUT_FILES,
CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT);
}
}
private void validateEnableUnsafeSort() {
- String unSafeSortStr = carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT);
+ String unSafeSortStr = carbonProperties.getProperty(ENABLE_UNSAFE_SORT);
boolean isValidBooleanValue = CarbonUtil.validateBoolean(unSafeSortStr);
if (!isValidBooleanValue) {
LOGGER.warn("The enable unsafe sort value \"" + unSafeSortStr
+ "\" is invalid. Using the default value \""
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ carbonProperties.setProperty(ENABLE_UNSAFE_SORT,
CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT);
}
}
@@ -255,10 +415,10 @@ public final class CarbonProperties {
private void validatePrefetchBufferSize() {
String prefetchBufferSizeStr =
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE);
+ carbonProperties.getProperty(CARBON_PREFETCH_BUFFERSIZE);
if (null == prefetchBufferSizeStr || prefetchBufferSizeStr.length() == 0) {
- carbonProperties.setProperty(CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE,
+ carbonProperties.setProperty(CARBON_PREFETCH_BUFFERSIZE,
CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT);
} else {
try {
@@ -267,16 +427,16 @@ public final class CarbonProperties {
LOGGER.info("The prefetch buffer size value \"" + prefetchBufferSizeStr
+ "\" is invalid. Using the default value \""
+ CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT + "\"");
- carbonProperties.setProperty(CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE,
+ carbonProperties.setProperty(CARBON_PREFETCH_BUFFERSIZE,
CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT);
}
}
}
private void validateHandoffSize() {
- String handoffSizeStr = carbonProperties.getProperty(CarbonCommonConstants.HANDOFF_SIZE);
+ String handoffSizeStr = carbonProperties.getProperty(HANDOFF_SIZE);
if (null == handoffSizeStr || handoffSizeStr.length() == 0) {
- carbonProperties.setProperty(CarbonCommonConstants.HANDOFF_SIZE,
+ carbonProperties.setProperty(HANDOFF_SIZE,
"" + CarbonCommonConstants.HANDOFF_SIZE_DEFAULT);
} else {
try {
@@ -285,14 +445,14 @@ public final class CarbonProperties {
LOGGER.info("The streaming segment max size configured value " + handoffSizeStr +
" is invalid. Using the default value "
+ CarbonCommonConstants.HANDOFF_SIZE_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.HANDOFF_SIZE,
+ carbonProperties.setProperty(HANDOFF_SIZE,
"" + CarbonCommonConstants.HANDOFF_SIZE_DEFAULT);
}
} catch (NumberFormatException e) {
LOGGER.info("The streaming segment max size value \"" + handoffSizeStr
+ "\" is invalid. Using the default value \""
+ CarbonCommonConstants.HANDOFF_SIZE_DEFAULT + "\"");
- carbonProperties.setProperty(CarbonCommonConstants.HANDOFF_SIZE,
+ carbonProperties.setProperty(HANDOFF_SIZE,
"" + CarbonCommonConstants.HANDOFF_SIZE_DEFAULT);
}
}
@@ -300,13 +460,13 @@ public final class CarbonProperties {
private void validateEnableAutoHandoff() {
String enableAutoHandoffStr =
- carbonProperties.getProperty(CarbonCommonConstants.ENABLE_AUTO_HANDOFF);
+ carbonProperties.getProperty(ENABLE_AUTO_HANDOFF);
boolean isValid = CarbonUtil.validateBoolean(enableAutoHandoffStr);
if (!isValid) {
LOGGER.warn("The enable auto handoff value \"" + enableAutoHandoffStr
+ "\" is invalid. Using the default value \""
+ CarbonCommonConstants.ENABLE_AUTO_HANDOFF_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.ENABLE_AUTO_HANDOFF,
+ carbonProperties.setProperty(ENABLE_AUTO_HANDOFF,
CarbonCommonConstants.ENABLE_AUTO_HANDOFF_DEFAULT);
}
}
@@ -316,7 +476,7 @@ public final class CarbonProperties {
*/
private void validateBlockletGroupSizeInMB() {
String numberOfPagePerBlockletColumnString = carbonProperties
- .getProperty(CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB,
+ .getProperty(BLOCKLET_SIZE_IN_MB,
CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE);
try {
short numberOfPagePerBlockletColumn = Short.parseShort(numberOfPagePerBlockletColumnString);
@@ -324,18 +484,18 @@ public final class CarbonProperties {
LOGGER.info("Blocklet Size Configured value \"" + numberOfPagePerBlockletColumnString
+ "\" is invalid. Using the default value \""
+ CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE);
- carbonProperties.setProperty(CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB,
+ carbonProperties.setProperty(BLOCKLET_SIZE_IN_MB,
CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE);
}
} catch (NumberFormatException e) {
LOGGER.info("Blocklet Size Configured value \"" + numberOfPagePerBlockletColumnString
+ "\" is invalid. Using the default value \""
+ CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE);
- carbonProperties.setProperty(CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB,
+ carbonProperties.setProperty(BLOCKLET_SIZE_IN_MB,
CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE);
}
LOGGER.info("Blocklet Size Configured value is \"" + carbonProperties
- .getProperty(CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB,
+ .getProperty(BLOCKLET_SIZE_IN_MB,
CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE));
}
@@ -344,7 +504,7 @@ public final class CarbonProperties {
*/
private void validateNumberOfColumnPerIORead() {
String numberofColumnPerIOString = carbonProperties
- .getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO,
+ .getProperty(NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE);
try {
short numberofColumnPerIO = Short.parseShort(numberofColumnPerIOString);
@@ -353,14 +513,14 @@ public final class CarbonProperties {
LOGGER.info("The Number Of pages per blocklet column value \"" + numberofColumnPerIOString
+ "\" is invalid. Using the default value \""
+ CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE);
- carbonProperties.setProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO,
+ carbonProperties.setProperty(NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE);
}
} catch (NumberFormatException e) {
LOGGER.info("The Number Of pages per blocklet column value \"" + numberofColumnPerIOString
+ "\" is invalid. Using the default value \""
+ CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE);
- carbonProperties.setProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO,
+ carbonProperties.setProperty(NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE);
}
}
@@ -369,7 +529,7 @@ public final class CarbonProperties {
* This method validates the blocklet size
*/
private void validateBlockletSize() {
- String blockletSizeStr = carbonProperties.getProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+ String blockletSizeStr = carbonProperties.getProperty(BLOCKLET_SIZE,
CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
try {
int blockletSize = Integer.parseInt(blockletSizeStr);
@@ -379,14 +539,14 @@ public final class CarbonProperties {
LOGGER.info("The blocklet size value \"" + blockletSizeStr
+ "\" is invalid. Using the default value \""
+ CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+ carbonProperties.setProperty(BLOCKLET_SIZE,
CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
}
} catch (NumberFormatException e) {
LOGGER.info("The blocklet size value \"" + blockletSizeStr
+ "\" is invalid. Using the default value \""
+ CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+ carbonProperties.setProperty(BLOCKLET_SIZE,
CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
}
}
@@ -396,7 +556,7 @@ public final class CarbonProperties {
*/
private void validateNumCores() {
String numCoresStr = carbonProperties
- .getProperty(CarbonCommonConstants.NUM_CORES, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
+ .getProperty(NUM_CORES, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
try {
int numCores = Integer.parseInt(numCoresStr);
@@ -405,14 +565,14 @@ public final class CarbonProperties {
LOGGER.info(
"The num Cores value \"" + numCoresStr + "\" is invalid. Using the default value \""
+ CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.NUM_CORES,
+ carbonProperties.setProperty(NUM_CORES,
CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
}
} catch (NumberFormatException e) {
LOGGER.info(
"The num Cores value \"" + numCoresStr + "\" is invalid. Using the default value \""
+ CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.NUM_CORES,
+ carbonProperties.setProperty(NUM_CORES,
CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
}
}
@@ -421,7 +581,7 @@ public final class CarbonProperties {
* This method validates the number cores specified for mdk block sort
*/
private void validateNumCoresBlockSort() {
- String numCoresStr = carbonProperties.getProperty(CarbonCommonConstants.NUM_CORES_BLOCK_SORT,
+ String numCoresStr = carbonProperties.getProperty(NUM_CORES_BLOCK_SORT,
CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
try {
int numCores = Integer.parseInt(numCoresStr);
@@ -431,14 +591,14 @@ public final class CarbonProperties {
LOGGER.info("The num cores value \"" + numCoresStr
+ "\" for block sort is invalid. Using the default value \""
+ CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.NUM_CORES_BLOCK_SORT,
+ carbonProperties.setProperty(NUM_CORES_BLOCK_SORT,
CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
}
} catch (NumberFormatException e) {
LOGGER.info("The num cores value \"" + numCoresStr
+ "\" for block sort is invalid. Using the default value \""
+ CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.NUM_CORES_BLOCK_SORT,
+ carbonProperties.setProperty(NUM_CORES_BLOCK_SORT,
CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
}
}
@@ -448,7 +608,7 @@ public final class CarbonProperties {
*/
private void validateSortSize() {
String sortSizeStr = carbonProperties
- .getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
+ .getProperty(SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
try {
int sortSize = Integer.parseInt(sortSizeStr);
@@ -456,14 +616,14 @@ public final class CarbonProperties {
LOGGER.info(
"The batch size value \"" + sortSizeStr + "\" is invalid. Using the default value \""
+ CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.SORT_SIZE,
+ carbonProperties.setProperty(SORT_SIZE,
CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
}
} catch (NumberFormatException e) {
LOGGER.info(
"The batch size value \"" + sortSizeStr + "\" is invalid. Using the default value \""
+ CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.SORT_SIZE,
+ carbonProperties.setProperty(SORT_SIZE,
CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
}
}
@@ -474,10 +634,10 @@ public final class CarbonProperties {
*/
private void validateCarbonDataFileVersion() {
String carbondataFileVersionString =
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION);
+ carbonProperties.getProperty(CARBON_DATA_FILE_VERSION);
if (carbondataFileVersionString == null) {
// use default property if user does not specify version property
- carbonProperties.setProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION,
+ carbonProperties.setProperty(CARBON_DATA_FILE_VERSION,
CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION);
} else {
try {
@@ -487,12 +647,12 @@ public final class CarbonProperties {
LOGGER.warn("Specified file version property is invalid: " + carbondataFileVersionString
+ ". Using " + CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION
+ " as default file version");
- carbonProperties.setProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION,
+ carbonProperties.setProperty(CARBON_DATA_FILE_VERSION,
CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION);
}
}
LOGGER.info("Carbon Current data file version: " + carbonProperties
- .setProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION,
+ .setProperty(CARBON_DATA_FILE_VERSION,
CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION));
}
@@ -533,6 +693,11 @@ public final class CarbonProperties {
}
print();
+ try {
+ initPropertySet();
+ } catch (IllegalAccessException e) {
+ LOGGER.error("Illegal access to declared field" + e.getMessage());
+ }
}
/**
@@ -605,6 +770,9 @@ public final class CarbonProperties {
public CarbonProperties addProperty(String key, String value) {
carbonProperties.setProperty(key, value);
addedProperty.put(key, value);
+ // the method will validate the added property
+ // if the added property is not valid then will reset to default value.
+ validateAndLoadDefaultProperties(key.toLowerCase());
return this;
}
@@ -613,7 +781,7 @@ public final class CarbonProperties {
}
public ColumnarFormatVersion getFormatVersion() {
- String versionStr = getInstance().getProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION);
+ String versionStr = getInstance().getProperty(CARBON_DATA_FILE_VERSION);
if (versionStr == null) {
return getDefaultFormatVersion();
} else {
@@ -796,7 +964,7 @@ public final class CarbonProperties {
try {
handoffSize = Long.parseLong(
CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.HANDOFF_SIZE,
+ HANDOFF_SIZE,
"" + CarbonCommonConstants.HANDOFF_SIZE_DEFAULT
)
);
@@ -808,7 +976,7 @@ public final class CarbonProperties {
public boolean isEnableAutoHandoff() {
String enableAutoHandoffStr = CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.ENABLE_AUTO_HANDOFF,
+ ENABLE_AUTO_HANDOFF,
CarbonCommonConstants.ENABLE_AUTO_HANDOFF_DEFAULT);
return enableAutoHandoffStr.equalsIgnoreCase("true");
}
@@ -836,7 +1004,7 @@ public final class CarbonProperties {
int executorStartUpTimeOut = 0;
try {
executorStartUpTimeOut = Integer.parseInt(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_EXECUTOR_STARTUP_TIMEOUT,
+ .getProperty(CARBON_EXECUTOR_STARTUP_TIMEOUT,
CarbonCommonConstants.CARBON_EXECUTOR_WAITING_TIMEOUT_DEFAULT));
// If value configured by user is more than max value of time out then consider the max value
if (executorStartUpTimeOut > CarbonCommonConstants.CARBON_EXECUTOR_WAITING_TIMEOUT_MAX) {
@@ -846,7 +1014,7 @@ public final class CarbonProperties {
executorStartUpTimeOut =
Integer.parseInt(CarbonCommonConstants.CARBON_EXECUTOR_WAITING_TIMEOUT_DEFAULT);
}
- carbonProperties.setProperty(CarbonCommonConstants.CARBON_EXECUTOR_STARTUP_TIMEOUT,
+ carbonProperties.setProperty(CARBON_EXECUTOR_STARTUP_TIMEOUT,
String.valueOf(executorStartUpTimeOut));
LOGGER.info("Executor start up wait time: " + executorStartUpTimeOut);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bcf3ca3f/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
index 27a9a6d..ae29b03 100644
--- a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
@@ -60,7 +60,7 @@ public class CarbonPropertiesValidationTest extends TestCase {
validateMethodType.invoke(carbonProperties);
String valueAfterValidation =
carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT);
- assertTrue(!valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
assertTrue(
CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT.equalsIgnoreCase(valueAfterValidation));
}
@@ -76,7 +76,7 @@ public class CarbonPropertiesValidationTest extends TestCase {
validateMethodType.invoke(carbonProperties);
String valueAfterValidation =
carbonProperties.getProperty(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION);
- assertTrue(!valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
assertTrue(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT
.equalsIgnoreCase(valueAfterValidation));
}
@@ -92,7 +92,7 @@ public class CarbonPropertiesValidationTest extends TestCase {
validateMethodType.invoke(carbonProperties);
String valueAfterValidation =
carbonProperties.getProperty(CarbonCommonConstants.ENABLE_VECTOR_READER);
- assertTrue(!valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
assertTrue(
CarbonCommonConstants.ENABLE_VECTOR_READER_DEFAULT.equalsIgnoreCase(valueAfterValidation));
}
@@ -108,7 +108,7 @@ public class CarbonPropertiesValidationTest extends TestCase {
validateMethodType.invoke(carbonProperties);
String valueAfterValidation =
carbonProperties.getProperty(CarbonCommonConstants.CSV_READ_BUFFER_SIZE);
- assertTrue(!valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
assertTrue(
CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT.equalsIgnoreCase(valueAfterValidation));
}
@@ -124,7 +124,7 @@ public class CarbonPropertiesValidationTest extends TestCase {
validateMethodType.invoke(carbonProperties);
String valueAfterValidation =
carbonProperties.getProperty(CarbonCommonConstants.CSV_READ_BUFFER_SIZE);
- assertTrue(!valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
assertTrue(
CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT.equalsIgnoreCase(valueAfterValidation));
carbonProperties.addProperty(CarbonCommonConstants.CSV_READ_BUFFER_SIZE, "10240");
@@ -140,7 +140,7 @@ public class CarbonPropertiesValidationTest extends TestCase {
validateMethodType.invoke(carbonProperties);
valueAfterValidation =
carbonProperties.getProperty(CarbonCommonConstants.CSV_READ_BUFFER_SIZE);
- assertTrue(!valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
assertTrue(
CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT.equalsIgnoreCase(valueAfterValidation));
}
@@ -151,4 +151,58 @@ public class CarbonPropertiesValidationTest extends TestCase {
carbonProperties.addProperty(CarbonCommonConstants.HANDOFF_SIZE, "" + newSize);
assertEquals(newSize, carbonProperties.getHandoffSize());
}
+
+ @Test public void testValidateTimeStampFormat()
+ throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ Method validateMethodType = carbonProperties.getClass()
+ .getDeclaredMethod("validateTimeFormatKey", new Class[] { String.class, String.class });
+ validateMethodType.setAccessible(true);
+ carbonProperties.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "agdgaJIASDG667");
+ String valueBeforeValidation =
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT);
+ validateMethodType.invoke(carbonProperties, CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ String valueAfterValidation =
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT);
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
+ .equalsIgnoreCase(valueAfterValidation));
+ carbonProperties.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ "yyyy-MM-dd hh:mm:ss");
+ validateMethodType.invoke(carbonProperties, CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ assertEquals("yyyy-MM-dd hh:mm:ss",
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT));
+ }
+
+ @Test public void testValidateSortFileWriteBufferSize()
+ throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ Method validateMethodType =
+ carbonProperties.getClass().getDeclaredMethod("validateSortFileWriteBufferSize");
+ validateMethodType.setAccessible(true);
+ carbonProperties.addProperty(CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE, "test");
+ String valueBeforeValidation =
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE);
+ validateMethodType.invoke(carbonProperties);
+ String valueAfterValidation =
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE);
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE
+ .equalsIgnoreCase(valueAfterValidation));
+ }
+ @Test public void testValidateSortIntermediateFilesLimit()
+ throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ Method validateMethodType =
+ carbonProperties.getClass().getDeclaredMethod("validateSortIntermediateFilesLimit");
+ validateMethodType.setAccessible(true);
+ carbonProperties.addProperty(CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT, "test");
+ String valueBeforeValidation =
+ carbonProperties.getProperty(CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT);
+ validateMethodType.invoke(carbonProperties);
+ String valueAfterValidation =
+ carbonProperties.getProperty(CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT);
+ assertTrue(valueBeforeValidation.equals(valueAfterValidation));
+ assertTrue(CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE
+ .equalsIgnoreCase(valueAfterValidation));
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bcf3ca3f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index d9f119c..74216ac 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -138,6 +138,8 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
assert(DataMapWriterSuite.callbackSeq.head.contains("block start"))
assert(DataMapWriterSuite.callbackSeq.last.contains("block end"))
+ // corrected test case the min "carbon.blockletgroup.size.in.mb" size could not be less than
+ // 64 MB
assert(
DataMapWriterSuite.callbackSeq.slice(1, DataMapWriterSuite.callbackSeq.length - 1) == Seq(
"blocklet start 0",
@@ -145,17 +147,13 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
"add page data: blocklet 0, page 1",
"add page data: blocklet 0, page 2",
"add page data: blocklet 0, page 3",
- "blocklet end: 0",
- "blocklet start 1",
- "add page data: blocklet 1, page 0",
- "add page data: blocklet 1, page 1",
- "add page data: blocklet 1, page 2",
- "add page data: blocklet 1, page 3",
- "blocklet end: 1",
- "blocklet start 2",
- "add page data: blocklet 2, page 0",
- "add page data: blocklet 2, page 1",
- "blocklet end: 2"
+ "add page data: blocklet 0, page 4",
+ "add page data: blocklet 0, page 5",
+ "add page data: blocklet 0, page 6",
+ "add page data: blocklet 0, page 7",
+ "add page data: blocklet 0, page 8",
+ "add page data: blocklet 0, page 9",
+ "blocklet end: 0"
))
DataMapWriterSuite.callbackSeq = Seq()
}