You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/02/03 19:43:33 UTC
[26/50] [abbrv] carbondata git commit: [CARBONDATA-2108]Updated
unsafe sort memory configuration
[CARBONDATA-2108]Updated unsafe sort memory configuration
Deprecated old property: sort.inmemory.size.inmb
Added new property: carbon.sort.storage.inmemory.size.inmb,
If user has configured old property then internally it will be converted to new property
for ex: If user has configured sort.inmemory.size.inmb then 20% memory will be used as working memory and rest for storage memory
This closes #1896
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/27ec6515
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/27ec6515
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/27ec6515
Branch: refs/heads/branch-1.3
Commit: 27ec6515a143dc3b697ac914bfcd4cfe10a49e17
Parents: 2610a60
Author: kumarvishal <ku...@gmail.com>
Authored: Wed Jan 31 18:43:02 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Feb 2 23:18:21 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 5 +
.../core/memory/UnsafeMemoryManager.java | 2 +-
.../core/memory/UnsafeSortMemoryManager.java | 6 +-
.../carbondata/core/util/CarbonProperties.java | 99 ++++++++++++++++++++
4 files changed, 108 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 87eec8a..8480758 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1585,6 +1585,11 @@ public final class CarbonCommonConstants {
public static final String CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION_DEFAULT = "true";
+ @CarbonProperty
+ public static final String IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB =
+ "carbon.sort.storage.inmemory.size.inmb";
+ public static final String IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT = "512";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
index 4222e14..d3b9b48 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
@@ -47,7 +47,7 @@ public class UnsafeMemoryManager {
.getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT));
} catch (Exception e) {
- size = Long.parseLong(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+ size = Long.parseLong(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT);
LOGGER.info("Wrong memory size given, "
+ "so setting default value to " + size);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
index c63b320..67bb6cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
@@ -75,10 +75,10 @@ public class UnsafeSortMemoryManager {
long size;
try {
size = Long.parseLong(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB,
- CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT));
+ .getProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB,
+ CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT));
} catch (Exception e) {
- size = Long.parseLong(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+ size = Long.parseLong(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT);
LOGGER.info("Wrong memory size given, " + "so setting default value to " + size);
}
if (size < 1024) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 39a0b80..3dc7b8f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -223,6 +223,9 @@ public final class CarbonProperties {
validateSortIntermediateFilesLimit();
validateEnableAutoHandoff();
validateSchedulerMinRegisteredRatio();
+ validateSortMemorySizeInMB();
+ validateWorkingMemory();
+ validateSortStorageMemory();
}
/**
@@ -1252,4 +1255,100 @@ public final class CarbonProperties {
public void addPropertyToPropertySet(Set<String> externalPropertySet) {
propertySet.addAll(externalPropertySet);
}
+
+ private void validateSortMemorySizeInMB() {
+ int sortMemorySizeInMBDefault =
+ Integer.parseInt(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+ int sortMemorySizeInMB = 0;
+ try {
+ sortMemorySizeInMB = Integer.parseInt(
+ carbonProperties.getProperty(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB));
+ } catch (NumberFormatException e) {
+ LOGGER.error(
+ "The specified value for property " + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB
+ + "is Invalid." + " Taking the default value."
+ + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+ sortMemorySizeInMB = sortMemorySizeInMBDefault;
+ }
+ if (sortMemorySizeInMB < sortMemorySizeInMBDefault) {
+ LOGGER.error(
+ "The specified value for property " + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB
+ + "is less than default value." + ". Taking the default value."
+ + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+ sortMemorySizeInMB = sortMemorySizeInMBDefault;
+ }
+ String unsafeWorkingMemoryString =
+ carbonProperties.getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB);
+ String unsafeSortStorageMemoryString =
+ carbonProperties.getProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB);
+ int workingMemory = 512;
+ int sortStorageMemory;
+ if (null == unsafeWorkingMemoryString && null == unsafeSortStorageMemoryString) {
+ workingMemory = workingMemory > ((sortMemorySizeInMB * 20) / 100) ?
+ workingMemory :
+ ((sortMemorySizeInMB * 20) / 100);
+ sortStorageMemory = sortMemorySizeInMB - workingMemory;
+ carbonProperties
+ .setProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, workingMemory + "");
+ carbonProperties.setProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB,
+ sortStorageMemory + "");
+ } else if (null != unsafeWorkingMemoryString && null == unsafeSortStorageMemoryString) {
+ carbonProperties.setProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB,
+ sortMemorySizeInMB + "");
+ } else if (null == unsafeWorkingMemoryString && null != unsafeSortStorageMemoryString) {
+ carbonProperties
+ .setProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, sortMemorySizeInMB + "");
+ }
+ }
+
+ private void validateWorkingMemory() {
+ int unsafeWorkingMemoryDefault =
+ Integer.parseInt(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT);
+ int unsafeWorkingMemory = 0;
+ try {
+ unsafeWorkingMemory = Integer.parseInt(
+ carbonProperties.getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB));
+ } catch (NumberFormatException e) {
+ LOGGER.error("The specified value for property "
+ + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT + "is invalid."
+ + " Taking the default value."
+ + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT);
+ unsafeWorkingMemory = unsafeWorkingMemoryDefault;
+ }
+ if (unsafeWorkingMemory < unsafeWorkingMemoryDefault) {
+ LOGGER.error("The specified value for property "
+ + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT
+ + "is less than the default value." + ". Taking the default value."
+ + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT);
+ unsafeWorkingMemory = unsafeWorkingMemoryDefault;
+ }
+ carbonProperties
+ .setProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, unsafeWorkingMemory + "");
+ }
+
+ private void validateSortStorageMemory() {
+ int unsafeSortStorageMemoryDefault =
+ Integer.parseInt(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT);
+ int unsafeSortStorageMemory = 0;
+ try {
+ unsafeSortStorageMemory = Integer.parseInt(carbonProperties
+ .getProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB));
+ } catch (NumberFormatException e) {
+ LOGGER.error("The specified value for property "
+ + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB + "is invalid."
+ + " Taking the default value."
+ + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT);
+ unsafeSortStorageMemory = unsafeSortStorageMemoryDefault;
+ }
+ if (unsafeSortStorageMemory < unsafeSortStorageMemoryDefault) {
+ LOGGER.error("The specified value for property "
+ + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB
+ + "is less than the default value." + " Taking the default value."
+ + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT);
+ unsafeSortStorageMemory = unsafeSortStorageMemoryDefault;
+ }
+ carbonProperties.setProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB,
+ unsafeSortStorageMemory + "");
+ }
+
}