You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/08/02 11:37:26 UTC
carbondata git commit: [Documentation] [Unsafe Configuration] Added
carbon.unsafe.driver.working.memory.in.mb parameter to differentiate between
driver and executor unsafe memory
Repository: carbondata
Updated Branches:
refs/heads/master 7e93d7b87 -> a2928e314
[Documentation] [Unsafe Configuration] Added carbon.unsafe.driver.working.memory.in.mb parameter to differentiate between driver and executor unsafe memory
Added carbon.unsafe.driver.working.memory.in.mb parameter to differentiate between driver and executor unsafe memory
Usually in production scenarios driver memory will be less than the executor memory. Now we are using unsafe for caching block/blocklet dataMap in driver. Current unsafe memory configured for executor is getting used for driver also which is not a good idea.
Therefore it is required to separate out driver and executor unsafe memory.
You can observe the same in spark configuration also that spark has given different parameters for configuring driver and executor memory overhead to control the unsafe memory usage.
spark.yarn.driver.memoryOverhead and spark.yarn.executor.memoryOverhead
This closes #2595
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a2928e31
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a2928e31
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a2928e31
Branch: refs/heads/master
Commit: a2928e314a4c45dd35923d7d29b75508e401dd3f
Parents: 7e93d7b
Author: manishgupta88 <to...@gmail.com>
Authored: Wed Aug 1 19:38:30 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Aug 2 17:07:16 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 ++++
.../core/memory/UnsafeMemoryManager.java | 25 ++++++++++++++++----
docs/configuration-parameters.md | 2 ++
3 files changed, 27 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a2928e31/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 6d7215e..e480007 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1276,6 +1276,10 @@ public final class CarbonCommonConstants {
@CarbonProperty
public static final String UNSAFE_WORKING_MEMORY_IN_MB = "carbon.unsafe.working.memory.in.mb";
public static final String UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT = "512";
+
+ @CarbonProperty
+ public static final String UNSAFE_DRIVER_WORKING_MEMORY_IN_MB =
+ "carbon.unsafe.driver.working.memory.in.mb";
/**
* Sorts the data in batches and writes the batch data to store with index file.
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a2928e31/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
index 2115f82..9133f0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
@@ -41,11 +41,28 @@ public class UnsafeMemoryManager {
CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT));
private static Map<Long,Set<MemoryBlock>> taskIdToMemoryBlockMap;
static {
- long size;
+ long size = 0L;
try {
- size = Long.parseLong(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
- CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT));
+ // check if driver unsafe memory is configured and JVM process is in driver. In that case
+ // initialize unsafe memory configured for driver
+ boolean isDriver = Boolean.parseBoolean(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "false"));
+ boolean initializedWithUnsafeDriverMemory = false;
+ if (isDriver) {
+ String driverUnsafeMemorySize = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.UNSAFE_DRIVER_WORKING_MEMORY_IN_MB);
+ if (null != driverUnsafeMemorySize) {
+ size = Long.parseLong(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.UNSAFE_DRIVER_WORKING_MEMORY_IN_MB,
+ CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT));
+ initializedWithUnsafeDriverMemory = true;
+ }
+ }
+ if (!initializedWithUnsafeDriverMemory) {
+ size = Long.parseLong(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
+ CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT));
+ }
} catch (Exception e) {
size = Long.parseLong(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT);
LOGGER.info("Wrong memory size given, "
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a2928e31/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index b614918..6e4dea5 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -40,6 +40,8 @@ This section provides the details of all the configurations required for the Car
| carbon.streaming.segment.max.size | 1024000000 | This parameter defines the maximum size of the streaming segment. Setting this parameter to appropriate value will avoid impacting the streaming ingestion. The value is in bytes.|
| carbon.query.show.datamaps | true | If this parameter value is set to true, show tables command will list all the tables including datatmaps(eg: Preaggregate table), else datamaps will be excluded from the table list. |
| carbon.segment.lock.files.preserve.hours | 48 | This property value indicates the number of hours the segment lock files will be preserved after dataload. These lock files will be deleted with the clean command after the configured number of hours. |
+| carbon.unsafe.working.memory.in.mb | 512 | Specifies the size of executor unsafe working memory. Used for sorting data, storing column pages,etc. This value is expressed in MB. |
+| carbon.unsafe.driver.working.memory.in.mb | 512 | Specifies the size of driver unsafe working memory. Used for storing block or blocklet datamap cache. If not configured then carbon.unsafe.working.memory.in.mb value is considered. This value is expressed in MB. |
## Performance Configuration
This section provides the details of all the configurations required for CarbonData Performance Optimization.