You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/11/30 16:34:13 UTC
[20/26] carbondata git commit: [CARBONDATA-3124] Updated log message
in UnsafeMemoryManager
[CARBONDATA-3124] Updated log message in UnsafeMemoryManager
This closes #2948
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5dd242bf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5dd242bf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5dd242bf
Branch: refs/heads/branch-1.5
Commit: 5dd242bfec568a0a725a931376161625e48dbf86
Parents: 63d1609
Author: shardul-cr7 <sh...@gmail.com>
Authored: Fri Nov 23 18:45:44 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Nov 30 21:57:21 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/memory/UnsafeMemoryManager.java | 12 ++++--------
docs/faq.md | 8 +++-----
2 files changed, 7 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5dd242bf/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
index 725f003..493ec0b 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
@@ -103,7 +103,7 @@ public class UnsafeMemoryManager {
private UnsafeMemoryManager(long totalMemory, MemoryType memoryType) {
this.totalMemory = totalMemory;
this.memoryType = memoryType;
- LOGGER.info("offheap Working Memory manager is created with size " + totalMemory + " with "
+ LOGGER.info("Offheap Working Memory manager is created with size " + totalMemory + " with "
+ memoryType);
}
@@ -128,8 +128,8 @@ public class UnsafeMemoryManager {
// not adding on heap memory block to map as JVM will take care of freeing the memory
memoryBlock = MemoryAllocator.HEAP.allocate(memoryRequested);
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(
- String.format("Creating onheap working Memory block (%s) with size:", memoryBlock));
+ LOGGER.debug(String
+ .format("Creating onheap working Memory block (%d) with size: ", memoryBlock.size()));
}
}
return memoryBlock;
@@ -143,7 +143,7 @@ public class UnsafeMemoryManager {
getMemoryAllocator(memoryBlock.getMemoryType()).free(memoryBlock);
memoryUsed -= memoryBlock.size();
memoryUsed = memoryUsed < 0 ? 0 : memoryUsed;
- if (LOGGER.isDebugEnabled()) {
+ if (LOGGER.isDebugEnabled() && memoryBlock.getMemoryType() == MemoryType.OFFHEAP) {
LOGGER.debug(String.format("Freeing offheap working memory block (%s) with size: %d, "
+ "current available memory is: %d", memoryBlock.toString(), memoryBlock.size(),
totalMemory - memoryUsed));
@@ -178,10 +178,6 @@ public class UnsafeMemoryManager {
taskId, memoryUsed, StringUtils.join(taskIdToOffheapMemoryBlockMap.keySet(), ", ")));
}
- public synchronized boolean isMemoryAvailable() {
- return memoryUsed > totalMemory;
- }
-
public long getUsableMemory() {
return totalMemory;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5dd242bf/docs/faq.md
----------------------------------------------------------------------
diff --git a/docs/faq.md b/docs/faq.md
index 3ac9a0a..dbcda4f 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -216,20 +216,18 @@ TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai"))
## How to check LRU cache memory footprint?
To observe the LRU cache memory footprint in the logs, configure the below properties in log4j.properties file.
```
-log4j.logger.org.apache.carbondata.core.memory.UnsafeMemoryManager = DEBUG
log4j.logger.org.apache.carbondata.core.cache.CarbonLRUCache = DEBUG
```
-These properties will enable the DEBUG log for the CarbonLRUCache and UnsafeMemoryManager which will print the information of memory consumed using which the LRU cache size can be decided. **Note:** Enabling the DEBUG log will degrade the query performance.
+This property will enable the DEBUG log for the CarbonLRUCache and UnsafeMemoryManager which will print the information of memory consumed using which the LRU cache size can be decided. **Note:** Enabling the DEBUG log will degrade the query performance. Ensure carbon.max.driver.lru.cache.size is configured to observe the current cache size.
**Example:**
```
-18/09/26 15:05:28 DEBUG UnsafeMemoryManager: pool-44-thread-1 Memory block (org.apache.carbondata.core.memory.MemoryBlock@21312095) is created with size 10. Total memory used 413Bytes, left 536870499Bytes
18/09/26 15:05:29 DEBUG CarbonLRUCache: main Required size for entry /home/target/store/default/stored_as_carbondata_table/Fact/Part0/Segment_0/0_1537954529044.carbonindexmerge :: 181 Current cache size :: 0
-18/09/26 15:05:30 DEBUG UnsafeMemoryManager: main Freeing memory of size: 105available memory: 536870836
-18/09/26 15:05:30 DEBUG UnsafeMemoryManager: main Freeing memory of size: 76available memory: 536870912
18/09/26 15:05:30 INFO CarbonLRUCache: main Removed entry from InMemory lru cache :: /home/target/store/default/stored_as_carbondata_table/Fact/Part0/Segment_0/0_1537954529044.carbonindexmerge
```
+**Note:** If `Removed entry from InMemory LRU cache` are frequently observed in logs, you may have to increase the configured LRU size.
+To observe the LRU cache from heap dump, check the heap used by CarbonLRUCache class.
## Getting tablestatus.lock issues When loading data
**Symptom**