[carbondata] 11/41: [CARBONDATA-3281] Add validation for the size of the LRU cache

[ra...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ravipesala pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/carbondata.git

commit ffa77304a794ec5efa9eabe73a6eef68740e48fd
Author: litt <li...@126.com>
AuthorDate: Tue Jan 29 16:34:40 2019 +0800

    [CARBONDATA-3281] Add validation for the size of the LRU cache
    
    If configure the LRU bigger than jvm xmx size, then use CARBON_MAX_LRU_CACHE_SIZE_DEFAULT instead.Because if setting LRU bigger than xmx size,if we query for a big table with too many carbonfiles, it may cause "Error: java.io.IOException: Problem in loading segment blocks: GC overhead
    limit exceeded (state=,code=0)" and the jdbc server will restart.
    
    This closes #3118
---
 .../carbondata/core/cache/CarbonLRUCache.java      | 32 ++++++++++++++++++++++
 .../core/constants/CarbonCommonConstants.java      |  5 ++++
 .../carbondata/core/cache/CarbonLRUCacheTest.java  |  7 +++++
 3 files changed, 44 insertions(+)

diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java b/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
index 0c75173..3371d0d 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
@@ -70,6 +70,16 @@ public final class CarbonLRUCache {
           + CarbonCommonConstants.CARBON_MAX_LRU_CACHE_SIZE_DEFAULT);
       lruCacheMemorySize = Long.parseLong(defaultPropertyName);
     }
+
+    // if lru cache is bigger than jvm max heap then set part size of max heap (60% default)
+    if (isBeyondMaxMemory()) {
+      double changeSize = getPartOfXmx();
+      LOGGER.warn("Configured LRU size " + lruCacheMemorySize +
+              "MB exceeds the max size of JVM heap. Carbon will fallback to use " +
+              changeSize + " MB instead");
+      lruCacheMemorySize = (long)changeSize;
+    }
+
     initCache();
     if (lruCacheMemorySize > 0) {
       LOGGER.info("Configured LRU cache size is " + lruCacheMemorySize + " MB");
@@ -326,4 +336,26 @@ public final class CarbonLRUCache {
   public Map<String, Cacheable> getCacheMap() {
     return lruCacheMap;
   }
+
+  /**
+   * Check if LRU cache setting is bigger than max memory of jvm.
+   * if LRU cache is bigger than max memory of jvm when query for a big segments table,
+   * may cause JDBC server crash.
+   * @return true LRU cache is bigger than max memory of jvm, false otherwise
+   */
+  private boolean isBeyondMaxMemory() {
+    long mSize = Runtime.getRuntime().maxMemory();
+    long lruSize = lruCacheMemorySize * BYTE_CONVERSION_CONSTANT;
+    return lruSize >= mSize;
+  }
+
+  /**
+   * when LRU cache is bigger than max heap of jvm.
+   * set to part of  max heap size, use CARBON_LRU_CACHE_PERCENT_OVER_MAX_SIZE default 60%.
+   * @return the LRU cache size
+   */
+  private double getPartOfXmx() {
+    long mSizeMB = Runtime.getRuntime().maxMemory() / BYTE_CONVERSION_CONSTANT;
+    return mSizeMB * CarbonCommonConstants.CARBON_LRU_CACHE_PERCENT_OVER_MAX_SIZE;
+  }
 }
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index f5c07a4..69374ad 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1257,6 +1257,11 @@ public final class CarbonCommonConstants {
   public static final String CARBON_MAX_LRU_CACHE_SIZE_DEFAULT = "-1";
 
   /**
+   * when LRU cache if beyond the jvm max memory size,set 60% percent of max size
+   */
+  public static final double CARBON_LRU_CACHE_PERCENT_OVER_MAX_SIZE = 0.6d;
+
+  /**
    * property to enable min max during filter query
    */
   @CarbonProperty
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/CarbonLRUCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/CarbonLRUCacheTest.java
index 0493655..8ef6684 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/CarbonLRUCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/CarbonLRUCacheTest.java
@@ -60,6 +60,13 @@ public class CarbonLRUCacheTest {
     assertNull(carbonLRUCache.get("Column2"));
   }
 
+  @Test public void testBiggerThanMaxSizeConfiguration() {
+    CarbonLRUCache carbonLRUCacheForConfig =
+            new CarbonLRUCache("prop2", "200000");//200GB
+    assertTrue(carbonLRUCacheForConfig.put("Column1", cacheable, 10L));
+    assertFalse(carbonLRUCacheForConfig.put("Column2", cacheable, 107374182400L));//100GB
+  }
+
   @AfterClass public static void cleanUp() {
     carbonLRUCache.clear();
     assertNull(carbonLRUCache.get("Column1"));