[carbondata] branch master updated: [CARBONDATA-3233]Fix JVM crash issue in snappy compressor and update the pagesize correctly

[ma...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

manishgupta88 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 92c9ce3  [CARBONDATA-3233]Fix JVM crash issue in snappy compressor and update the pagesize correctly
92c9ce3 is described below

commit 92c9ce3ff0da23f207376e4f8861717e2e3de1e5
Author: akashrn5 <ak...@gmail.com>
AuthorDate: Mon Jan 7 16:34:48 2019 +0530

    [CARBONDATA-3233]Fix JVM crash issue in snappy compressor and update the pagesize correctly
    
    Problem:
    1. During dataload sometimes the JVM is crashed during offheap snappy compression. We get the maximun compress size from compressor and allocate
    that much memory and then call rawCompress with the base offset of page and then base offset of newly created memory block. During this call
    sometimes JVM crashes from Snappy. This issue is random one and fails only sometimes.
    2. PageSize is getting updated wrongly, actaul pageSize is number of rows in page, we were updaing the rowId not rowSize
    
    Solution:
    Remove the method implementation and let super class handle the compression based on the datatype. which will fix this random JVM crash issue
    
    This closes #3053
---
 .../datastore/page/UnsafeFixLengthColumnPage.java  | 29 +++-------------------
 1 file changed, 4 insertions(+), 25 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index da0e487..2e576bc 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -17,10 +17,8 @@
 
 package org.apache.carbondata.core.datastore.page;
 
-import java.io.IOException;
 import java.math.BigDecimal;
 
-import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
 import org.apache.carbondata.core.memory.CarbonUnsafe;
 import org.apache.carbondata.core.memory.MemoryBlock;
@@ -112,7 +110,8 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
 
   private void updatePageSize(int rowId) {
     if (pageSize < rowId) {
-      pageSize = rowId;
+      // update the actual number of rows
+      pageSize = rowId + 1;
     }
   }
 
@@ -359,7 +358,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
 
   @Override
   public float[] getFloatPage() {
-    float[] data = new float[getPageSize()];
+    float[] data = new float[getEndLoop()];
     for (long i = 0; i < data.length; i++) {
       long offset = i << floatBits;
       data[(int)i] = CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset);
@@ -369,7 +368,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
 
   @Override
   public double[] getDoublePage() {
-    double[] data = new double[getPageSize()];
+    double[] data = new double[getEndLoop()];
     for (long i = 0; i < data.length; i++) {
       long offset = i << doubleBits;
       data[(int)i] = CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset);
@@ -541,26 +540,6 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
     return totalLength;
   }
 
-  @Override public byte[] compress(Compressor compressor) throws MemoryException, IOException {
-    if (UnsafeMemoryManager.isOffHeap() && compressor.supportUnsafe()) {
-      // use raw compression and copy to byte[]
-      int inputSize = totalLength;
-      long compressedMaxSize = compressor.maxCompressedLength(inputSize);
-      MemoryBlock compressed =
-          UnsafeMemoryManager.allocateMemoryWithRetry(taskId, compressedMaxSize);
-      long outSize = compressor.rawCompress(baseOffset, inputSize, compressed.getBaseOffset());
-      assert outSize < Integer.MAX_VALUE;
-      byte[] output = new byte[(int) outSize];
-      CarbonUnsafe.getUnsafe()
-          .copyMemory(compressed.getBaseObject(), compressed.getBaseOffset(), output,
-              CarbonUnsafe.BYTE_ARRAY_OFFSET, outSize);
-      UnsafeMemoryManager.INSTANCE.freeMemory(taskId, compressed);
-      return output;
-    } else {
-      return super.compress(compressor);
-    }
-  }
-
   /**
    * reallocate memory if capacity length than current size + request size
    */