You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/09/21 00:27:31 UTC
[21/23] carbondata git commit: [CARBONDATA-1488] JVM crashes when
unsafe columnpage is enabled
[CARBONDATA-1488] JVM crashes when unsafe columnpage is enabled
Fixed capacity size calculation when column value size is larger than initial capacity in this PR.
This closes #1370
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e9c24c50
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e9c24c50
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e9c24c50
Branch: refs/heads/streaming_ingest
Commit: e9c24c50536e7700fe4ce468114820cb21942ee3
Parents: 36ceb59
Author: Ravindra Pesala <ra...@gmail.com>
Authored: Tue Sep 19 16:24:35 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Sep 19 20:13:32 2017 +0800
----------------------------------------------------------------------
.../core/datastore/page/UnsafeVarLengthColumnPage.java | 12 ------------
.../core/datastore/page/VarLengthColumnPageBase.java | 2 +-
.../vectorreader/VectorizedCarbonRecordReader.java | 4 ++--
3 files changed, 3 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9c24c50/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
index c9737a4..9d6e161 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
@@ -42,18 +42,6 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
baseOffset = memoryBlock.getBaseOffset();
}
- /**
- * create a page with initial capacity
- */
- UnsafeVarLengthColumnPage(TableSpec.ColumnSpec columnSpec, DataType dataType, int pageSize,
- int capacity) throws MemoryException {
- super(columnSpec, dataType, pageSize);
- this.capacity = capacity;
- memoryBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, (long)(capacity));
- baseAddress = memoryBlock.getBaseObject();
- baseOffset = memoryBlock.getBaseOffset();
- }
-
@Override
public void freeMemory() {
if (memoryBlock != null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9c24c50/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
index 83e6ef0..276b7ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
@@ -363,7 +363,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
*/
protected void ensureMemory(int requestSize) throws MemoryException {
if (totalLength + requestSize > capacity) {
- int newSize = 2 * capacity;
+ int newSize = Math.max(2 * capacity, totalLength + requestSize);
MemoryBlock newBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, newSize);
CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset,
newBlock.getBaseObject(), newBlock.getBaseOffset(), capacity);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9c24c50/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 5356ad5..e1946a9 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -72,9 +72,9 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
private boolean returnColumnarBatch;
/**
- * The default config on whether columnarBatch should be offheap.
+ * The default config on whether columnarBatch should be onheap.
*/
- private static final MemoryMode DEFAULT_MEMORY_MODE = MemoryMode.OFF_HEAP;
+ private static final MemoryMode DEFAULT_MEMORY_MODE = MemoryMode.ON_HEAP;
private QueryModel queryModel;