You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/06/22 01:34:21 UTC
[20/50] [abbrv] carbondata git commit: [CARBONDATA-2578] fixed memory
leak inside CarbonReader and handled failure case for creation of multi
reader for non-transactional table
[CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure case for creation of
multi reader for non-transactional table
Issue :
CarbonIterator inside CarbonRecordReader was keeping reference of RowBatch and it is not being
closed inside CarbonRecordReader. sort_column with measure was considering all the dimension
column along with given column. if creation of one CarbonReader for non transactional table
is failed then we are not able to create another CarbonReader.
Solution :
close() called inside hasNext() to clear the previous iterator before iterating over the next CarbonReader..
if sortcolumn is not empty and sortcolumnsList contains the fields check is added to finally in sortcolumn props.
Clear the datamap in catch block, if creation of CarbonReader is failed.
This closes #2362
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/19312ab5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/19312ab5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/19312ab5
Branch: refs/heads/carbonstore
Commit: 19312ab5b35654e89f04ec79881d928c859f8a07
Parents: 290ef5a
Author: rahul <ra...@knoldus.in>
Authored: Tue Jun 5 13:09:36 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Tue Jun 12 17:01:53 2018 +0530
----------------------------------------------------------------------
.../createTable/TestNonTransactionalCarbonTable.scala | 7 ++++---
.../java/org/apache/carbondata/sdk/file/CarbonReader.java | 2 ++
.../org/apache/carbondata/sdk/file/CarbonWriterBuilder.java | 5 -----
3 files changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index b275bb8..805fc71 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -368,8 +368,9 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
- checkExistence(sql("describe formatted sdkOutputTable"), true, "age")
-
+ checkExistence(sql("describe formatted sdkOutputTable"), true, "SORT_COLUMNS age")
+ checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS name,age")
+ checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS age,name")
buildTestDataSingleFile()
assert(new File(writerPath).exists())
sql("DROP TABLE IF EXISTS sdkOutputTable")
@@ -402,7 +403,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
intercept[RuntimeException] {
buildTestDataWithSortColumns(List(""))
}
-
+
assert(!(new File(writerPath).exists()))
cleanTestData()
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index 9af710f..be809e6 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -74,6 +74,8 @@ public class CarbonReader<T> {
return false;
} else {
index++;
+ // current reader is closed
+ currentReader.close();
currentReader = readers.get(index);
return currentReader.nextKeyValue();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index bd64568..0f670fe 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -484,11 +484,6 @@ public class CarbonWriterBuilder {
if (isSortColumn > -1) {
columnSchema.setSortColumn(true);
sortColumnsSchemaList[isSortColumn] = columnSchema;
- } else if (!sortColumnsList.isEmpty() && columnSchema.isDimensionColumn()
- && columnSchema.getNumberOfChild() < 1) {
- columnSchema.setSortColumn(true);
- sortColumnsSchemaList[i] = columnSchema;
- i++;
}
}
}