You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/03/21 10:16:19 UTC
carbondata git commit: [CARBONDATA-2265] [DFX]-Load]: Load job fails
if 1 folder contains 1000 files
Repository: carbondata
Updated Branches:
refs/heads/master dfc5e8c53 -> df002af3d
[CARBONDATA-2265] [DFX]-Load]: Load job fails if 1 folder contains 1000 files
Problem : We are keeping the rawTablePages also in memory, but we are no where using it
Solution : Removed storing rawTablePages
This closes #2082
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/df002af3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/df002af3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/df002af3
Branch: refs/heads/master
Commit: df002af3d60abc73a4ffd74fbb5d505a2e33196e
Parents: dfc5e8c
Author: dhatchayani <dh...@gmail.com>
Authored: Tue Mar 20 18:11:58 2018 +0530
Committer: kunal642 <ku...@gmail.com>
Committed: Wed Mar 21 15:43:45 2018 +0530
----------------------------------------------------------------------
.../processing/store/writer/v3/BlockletDataHolder.java | 7 -------
.../store/writer/v3/CarbonFactDataWriterImplV3.java | 5 +++--
2 files changed, 3 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/df002af3/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java
index 68aee95..36fda3c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java
@@ -24,24 +24,20 @@ import org.apache.carbondata.processing.store.TablePage;
public class BlockletDataHolder {
private List<EncodedTablePage> encodedTablePage;
- private List<TablePage> rawTablePages;
private long currentSize;
public BlockletDataHolder() {
this.encodedTablePage = new ArrayList<>();
- this.rawTablePages = new ArrayList<>();
}
public void clear() {
encodedTablePage.clear();
- rawTablePages.clear();
currentSize = 0;
}
public void addPage(TablePage rawTablePage) {
EncodedTablePage encodedTablePage = rawTablePage.getEncodedTablePage();
this.encodedTablePage.add(encodedTablePage);
- this.rawTablePages.add(rawTablePage);
currentSize += encodedTablePage.getEncodedSize();
}
@@ -66,7 +62,4 @@ public class BlockletDataHolder {
return encodedTablePage;
}
- public List<TablePage> getRawTablePages() {
- return rawTablePages;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/df002af3/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index 3e9be7e..d1deef1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -189,9 +189,10 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter {
} catch (IOException e) {
LOGGER.error(e, "Problem while writing file");
throw new CarbonDataWriterException("Problem while writing file", e);
+ } finally {
+ // clear the data holder
+ blockletDataHolder.clear();
}
- // clear the data holder
- blockletDataHolder.clear();
}