You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/10/10 03:07:53 UTC
[06/50] [abbrv] carbondata git commit: [CARBONDATA-1515]Fixed NPE in
data loading in long run
[CARBONDATA-1515]Fixed NPE in data loading in long run
Scenario:
Data size: 3.5 billion rows(4.1 tb data)
3 node cluster
Number of core while data loading 12.
No. of loads 100 times
Problem: In DataConverterProcessorStepImpl it is using array list for adding all the local converter, in case of multiple thread scenario it is creating a hole (null value)(as array list if not synchronized). while closing the converter it is it is throwing NPE
Solution: Add local converter in synchronized block
This closes #1384
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2ec69f61
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2ec69f61
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2ec69f61
Branch: refs/heads/streaming_ingest
Commit: 2ec69f6179832a394886f26b921f1dc3e916abe8
Parents: 2cd22e1
Author: kumarvishal <ku...@gmail.com>
Authored: Mon Sep 25 18:27:03 2017 +0530
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Tue Sep 26 20:16:51 2017 +0530
----------------------------------------------------------------------
.../newflow/steps/DataConverterProcessorStepImpl.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ec69f61/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index de96cc9..728aa18 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -87,7 +87,9 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
if (first) {
first = false;
localConverter = converters.get(0).createCopyForNewThread();
- converters.add(localConverter);
+ synchronized (converters) {
+ converters.add(localConverter);
+ }
}
return childIter.hasNext();
}
@@ -185,7 +187,9 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
super.close();
if (converters != null) {
for (RowConverter converter : converters) {
- converter.finish();
+ if (null != converter) {
+ converter.finish();
+ }
}
}
}