You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/11/18 00:53:30 UTC
[1/2] incubator-carbondata git commit: Fixed data loading performance
issue
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 2f24b523f -> 32f2d2786
Fixed data loading performance issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9a3b2218
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9a3b2218
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9a3b2218
Branch: refs/heads/master
Commit: 9a3b22183a63324bad3dc2990b304e689adc7c35
Parents: 2f24b52
Author: kumarvishal <ku...@gmail.com>
Authored: Wed Nov 16 18:32:25 2016 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Fri Nov 18 08:32:10 2016 +0800
----------------------------------------------------------------------
.../csvbased/CarbonCSVBasedSeqGenStep.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9a3b2218/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index 23bf6d9..b1ef7fc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -267,7 +267,10 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
private GenericDataType[] complexTypes;
private DirectDictionaryGenerator[] directDictionaryGenerators;
-
+ /**
+ * dimension column ids
+ */
+ private String[] dimensionColumnIds;
/**
* Constructor
*
@@ -350,7 +353,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
if (null != getInputRowMeta()) {
data.setOutputRowMeta((RowMetaInterface) getInputRowMeta().clone());
}
-
+ this.dimensionColumnIds = meta.getDimensionColumnIds();
ColumnsInfo columnsInfo = new ColumnsInfo();
columnsInfo.setDims(meta.dims);
columnsInfo.setDimColNames(meta.dimColNames);
@@ -373,7 +376,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
columnsInfo.setPrimaryKeyMap(meta.getPrimaryKeyMap());
columnsInfo.setMeasureColumns(meta.measureColumn);
columnsInfo.setComplexTypesMap(meta.getComplexTypes());
- columnsInfo.setDimensionColumnIds(meta.getDimensionColumnIds());
+ columnsInfo.setDimensionColumnIds(this.dimensionColumnIds);
columnsInfo.setColumnSchemaDetailsWrapper(meta.getColumnSchemaDetailsWrapper());
columnsInfo.setColumnProperties(meta.getColumnPropertiesMap());
updateBagLogFileName();
@@ -945,7 +948,6 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
int inputColumnsSize = metaColumnNames.length;
boolean isGenerated = false;
int generatedSurrogate = -1;
- String[] dimensionColumnIds = meta.getDimensionColumnIds();
//If CSV Exported from DB and we enter one row down then that row become empty.
// In that case it will have first value empty and other values will be null
[2/2] incubator-carbondata git commit: [CARBONDATA-418]Fixed data
loading performance issue This closes #325
Posted by ja...@apache.org.
[CARBONDATA-418]Fixed data loading performance issue This closes #325
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/32f2d278
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/32f2d278
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/32f2d278
Branch: refs/heads/master
Commit: 32f2d2786e7dcfe6702ae4b40b4f7afc34d3af90
Parents: 2f24b52 9a3b221
Author: jackylk <ja...@huawei.com>
Authored: Fri Nov 18 08:53:19 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Fri Nov 18 08:53:19 2016 +0800
----------------------------------------------------------------------
.../csvbased/CarbonCSVBasedSeqGenStep.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------