You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/03/30 05:17:33 UTC
[1/2] incubator-carbondata git commit: Fixed data loading issue with
duplicate column
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 731908954 -> 32bb7fef1
Fixed data loading issue with duplicate column
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/71fe32a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/71fe32a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/71fe32a4
Branch: refs/heads/master
Commit: 71fe32a474caff36b33db19a8aaca711522d2e2d
Parents: 7319089
Author: kumarvishal <ku...@gmail.com>
Authored: Tue Mar 21 22:35:07 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Thu Mar 30 10:46:31 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/util/DataTypeUtil.java | 19 ++++++++++---------
.../newflow/parser/impl/RowParserImpl.java | 11 +++++------
2 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/71fe32a4/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 3162d84..10411b0 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -324,7 +324,7 @@ public final class DataTypeUtil {
* Below method will be used to convert the data passed to its actual data
* type
*
- * @param dataInBytes data
+ * @param dataInBytes data
* @param dimension
* @return actual data after conversion
*/
@@ -391,8 +391,7 @@ public final class DataTypeUtil {
}
java.math.BigDecimal javaDecVal = new java.math.BigDecimal(data7);
if (dimension.getColumnSchema().getScale() > javaDecVal.scale()) {
- javaDecVal =
- javaDecVal.setScale(dimension.getColumnSchema().getScale());
+ javaDecVal = javaDecVal.setScale(dimension.getColumnSchema().getScale());
}
return org.apache.spark.sql.types.Decimal.apply(javaDecVal);
default:
@@ -510,6 +509,9 @@ public final class DataTypeUtil {
* @return
*/
public static String parseValue(String value, CarbonDimension dimension) {
+ if (null == value) {
+ return null;
+ }
try {
switch (dimension.getDataType()) {
case DECIMAL:
@@ -617,9 +619,9 @@ public final class DataTypeUtil {
/**
* This method will parse a given string value corresponding to its data type
*
- * @param value value to parse
+ * @param value value to parse
* @param columnSchema dimension to get data type and precision and scale in case of decimal
- * data type
+ * data type
* @return
*/
public static String normalizeColumnValueForItsDataType(String value, ColumnSchema columnSchema) {
@@ -649,10 +651,9 @@ public final class DataTypeUtil {
}
private static String parseStringToBigDecimal(String value, ColumnSchema columnSchema) {
- BigDecimal bigDecimal = new BigDecimal(value)
- .setScale(columnSchema.getScale(), RoundingMode.HALF_UP);
- BigDecimal normalizedValue =
- normalizeDecimalValue(bigDecimal, columnSchema.getPrecision());
+ BigDecimal bigDecimal =
+ new BigDecimal(value).setScale(columnSchema.getScale(), RoundingMode.HALF_UP);
+ BigDecimal normalizedValue = normalizeDecimalValue(bigDecimal, columnSchema.getPrecision());
if (null != normalizedValue) {
return normalizedValue.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/71fe32a4/processing/src/main/java/org/apache/carbondata/processing/newflow/parser/impl/RowParserImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/parser/impl/RowParserImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/parser/impl/RowParserImpl.java
index 5e7aaa6..61e4a31 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/parser/impl/RowParserImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/parser/impl/RowParserImpl.java
@@ -63,12 +63,11 @@ public class RowParserImpl implements RowParser {
DataField[] input = new DataField[fields.length];
inputMapping = new int[input.length];
int k = 0;
- // value of k can be max equivalent to the number of columns in schema
- for (int i = 0; i < numberOfColumns && k < fields.length; i++) {
- for (int j = 0; j < fields.length; j++) {
- if (header[i].equalsIgnoreCase(fields[j].getColumn().getColName())) {
- input[k] = fields[j];
- inputMapping[k] = i;
+ for (int i = 0; i < fields.length; i++) {
+ for (int j = 0; j < numberOfColumns; j++) {
+ if (header[j].equalsIgnoreCase(fields[i].getColumn().getColName())) {
+ input[k] = fields[i];
+ inputMapping[k] = j;
k++;
break;
}
[2/2] incubator-carbondata git commit: [CARBONDATA-832] Fixed data
loading issue with duplicate column in Csv header This closes #682
Posted by ja...@apache.org.
[CARBONDATA-832] Fixed data loading issue with duplicate column in Csv header This closes #682
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/32bb7fef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/32bb7fef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/32bb7fef
Branch: refs/heads/master
Commit: 32bb7fef13ab6810a48584f045f7131674e91906
Parents: 7319089 71fe32a
Author: jackylk <ja...@huawei.com>
Authored: Thu Mar 30 10:47:09 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Thu Mar 30 10:47:09 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/util/DataTypeUtil.java | 19 ++++++++++---------
.../newflow/parser/impl/RowParserImpl.java | 11 +++++------
2 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------