You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2017/05/09 07:14:27 UTC
[1/2] carbondata git commit: [CARBONDATA-1027] insert into/data load
failing for numeric dictionary included column having null value
Repository: carbondata
Updated Branches:
refs/heads/master 77dab1d7f -> c677c5b25
[CARBONDATA-1027] insert into/data load failing for numeric dictionary included column having null value
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b41d4404
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b41d4404
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b41d4404
Branch: refs/heads/master
Commit: b41d44045f200157c6b63b5dc4995a35d1668f99
Parents: 77dab1d
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Fri May 5 16:43:23 2017 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Tue May 9 15:13:30 2017 +0800
----------------------------------------------------------------------
.../NumericDimensionBadRecordTest.scala | 23 +++++++++++++++++++-
.../impl/DictionaryFieldConverterImpl.java | 6 +++--
2 files changed, 26 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b41d4404/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala
index 1301f5d..e8590dd 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala
@@ -99,7 +99,26 @@ class NumericDimensionBadRecordTest extends QueryTest with BeforeAndAfterAll {
}
}
- test("select count(*) from intDataType") {
+ test("insert into numeric dictionary col having null values") {
+ sql("drop table if exists num_dic")
+ sql("drop table if exists num_dicc")
+ sql("create table num_dic(cust_name string, cust_id int) " +
+ "row format delimited fields terminated by ','")
+ sql("""insert into num_dic select 'sam','\N'""")
+ sql("select * from num_dic").show()
+ sql("create table num_dicc(cust_name string, cust_id int) stored by 'carbondata'" +
+ " TBLPROPERTIES('DICTIONARY_INCLUDE'='cust_id')")
+ try {
+ sql("insert into table num_dicc select * from num_dic")
+ } catch {
+ case x : Throwable => {
+ System.out.println(x)
+ assert(false)
+ }
+ }
+ }
+
+ test("select count(*) from intDataType") {
checkAnswer(
sql("select count(*) from intDataType"),
Seq(Row(2)
@@ -155,6 +174,8 @@ class NumericDimensionBadRecordTest extends QueryTest with BeforeAndAfterAll {
sql("drop table IF EXISTS floatDataType")
sql("drop table IF EXISTS bigDecimalDataType")
sql("drop table IF EXISTS stringDataType")
+ sql("drop table if exists num_dic")
+ sql("drop table if exists num_dicc")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b41d4404/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
index 62073e2..23e9c8c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
@@ -97,11 +97,13 @@ public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConvert
@Override public void convert(CarbonRow row, BadRecordLogHolder logHolder)
throws CarbonDataLoadingException {
try {
+ String parsedValue = null;
String dimensionValue = row.getString(index);
if (dimensionValue == null || dimensionValue.equals(nullFormat)) {
- dimensionValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
+ parsedValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
+ } else {
+ parsedValue = DataTypeUtil.parseValue(dimensionValue, carbonDimension);
}
- String parsedValue = DataTypeUtil.parseValue(dimensionValue, carbonDimension);
if (null == parsedValue) {
if ((dimensionValue.length() > 0) || (dimensionValue.length() == 0 && isEmptyBadRecord)) {
String dataType = carbonDimension.getDataType().getName();
[2/2] carbondata git commit: [CARBONDATA-1027] insert into/data load
failing for numeric dictionary included column having null value This closes
#886
Posted by ch...@apache.org.
[CARBONDATA-1027] insert into/data load failing for numeric dictionary included column having null value This closes #886
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c677c5b2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c677c5b2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c677c5b2
Branch: refs/heads/master
Commit: c677c5b259b3a09f8defaf736d58fd9744c4c899
Parents: 77dab1d b41d440
Author: chenliang613 <ch...@huawei.com>
Authored: Tue May 9 15:14:13 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Tue May 9 15:14:13 2017 +0800
----------------------------------------------------------------------
.../NumericDimensionBadRecordTest.scala | 23 +++++++++++++++++++-
.../impl/DictionaryFieldConverterImpl.java | 6 +++--
2 files changed, 26 insertions(+), 3 deletions(-)
----------------------------------------------------------------------