You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/12/18 11:05:54 UTC
carbondata git commit: [CARBONDATA-3174]varchar column trailing space
issue fixed
Repository: carbondata
Updated Branches:
refs/heads/master 7cd7623d1 -> f82254021
[CARBONDATA-3174]varchar column trailing space issue fixed
What was the issue?
After doing SDK Write, Select * was failing for 'long_string_columns' with trailing space.
What has been changed?
Removed the trailing space in ColumnName.
This closes #2988
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f8225402
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f8225402
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f8225402
Branch: refs/heads/master
Commit: f8225402134276552b52f433792da3b6772056af
Parents: 7cd7623
Author: Shubh18s <si...@gmail.com>
Authored: Fri Dec 14 14:41:06 2018 +0530
Committer: kunal642 <ku...@gmail.com>
Committed: Tue Dec 18 16:34:53 2018 +0530
----------------------------------------------------------------------
.../TestCreateTableUsingSparkCarbonFileFormat.scala | 2 +-
.../carbondata/sdk/file/CarbonWriterBuilder.java | 5 ++---
.../java/org/apache/carbondata/sdk/file/Field.java | 15 +++++----------
3 files changed, 8 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8225402/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/TestCreateTableUsingSparkCarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/TestCreateTableUsingSparkCarbonFileFormat.scala b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/TestCreateTableUsingSparkCarbonFileFormat.scala
index 607b7d5..0d2fcd8 100644
--- a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/TestCreateTableUsingSparkCarbonFileFormat.scala
+++ b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/TestCreateTableUsingSparkCarbonFileFormat.scala
@@ -428,7 +428,7 @@ class TestCreateTableUsingSparkCarbonFileFormat extends FunSuite with BeforeAndA
val schema = new StringBuilder()
.append("[ \n")
.append(" {\"name\":\"string\"},\n")
- .append(" {\"address\":\"varchar\"},\n")
+ .append(" {\" address \":\"varchar\"},\n")
.append(" {\"age\":\"int\"},\n")
.append(" {\"note\":\"varchar\"}\n")
.append("]")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8225402/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index 5f8cdfe..5c8f701 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -99,7 +99,7 @@ public class CarbonWriterBuilder {
public CarbonWriterBuilder sortBy(String[] sortColumns) {
if (sortColumns != null) {
for (int i = 0; i < sortColumns.length; i++) {
- sortColumns[i] = sortColumns[i].toLowerCase();
+ sortColumns[i] = sortColumns[i].toLowerCase().trim();
}
}
this.sortColumns = sortColumns;
@@ -116,7 +116,7 @@ public class CarbonWriterBuilder {
public CarbonWriterBuilder invertedIndexFor(String[] invertedIndexColumns) {
if (invertedIndexColumns != null) {
for (int i = 0; i < invertedIndexColumns.length; i++) {
- invertedIndexColumns[i] = invertedIndexColumns[i].toLowerCase();
+ invertedIndexColumns[i] = invertedIndexColumns[i].toLowerCase().trim();
}
}
this.invertedIndexColumns = invertedIndexColumns;
@@ -747,7 +747,6 @@ public class CarbonWriterBuilder {
Field[] fields = schema.getFields();
for (int i = 0; i < fields.length; i++) {
if (fields[i] != null) {
- fields[i].updateNameToLowerCase();
if (longStringColumns != null) {
/* Also update the string type to varchar */
if (longStringColumns.contains(fields[i].getFieldName())) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8225402/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
index fb9a6d8..f7fceda 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
@@ -55,7 +55,7 @@ public class Field {
* @param type datatype of field, specified in strings.
*/
public Field(String name, String type) {
- this.name = name;
+ this.name = name.toLowerCase().trim();
if (type.equalsIgnoreCase("string")) {
this.type = DataTypes.STRING;
} else if (type.equalsIgnoreCase("varchar")) {
@@ -90,7 +90,7 @@ public class Field {
}
public Field(String name, String type, List<StructField> fields) {
- this.name = name;
+ this.name = name.toLowerCase().trim();
this.children = fields;
if (type.equalsIgnoreCase("string")) {
this.type = DataTypes.STRING;
@@ -126,13 +126,13 @@ public class Field {
public Field(String name, DataType type, List<StructField> fields) {
- this.name = name;
+ this.name = name.toLowerCase().trim();
this.type = type;
this.children = fields;
}
public Field(String name, DataType type) {
- this.name = name;
+ this.name = name.toLowerCase().trim();
this.type = type;
initComplexTypeChildren();
}
@@ -143,7 +143,7 @@ public class Field {
* @param columnSchema ColumnSchema, Store the information about the column meta data
*/
public Field(ColumnSchema columnSchema) {
- this.name = columnSchema.getColumnName();
+ this.name = columnSchema.getColumnName().toLowerCase().trim();
this.type = columnSchema.getDataType();
children = new LinkedList<>();
schemaOrdinal = columnSchema.getSchemaOrdinal();
@@ -224,11 +224,6 @@ public class Field {
this.type = DataTypes.VARCHAR;
}
- /*can use to change the case of the schema */
- public void updateNameToLowerCase() {
- this.name = name.toLowerCase();
- }
-
private void initComplexTypeChildren() {
if (getDataType().isComplexType()) {
StructField subFields = prepareSubFields(getFieldName(), getDataType());