You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/05/18 08:52:54 UTC
carbondata git commit: [CARBONDATA-2467] sdk writer log shouldnot
print null
Repository: carbondata
Updated Branches:
refs/heads/master 201804858 -> 78efc7f71
[CARBONDATA-2467] sdk writer log shouldnot print null
issue :
Logger prints the threadname along with log. In case of sdk writer we are assigning null for dbname and table name and table name is used to assign the thread name in CarbonTableOutputFormat and DataWriterProcessStepImpl. which leads to add null in many logs.
Comparision of two array type was being wrong.
Solution :
assign temp value with current time in millisec to tablename instead of null.
ArrayType class donot contains equls method. Hence it goes to Object.equals() which will perform == operation to compare both the object which is wrong . so add equals method to ArrayType class .
This closes #2292
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/78efc7f7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/78efc7f7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/78efc7f7
Branch: refs/heads/master
Commit: 78efc7f7110e152b8f3d2ec833e13278356a6a2d
Parents: 2018048
Author: rahulforallp <ra...@knoldus.in>
Authored: Thu May 10 14:37:44 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri May 18 14:22:44 2018 +0530
----------------------------------------------------------------------
.../core/metadata/datatype/ArrayType.java | 13 +++
.../TestNonTransactionalCarbonTable.scala | 84 ++++++++++++++++++++
.../sdk/file/CarbonWriterBuilder.java | 4 +-
3 files changed, 99 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/78efc7f7/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
index c327d7f..0f0f120 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
@@ -34,4 +34,17 @@ public class ArrayType extends DataType {
public DataType getElementType() {
return elementType;
}
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof ArrayType)) return false;
+
+ ArrayType arrayType = (ArrayType) o;
+
+ return elementType.equals(arrayType.elementType);
+ }
+
+ @Override public int hashCode() {
+ return elementType.hashCode();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/78efc7f7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index cc3cbb5..a15d0c2 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -1331,6 +1331,90 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
buildAvroTestDataStructOfArray(3, null)
}
+ // ArrayOfStruct test
+ def buildAvroTestDataArrayOfStructWithNoSortCol(rows: Int, options: util.Map[String, String]): Any = {
+ FileUtils.deleteDirectory(new File(writerPath))
+
+ val mySchema = """ {
+ | "name": "address",
+ | "type": "record",
+ | "fields": [
+ | {
+ | "name": "exp",
+ | "type": "int"
+ | },
+ | {
+ | "name": "age",
+ | "type": "int"
+ | },
+ | {
+ | "name": "doorNum",
+ | "type": {
+ | "type": "array",
+ | "items": {
+ | "type": "record",
+ | "name": "my_address",
+ | "fields": [
+ | {
+ | "name": "street",
+ | "type": "string"
+ | },
+ | {
+ | "name": "city",
+ | "type": "string"
+ | }
+ | ]
+ | }
+ | }
+ | }
+ | ]
+ |} """.stripMargin
+ val json =
+ """ {"exp":5,"age":10,"doorNum" :
+ |[{"street":"abc","city":"city1"},
+ |{"street":"def","city":"city2"},
+ |{"street":"ghi","city":"city3"},
+ |{"street":"jkl","city":"city4"}]} """.stripMargin
+
+ val fields = new Array[Field](3)
+ fields(0) = new Field("exp", DataTypes.INT)
+ fields(1) = new Field("age", DataTypes.INT)
+
+ val fld = new util.ArrayList[StructField]
+ fld.add(new StructField("street", DataTypes.STRING))
+ fld.add(new StructField("city", DataTypes.STRING))
+
+ val fld2 = new util.ArrayList[StructField]
+ fld2.add(new StructField("my_address", DataTypes.createStructType(fld), fld))
+ fields(2) = new Field("doorNum", DataTypes.createArrayType(fld2.get(0).getDataType), fld2)
+
+ WriteFilesWithAvroWriter(rows, mySchema, json, fields)
+ }
+
+ test("Read sdk writer Avro output Record Type with no sort columns") {
+ buildAvroTestDataArrayOfStructWithNoSortCol(3,null)
+ assert(new File(writerPath).exists())
+ sql("DROP TABLE IF EXISTS sdkOutputTable")
+ sql(
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY 'carbondata' LOCATION
+ |'$writerPath' """.stripMargin)
+
+ sql("desc formatted sdkOutputTable").show(false)
+ sql("select * from sdkOutputTable").show(false)
+
+ /*
+ +---+---+----------------------------------------------------+
+ |exp|age|doorNum |
+ +---+---+----------------------------------------------------+
+ |5 |10 |[[abc,city1], [def,city2], [ghi,city3], [jkl,city4]]|
+ |5 |10 |[[abc,city1], [def,city2], [ghi,city3], [jkl,city4]]|
+ |5 |10 |[[abc,city1], [def,city2], [ghi,city3], [jkl,city4]]|
+ +---+---+----------------------------------------------------+
+ */
+ sql("DROP TABLE sdkOutputTable")
+ // drop table should not delete the files
+ assert(new File(writerPath).listFiles().length > 0)
+ }
test("Read sdk writer Avro output Record Type") {
buildAvroTestDataStructType()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/78efc7f7/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index 36be65f..0f9c9d7 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -414,8 +414,8 @@ public class CarbonWriterBuilder {
tableName = "_tempTable";
dbName = "_tempDB";
} else {
- dbName = null;
- tableName = null;
+ dbName = "";
+ tableName = "_tempTable_" + String.valueOf(UUID);
}
TableSchema schema = tableSchemaBuilder.build();
schema.setTableName(tableName);