You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/10/09 03:19:19 UTC
carbondata git commit: [CARBONDATA-2912] Support CSV table load csv
data with spark2.2
Repository: carbondata
Updated Branches:
refs/heads/branch-1.3 9d8648ef8 -> 948ce2ebe
[CARBONDATA-2912] Support CSV table load csv data with spark2.2
1.upgrade commons-lang3 version
2.add some test case
This closes #2691
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/948ce2eb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/948ce2eb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/948ce2eb
Branch: refs/heads/branch-1.3
Commit: 948ce2ebe6896a7790f74fd8e8c1882d2793fb33
Parents: 9d8648e
Author: xubo245 <xu...@huawei.com>
Authored: Tue Sep 4 16:37:37 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Oct 9 11:18:52 2018 +0800
----------------------------------------------------------------------
.../InsertIntoCarbonTableTestCase.scala | 48 ++++++++++++++++++++
processing/pom.xml | 2 +-
2 files changed, 49 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/948ce2eb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index d59f0b5..0de3a43 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -85,6 +85,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("select * from TCarbon")
)
}
+
test("insert->carbon column is more then hive-fails") {
sql("drop table if exists TCarbon")
sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
@@ -92,6 +93,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("insert into TCarbon select imei,deviceInformationId,MAC,deviceColor,gamePointId from THive")
}
}
+
test("insert->insert wrong data types-pass") {
sql("drop table if exists TCarbon")
sql("create table TCarbon (imei string,deviceInformationId int,MAC string) STORED BY 'org.apache.carbondata.format'")
@@ -372,6 +374,52 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select count(*) from OverwriteTable_t1"), sql("select count(*) from OverwriteTable_t2"))
}
+ test("two insert into: insert into carbon table from default table") {
+ sql("drop table if exists carbon_1")
+ sql("drop table if exists carbon_2")
+ sql("create table carbon_1(name String,age int)")
+ sql("create table carbon_2(name String,age int)")
+ sql("insert into carbon_1 values('Bob',27)")
+ sql("insert into carbon_1 values('David',33)")
+ sql("insert into carbon_1 values('Jack',37)")
+ sql("from carbon_1 insert into carbon_2 select * where age<30 insert into carbon_2 select * where age>35")
+ checkAnswer(sql("select * from carbon_1"),
+ Seq(Row("Bob", 27), Row("David", 33), Row("Jack", 37)))
+ checkAnswer(sql("select * from carbon_2"),
+ Seq(Row("Bob", 27), Row("Jack", 37)))
+ }
+
+ test("two insert into: insert into carbon table from csv table") {
+ sql("drop table if exists carbon_1")
+ sql("drop table if exists carbon_2")
+ sql("create table carbon_1(name String,age int) using csv")
+ sql("create table carbon_2(name String,age int) stored by 'carbondata'")
+ sql("insert into carbon_1 values('Bob',27)")
+ sql("insert into carbon_1 values('David',33)")
+ sql("insert into carbon_1 values('Jack',37)")
+ sql("from carbon_1 insert into carbon_2 select * where age<30 insert into carbon_2 select * where age>35")
+ checkAnswer(sql("select * from carbon_1"),
+ Seq(Row("Bob", 27), Row("David", 33), Row("Jack", 37)))
+ checkAnswer(sql("select * from carbon_2"),
+ Seq(Row("Bob", 27), Row("Jack", 37)))
+ }
+
+ test("two insert into: insert into carbon table from csv table and load data") {
+ sql("drop table if exists carbon_1")
+ sql("drop table if exists carbon_2")
+ sql(
+ s"""
+ | create table carbon_1(name String,age int)
+ | using csv
+ | options(path "$resourcesPath/emptylines.csv",header "true")""".stripMargin)
+ sql("create table carbon_2(name String,age int) stored by 'carbondata'")
+ sql("from carbon_1 insert into carbon_2 select * where age<23 insert into carbon_2 select * where age>24")
+ checkAnswer(sql("select * from carbon_1"),
+ Seq(Row("a", 25), Row("b", 22), Row("c", 23)))
+ checkAnswer(sql("select * from carbon_2"),
+ Seq(Row("a", 25), Row("b", 22)))
+ }
+
private def queryExecution(csvFileName1: String , csvFileName2: String) : Unit ={
sql("drop table if exists OverwriteTable_t1")
sql("drop table if exists OverwriteTable_t2")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/948ce2eb/processing/pom.xml
----------------------------------------------------------------------
diff --git a/processing/pom.xml b/processing/pom.xml
index f4a9a6c..0ceb756 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -51,7 +51,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.3.2</version>
+ <version>3.5</version>
</dependency>
<dependency>
<groupId>org.jmockit</groupId>