You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/05/02 14:13:13 UTC
[49/50] [abbrv] incubator-carbondata git commit: Added option to
specify single pass load in data frame
Added option to specify single pass load in data frame
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ab85ae68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ab85ae68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ab85ae68
Branch: refs/heads/12-dev
Commit: ab85ae68dc5563fbb3630344dc1bc23a7aab71ac
Parents: 1814048
Author: Sanoj MG <sa...@gmail.com>
Authored: Wed Apr 12 22:32:51 2017 +0400
Committer: jackylk <ja...@huawei.com>
Committed: Tue May 2 09:43:27 2017 +0800
----------------------------------------------------------------------
.../testsuite/dataload/TestLoadDataFrame.scala | 32 ++++++++++++++++++++
.../spark/CarbonDataFrameWriter.scala | 3 +-
2 files changed, 34 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ab85ae68/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index 6e79a10..b790131 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -59,6 +59,8 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS carbon5")
sql("DROP TABLE IF EXISTS carbon6")
sql("DROP TABLE IF EXISTS carbon7")
+ sql("DROP TABLE IF EXISTS carbon8")
+ sql("DROP TABLE IF EXISTS carbon9")
}
@@ -160,6 +162,36 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test load dataframe with single pass enabled") {
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon8")
+ .option("tempCSV", "false")
+ .option("single_pass", "true")
+ .option("compress", "false")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon8 where c3 > 500"), Row(500)
+ )
+ }
+
+ test("test load dataframe with single pass disabled") {
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon9")
+ .option("tempCSV", "true")
+ .option("single_pass", "false")
+ .option("compress", "false")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon9 where c3 > 500"), Row(500)
+ )
+ }
+
override def afterAll {
dropTable
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ab85ae68/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
index 9f813a8..9089be1 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
@@ -194,7 +194,8 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) {
s"""
LOAD DATA INPATH '$csvFolder'
INTO TABLE ${options.dbName}.${options.tableName}
- OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}')
+ OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}',
+ 'SINGLE_PASS' = '${options.singlePass}')
"""
}