You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/11/15 15:45:34 UTC
[1/2] incubator-carbondata git commit: add no-kettle loading example
Repository: incubator-carbondata
Updated Branches:
refs/heads/master e60c9255f -> 24978ed7e
add no-kettle loading example
fix comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e84cd930
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e84cd930
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e84cd930
Branch: refs/heads/master
Commit: e84cd930d25961aff65588529316a7d93a21f34b
Parents: e60c925
Author: jackylk <ja...@huawei.com>
Authored: Thu Nov 10 23:20:52 2016 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Tue Nov 15 23:42:55 2016 +0800
----------------------------------------------------------------------
.../carbondata/examples/CarbonExample.scala | 21 ++++++++++++++++++++
.../carbondata/examples/util/ExampleUtils.scala | 1 +
.../spark/CarbonDataFrameWriter.scala | 10 +++++++++-
3 files changed, 31 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e84cd930/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
index df0a1e2..7c769ff 100644
--- a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
+++ b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
@@ -32,6 +32,7 @@ object CarbonExample {
cc.sql("DROP TABLE IF EXISTS t3")
+ // Create table, 6 dimensions, 1 measure
cc.sql("""
CREATE TABLE IF NOT EXISTS t3
(ID Int, date Timestamp, country String,
@@ -39,10 +40,15 @@ object CarbonExample {
STORED BY 'carbondata'
""")
+ // Currently there are two data loading flows in CarbonData, one uses Kettle as ETL tool
+ // in each node to do data loading, another uses a multi-thread framework without Kettle\uff08See
+ // AbstractDataLoadProcessorStep)
+ // Load data with Kettle
cc.sql(s"""
LOAD DATA LOCAL INPATH '$testData' into table t3
""")
+ // Perform a query
cc.sql("""
SELECT country, count(salary) AS amount
FROM t3
@@ -50,6 +56,21 @@ object CarbonExample {
GROUP BY country
""").show()
+ // Load data without kettle
+ cc.sql(s"""
+ LOAD DATA LOCAL INPATH '$testData' into table t3
+ OPTIONS('USE_KETTLE'='false')
+ """)
+
+ // Perform a query
+ cc.sql("""
+ SELECT country, count(salary) AS amount
+ FROM t3
+ WHERE country IN ('china','france')
+ GROUP BY country
+ """).show()
+
+ // Drop table
cc.sql("DROP TABLE IF EXISTS t3")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e84cd930/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala b/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
index cfcdde8..83010b6 100644
--- a/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
+++ b/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
@@ -84,6 +84,7 @@ object ExampleUtils {
.format("carbondata")
.option("tableName", tableName)
.option("compress", "true")
+ .option("useKettle", "false")
.mode(mode)
.save()
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e84cd930/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
index c2b2190..a02751e 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
@@ -161,11 +161,19 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) extends Logging {
}
private def makeLoadString(csvFolder: String, options: CarbonOption): String = {
- s"""
+ if (options.useKettle) {
+ s"""
LOAD DATA INPATH '$csvFolder'
INTO TABLE ${options.dbName}.${options.tableName}
OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}')
"""
+ } else {
+ s"""
+ LOAD DATA INPATH '$csvFolder'
+ INTO TABLE ${options.dbName}.${options.tableName}
+ OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}', 'USE_KETTLE' = 'false')
+ """
+ }
}
}
[2/2] incubator-carbondata git commit: [CARBONDATA-403] Add example
for data load without using kettle This closes #311
Posted by ch...@apache.org.
[CARBONDATA-403] Add example for data load without using kettle This closes #311
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/24978ed7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/24978ed7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/24978ed7
Branch: refs/heads/master
Commit: 24978ed7efab881eaa4dabe7333b2419a517ac05
Parents: e60c925 e84cd93
Author: chenliang613 <ch...@apache.org>
Authored: Tue Nov 15 23:44:27 2016 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Tue Nov 15 23:44:27 2016 +0800
----------------------------------------------------------------------
.../carbondata/examples/CarbonExample.scala | 21 ++++++++++++++++++++
.../carbondata/examples/util/ExampleUtils.scala | 1 +
.../spark/CarbonDataFrameWriter.scala | 10 +++++++++-
3 files changed, 31 insertions(+), 1 deletion(-)
----------------------------------------------------------------------