You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/12/26 02:47:12 UTC
[1/2] incubator-carbondata git commit: add example:directly write
dataframe to carbonfile without temp CSV
Repository: incubator-carbondata
Updated Branches:
refs/heads/master a0e66461b -> 2ee23acf4
add example:directly write dataframe to carbonfile without temp CSV
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9bbc937a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9bbc937a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9bbc937a
Branch: refs/heads/master
Commit: 9bbc937adaadb88c4395b295916036724491bb8f
Parents: a0e6646
Author: chenliang613 <ch...@apache.org>
Authored: Sun Dec 25 12:04:26 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Dec 26 10:46:19 2016 +0800
----------------------------------------------------------------------
.../carbondata/examples/AllDictionaryExample.scala | 2 +-
.../apache/carbondata/examples/HadoopFileExample.scala | 2 +-
.../org/apache/carbondata/examples/PerfTest.scala | 6 ++----
.../apache/carbondata/examples/util/ExampleUtils.scala | 13 ++++++++++++-
4 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9bbc937a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
index 9fecadb..4dcc868 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
@@ -23,7 +23,7 @@ import org.apache.carbondata.examples.util.{AllDictionaryUtil, ExampleUtils}
object AllDictionaryExample {
def main(args: Array[String]) {
- val cc = ExampleUtils.createCarbonContext("CarbonExample")
+ val cc = ExampleUtils.createCarbonContext("AllDictionaryExample")
val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
val csvHeader = "ID,date,country,name,phonetype,serialname,salary"
val dictCol = "|date|country|name|phonetype|serialname|"
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9bbc937a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
index 329b3c9..292a3b5 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
@@ -24,7 +24,7 @@ import org.apache.carbondata.hadoop.CarbonInputFormat
object HadoopFileExample {
def main(args: Array[String]): Unit = {
- val cc = ExampleUtils.createCarbonContext("DataFrameAPIExample")
+ val cc = ExampleUtils.createCarbonContext("HadoopFileExample")
ExampleUtils.writeSampleCarbonFile(cc, "carbon1")
val sc = cc.sparkContext
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9bbc937a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
index b1f6b24..824730f 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
@@ -145,15 +145,13 @@ class QueryRunner(sqlContext: SQLContext, dataFrame: DataFrame, datasources: Seq
def shutDown(): Unit = {
// drop all tables and temp files
- datasources.foreach { datasource =>
- datasource match {
- case "parquet" | "orc" =>
+ datasources.foreach {
+ case datasource @ ("parquet" | "orc") =>
val f = new File(PerfTest.savePath(datasource))
if (f.exists()) f.delete()
case "carbon" =>
sqlContext.sql(s"DROP TABLE IF EXISTS ${PerfTest.makeTableName("carbon")}")
case _ => sys.error("unsupported data source")
- }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9bbc937a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
index 653d78b..7649ac3 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
@@ -81,7 +81,7 @@ object ExampleUtils {
.map(x => ("a", "b", x))
.toDF("c1", "c2", "c3")
- // save dataframe to carbon file
+ // save dataframe to carbon file:(df->csv->carbon file)
df.write
.format("carbondata")
.option("tableName", tableName)
@@ -89,6 +89,17 @@ object ExampleUtils {
.option("useKettle", "false")
.mode(mode)
.save()
+
+ // save dataframe directl to carbon file without tempCSV
+ df.write
+ .format("carbondata")
+ .option("tableName", tableName)
+ .option("compress", "true")
+ .option("useKettle", "false")
+ .option("tempCSV", "false")
+ .mode(mode)
+ .save()
+
}
}
// scalastyle:on println
[2/2] incubator-carbondata git commit: Add example:directly write
dataframe to carbon file without temp CSV This closes #462
Posted by ja...@apache.org.
Add example:directly write dataframe to carbon file without temp CSV This closes #462
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2ee23acf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2ee23acf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2ee23acf
Branch: refs/heads/master
Commit: 2ee23acf4194b855a5264d17b57b48d5c869cf34
Parents: a0e6646 9bbc937
Author: jackylk <ja...@huawei.com>
Authored: Mon Dec 26 10:46:53 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Dec 26 10:46:53 2016 +0800
----------------------------------------------------------------------
.../carbondata/examples/AllDictionaryExample.scala | 2 +-
.../apache/carbondata/examples/HadoopFileExample.scala | 2 +-
.../org/apache/carbondata/examples/PerfTest.scala | 6 ++----
.../apache/carbondata/examples/util/ExampleUtils.scala | 13 ++++++++++++-
4 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------