You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/02/11 10:52:03 UTC
[1/2] incubator-carbondata git commit: Create DataFrame example in
example/spark2, read carbon data to dataframe
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 5639e1834 -> b1364398d
Create DataFrame example in example/spark2, read carbon data to dataframe
Create DataFrame example in example/spark2, read carbon data to dataframe
Create DataFrame example in example/spark2, read carbon data to dataframe
Create CarbonDataFrameExample in example/spark2
fix scalastyle
trigger travis ci
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/efe62fac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/efe62fac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/efe62fac
Branch: refs/heads/master
Commit: efe62fac227c84e97857ec3398a7d89540f9745e
Parents: 5639e18
Author: chenliang613 <ch...@huawei.com>
Authored: Wed Feb 8 00:06:29 2017 -0500
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Feb 11 16:20:33 2017 +0530
----------------------------------------------------------------------
.../examples/CarbonDataFrameExample.scala | 89 ++++++++++++++++++++
1 file changed, 89 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/efe62fac/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
new file mode 100644
index 0000000..e4d1646
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.spark.sql.{SaveMode, SparkSession}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+// scalastyle:off println
+object CarbonDataFrameExample {
+
+ def main(args: Array[String]) {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/examples/spark2/target/store"
+ val warehouse = s"$rootPath/examples/spark2/target/warehouse"
+ val metastoredb = s"$rootPath/examples/spark2/target"
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ import org.apache.spark.sql.CarbonSession._
+ val spark = SparkSession
+ .builder()
+ .master("local")
+ .appName("CarbonDataFrameExample")
+ .config("spark.sql.warehouse.dir", warehouse)
+ .getOrCreateCarbonSession(storeLocation, metastoredb)
+
+ spark.sparkContext.setLogLevel("ERROR")
+
+ // Writes Dataframe to CarbonData file:
+ import spark.implicits._
+ val df = spark.sparkContext.parallelize(1 to 100)
+ .map(x => ("a", "b", x))
+ .toDF("c1", "c2", "number")
+
+ // Saves dataframe to carbondata file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon_table")
+ .option("compress", "true")
+ .option("tempCSV", "false")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ spark.sql(""" SELECT * FROM carbon_table """).show()
+
+ // Specify schema
+ import org.apache.spark.sql.types.{StructType, StructField, StringType, IntegerType}
+ val customSchema = StructType(Array(
+ StructField("c1", StringType),
+ StructField("c2", StringType),
+ StructField("number", IntegerType)))
+
+ // Reads carbondata to dataframe
+ val carbondf = spark.read
+ .format("carbondata")
+ .schema(customSchema)
+ .option("tableName", "carbon_table")
+ .load()
+
+ // Dataframe operations
+ carbondf.printSchema()
+ carbondf.select($"c1", $"number" + 10).show()
+ carbondf.filter($"number" > 31).show()
+
+ spark.sql("DROP TABLE IF EXISTS carbon_table")
+ }
+}
+// scalastyle:on println
[2/2] incubator-carbondata git commit: [CARBONDATA-695] Create
CarbonDataFrameExample in example/spark2 This closes #592
Posted by ra...@apache.org.
[CARBONDATA-695] Create CarbonDataFrameExample in example/spark2 This closes #592
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b1364398
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b1364398
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b1364398
Branch: refs/heads/master
Commit: b1364398d70903cf85150d7fc5ee481d50535a20
Parents: 5639e18 efe62fa
Author: ravipesala <ra...@gmail.com>
Authored: Sat Feb 11 16:21:28 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Feb 11 16:21:28 2017 +0530
----------------------------------------------------------------------
.../examples/CarbonDataFrameExample.scala | 89 ++++++++++++++++++++
1 file changed, 89 insertions(+)
----------------------------------------------------------------------