You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/05/02 14:12:55 UTC
[31/50] [abbrv] incubator-carbondata git commit: update the examples
update the examples
just update examples
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2e8ef57d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2e8ef57d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2e8ef57d
Branch: refs/heads/12-dev
Commit: 2e8ef57d5a72e7c35fbf851ca28f114501028d17
Parents: b728b8f
Author: chenerlu <ch...@huawei.com>
Authored: Thu Apr 27 19:20:09 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Apr 28 17:12:57 2017 +0800
----------------------------------------------------------------------
.../carbondata/examples/AlterTableExample.scala | 32 ++--
.../examples/CarbonDataFrameExample.scala | 4 +-
.../examples/CarbonSessionExample.scala | 114 +++++++-------
.../examples/SparkSessionExample.scala | 156 +++++++++----------
4 files changed, 158 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
index f37a83d..dd2a28a 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
@@ -24,6 +24,10 @@ import org.apache.spark.sql.SparkSession
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
+/**
+ * For alter table relative syntax, you can refer to DDL operation
+ * document (ddl-operation-on-carbondata.md)
+ */
object AlterTableExample {
def main(args: Array[String]): Unit = {
@@ -55,17 +59,17 @@ object AlterTableExample {
spark.sql(
s"""
| CREATE TABLE carbon_table(
- | shortField short,
- | intField int,
- | bigintField long,
- | doubleField double,
- | stringField string,
- | timestampField timestamp,
- | decimalField decimal(18,2),
- | dateField date,
- | charField char(5),
- | floatField float,
- | complexData array<string>
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
| )
| STORED BY 'carbondata'
| TBLPROPERTIES('DICTIONARY_INCLUDE'='dateField, charField')
@@ -73,11 +77,11 @@ object AlterTableExample {
// Alter table change data type
spark.sql("DESCRIBE FORMATTED carbon_table").show()
- spark.sql("ALTER TABLE carbon_table CHANGE intField intField bigint").show()
+ spark.sql("ALTER TABLE carbon_table CHANGE intField intField BIGINT").show()
// Alter table add columns
spark.sql("DESCRIBE FORMATTED carbon_table").show()
- spark.sql("ALTER TABLE carbon_table ADD COLUMNS (newField String) " +
+ spark.sql("ALTER TABLE carbon_table ADD COLUMNS (newField STRING) " +
"TBLPROPERTIES ('DEFAULT.VALUE.newField'='def')").show()
// Alter table drop columns
@@ -93,5 +97,7 @@ object AlterTableExample {
spark.sql("DROP TABLE IF EXISTS carbon_table")
spark.sql("DROP TABLE IF EXISTS new_carbon_table")
+ spark.stop()
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
index e4d1646..74740e7 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
@@ -24,7 +24,6 @@ import org.apache.spark.sql.{SaveMode, SparkSession}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-// scalastyle:off println
object CarbonDataFrameExample {
def main(args: Array[String]) {
@@ -84,6 +83,7 @@ object CarbonDataFrameExample {
carbondf.filter($"number" > 31).show()
spark.sql("DROP TABLE IF EXISTS carbon_table")
+
+ spark.stop()
}
}
-// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index 2ae9341..371b9bb 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -53,17 +53,17 @@ object CarbonSessionExample {
spark.sql(
s"""
| CREATE TABLE carbon_table(
- | shortField short,
- | intField int,
- | bigintField long,
- | doubleField double,
- | stringField string,
- | timestampField timestamp,
- | decimalField decimal(18,2),
- | dateField date,
- | charField char(5),
- | floatField float,
- | complexData array<string>
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
| )
| STORED BY 'carbondata'
| TBLPROPERTIES('DICTIONARY_INCLUDE'='dateField, charField')
@@ -76,63 +76,69 @@ object CarbonSessionExample {
s"""
| LOAD DATA LOCAL INPATH '$path'
| INTO TABLE carbon_table
- | options('FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData','COMPLEX_DELIMITER_LEVEL_1'='#')
+ | OPTIONS('FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData',
+ | 'COMPLEX_DELIMITER_LEVEL_1'='#')
""".stripMargin)
// scalastyle:on
- spark.sql("""
- SELECT *
- FROM carbon_table
- where stringfield = 'spark' and decimalField > 40
- """).show
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ | WHERE stringfield = 'spark' AND decimalField > 40
+ """.stripMargin).show()
- spark.sql("""
- SELECT *
- FROM carbon_table where length(stringField) = 5
- """).show
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table WHERE length(stringField) = 5
+ """.stripMargin).show()
+
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table WHERE date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
+ """.stripMargin).show()
- spark.sql("""
- SELECT *
- FROM carbon_table where date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
- """).show
+ spark.sql("SELECT count(stringField) FROM carbon_table").show()
- spark.sql("""
- select count(stringField) from carbon_table
- """.stripMargin).show
+ spark.sql(
+ s"""
+ | SELECT sum(intField), stringField
+ | FROM carbon_table
+ | GROUP BY stringField
+ """.stripMargin).show()
- spark.sql("""
- SELECT sum(intField), stringField
- FROM carbon_table
- GROUP BY stringField
- """).show
+ spark.sql(
+ s"""
+ | SELECT t1.*, t2.*
+ | FROM carbon_table t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
+ """.stripMargin).show()
spark.sql(
- """
- |select t1.*, t2.*
- |from carbon_table t1, carbon_table t2
- |where t1.stringField = t2.stringField
- """.stripMargin).show
+ s"""
+ | WITH t1 AS (
+ | SELECT * FROM carbon_table
+ | UNION ALL
+ | SELECT * FROM carbon_table
+ | )
+ | SELECT t1.*, t2.*
+ | FROM t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
+ """.stripMargin).show()
spark.sql(
- """
- |with t1 as (
- |select * from carbon_table
- |union all
- |select * from carbon_table
- |)
- |select t1.*, t2.*
- |from t1, carbon_table t2
- |where t1.stringField = t2.stringField
- """.stripMargin).show
-
- spark.sql("""
- SELECT *
- FROM carbon_table
- where stringfield = 'spark' and floatField > 2.8
- """).show
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ | WHERE stringField = 'spark' and floatField > 2.8
+ """.stripMargin).show()
// Drop table
spark.sql("DROP TABLE IF EXISTS carbon_table")
+
+ spark.stop()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
index 63e680a..b823be5 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
@@ -30,7 +30,7 @@ object SparkSessionExample {
def main(args: Array[String]): Unit = {
val rootPath = new File(this.getClass.getResource("/").getPath
- + "../../../..").getCanonicalPath
+ + "../../../..").getCanonicalPath
val storeLocation = s"$rootPath/examples/spark2/target/store"
val warehouse = s"$rootPath/examples/spark2/target/warehouse"
val metastoredb = s"$rootPath/examples/spark2/target/metastore_db"
@@ -44,14 +44,14 @@ object SparkSessionExample {
}
val spark = SparkSession
- .builder()
- .master("local")
- .appName("SparkSessionExample")
- .enableHiveSupport()
- .config("spark.sql.warehouse.dir", warehouse)
- .config("javax.jdo.option.ConnectionURL",
- s"jdbc:derby:;databaseName=$metastoredb;create=true")
- .getOrCreate()
+ .builder()
+ .master("local")
+ .appName("SparkSessionExample")
+ .enableHiveSupport()
+ .config("spark.sql.warehouse.dir", warehouse)
+ .config("javax.jdo.option.ConnectionURL",
+ s"jdbc:derby:;databaseName=$metastoredb;create=true")
+ .getOrCreate()
CarbonProperties.getInstance()
.addProperty("carbon.storelocation", storeLocation)
@@ -59,46 +59,43 @@ object SparkSessionExample {
spark.sparkContext.setLogLevel("WARN")
CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
// Create table
spark.sql(
s"""
| CREATE TABLE carbon_table(
- | shortField short,
- | intField int,
- | bigintField long,
- | doubleField double,
- | stringField string,
- | timestampField timestamp,
- | decimalField decimal(18,2),
- | dateField date,
- | charField char(5)
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5)
| )
| USING org.apache.spark.sql.CarbonSource
| OPTIONS('DICTIONARY_INCLUDE'='dateField, charField',
- | 'dbName'='default', 'tableName'='carbon_table')
+ | 'dbName'='default', 'tableName'='carbon_table')
""".stripMargin)
- // val prop = s"$rootPath/conf/dataload.properties.template"
- // val tableName = "carbon_table"
val path = s"$rootPath/examples/spark2/src/main/resources/data.csv"
- // TableLoader.main(Array[String](prop, tableName, path))
spark.sql(
s"""
- | CREATE TABLE csv_table
- | ( shortField short,
- | intField int,
- | bigintField long,
- | doubleField double,
- | stringField string,
- | timestampField string,
- | decimalField decimal(18,2),
- | dateField string,
- | charField char(5))
- | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ | CREATE TABLE csv_table(
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField STRING,
+ | decimalField DECIMAL(18,2),
+ | dateField STRING,
+ | charField CHAR(5))
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
""".stripMargin)
spark.sql(
@@ -107,10 +104,7 @@ object SparkSessionExample {
| INTO TABLE csv_table
""".stripMargin)
- spark.sql("""
- SELECT *
- FROM csv_table
- """).show
+ spark.sql("SELECT * FROM csv_table").show()
spark.sql(
s"""
@@ -121,56 +115,60 @@ object SparkSessionExample {
| FROM csv_table
""".stripMargin)
- spark.sql("""
- SELECT *
- FROM carbon_table
- where stringfield = 'spark' and decimalField > 40
- """).show
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ | WHERE stringfield = 'spark' AND decimalField > 40
+ """.stripMargin).show()
// Shows with raw data's timestamp format
- spark.sql("""
- SELECT
- stringField, date_format(timestampField, "yyyy/MM/dd HH:mm:ss") as timestampField
- FROM carbon_table where length(stringField) = 5
- """).show
-
- spark.sql("""
- SELECT *
- FROM carbon_table where date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
- """).show
-
- spark.sql("""
- select count(stringField) from carbon_table
- """.stripMargin).show
-
- spark.sql("""
- SELECT sum(intField), stringField
- FROM carbon_table
- GROUP BY stringField
- """).show
+ spark.sql(
+ s"""
+ | SELECT
+ | stringField, date_format(timestampField, "yyyy/MM/dd HH:mm:ss") AS
+ | timestampField
+ | FROM carbon_table WHERE length(stringField) = 5
+ """.stripMargin).show()
spark.sql(
- """
- |select t1.*, t2.*
- |from carbon_table t1, carbon_table t2
- |where t1.stringField = t2.stringField
- """.stripMargin).show
+ s"""
+ | SELECT *
+ | FROM carbon_table where date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
+ """.stripMargin).show()
+
+ spark.sql("SELECT count(stringField) FROM carbon_table").show()
spark.sql(
- """
- |with t1 as (
- |select * from carbon_table
- |union all
- |select * from carbon_table
- |)
- |select t1.*, t2.*
- |from t1, carbon_table t2
- |where t1.stringField = t2.stringField
- """.stripMargin).show
+ s"""
+ | SELECT sum(intField), stringField
+ | FROM carbon_table
+ | GROUP BY stringField
+ """.stripMargin).show()
+
+ spark.sql(
+ s"""
+ | SELECT t1.*, t2.*
+ | FROM carbon_table t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
+ """.stripMargin).show()
+
+ spark.sql(
+ s"""
+ | WITH t1 AS (
+ | SELECT * FROM carbon_table
+ | UNION ALL
+ | SELECT * FROM carbon_table
+ | )
+ | SELECT t1.*, t2.*
+ | FROM t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
+ """.stripMargin).show()
// Drop table
spark.sql("DROP TABLE IF EXISTS carbon_table")
spark.sql("DROP TABLE IF EXISTS csv_table")
+
spark.stop()
}
}