You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vk...@apache.org on 2017/12/29 21:32:14 UTC
ignite git commit: IGNITE-3084 - Minor fixes in Spark DataFrame
examples
Repository: ignite
Updated Branches:
refs/heads/master 44428f318 -> 286ea7367
IGNITE-3084 - Minor fixes in Spark DataFrame examples
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/286ea736
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/286ea736
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/286ea736
Branch: refs/heads/master
Commit: 286ea736791dc9487d5c6c4e47eeb969f6c37a98
Parents: 44428f3
Author: Valentin Kulichenko <va...@gmail.com>
Authored: Fri Dec 29 13:32:03 2017 -0800
Committer: Valentin Kulichenko <va...@gmail.com>
Committed: Fri Dec 29 13:32:03 2017 -0800
----------------------------------------------------------------------
.../examples/spark/IgniteCatalogExample.scala | 28 ++++++++++++++++----
.../examples/spark/IgniteDataFrameExample.scala | 22 ++++++++++++---
2 files changed, 41 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/286ea736/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteCatalogExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteCatalogExample.scala b/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteCatalogExample.scala
index 4382e10..dab8ec6 100644
--- a/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteCatalogExample.scala
+++ b/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteCatalogExample.scala
@@ -56,25 +56,47 @@ object IgniteCatalogExample extends App {
Logger.getRootLogger.setLevel(Level.ERROR)
Logger.getLogger("org.apache.ignite").setLevel(Level.INFO)
+ println("List of available tables:")
+
//Showing existing tables.
igniteSession.catalog.listTables().show()
+ println("PERSON table description:")
+
//Showing `person` schema.
igniteSession.catalog.listColumns("person").show()
+ println("CITY table description:")
+
//Showing `city` schema.
igniteSession.catalog.listColumns("city").show()
+ println("Querying all persons from city with ID=2.")
+ println
+
//Selecting data throw Spark SQL engine.
val df = igniteSession.sql("SELECT * FROM person WHERE CITY_ID = 2")
+ println("Result schema:")
+
df.printSchema()
+
+ println("Result content:")
+
df.show()
+ println("Querying all persons living in Denver.")
+ println
+
//Selecting data throw Spark SQL engine.
- val df2 = igniteSession.sql("SELECT * FROM person p JOIN city c ON c.ID = p.CITY_ID WHERE p.CITY_ID = 2")
+ val df2 = igniteSession.sql("SELECT * FROM person p JOIN city c ON c.ID = p.CITY_ID WHERE c.NAME = 'Denver'")
+
+ println("Result schema:")
df2.printSchema()
+
+ println("Result content:")
+
df2.show()
}
@@ -102,8 +124,6 @@ object IgniteCatalogExample extends App {
cache.query(new SqlFieldsQuery("CREATE INDEX on Person (city_id)")).getAll
- println("Created database objects.")
-
//Inserting some data into table.
var qry = new SqlFieldsQuery("INSERT INTO city (id, name) VALUES (?, ?)")
@@ -118,8 +138,6 @@ object IgniteCatalogExample extends App {
cache.query(qry.setArgs(3L.asInstanceOf[JLong], "Mary Major", 1L.asInstanceOf[JLong])).getAll
cache.query(qry.setArgs(4L.asInstanceOf[JLong], "Richard Miles", 2L.asInstanceOf[JLong])).getAll
- println("Populated data.")
-
ignite
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/286ea736/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteDataFrameExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteDataFrameExample.scala b/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteDataFrameExample.scala
index ae84915..61e56ee 100644
--- a/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteDataFrameExample.scala
+++ b/examples/src/main/spark/org/apache/ignite/examples/spark/IgniteDataFrameExample.scala
@@ -52,11 +52,11 @@ object IgniteDataFrameExample extends App {
.config("spark.executor.instances", "2")
.getOrCreate()
- //Adjust the logger to exclude the logs of no interest.
+ // Adjust the logger to exclude the logs of no interest.
Logger.getRootLogger.setLevel(Level.ERROR)
Logger.getLogger("org.apache.ignite").setLevel(Level.INFO)
- //Executing examples.
+ // Executing examples.
sparkDSLExample
@@ -70,6 +70,9 @@ object IgniteDataFrameExample extends App {
* @param spark SparkSession.
*/
def sparkDSLExample(implicit spark: SparkSession): Unit = {
+ println("Querying using Spark DSL.")
+ println
+
val igniteDF = spark.read
.format(FORMAT_IGNITE) //Data source type.
.option(OPTION_TABLE, "person") //Table to read.
@@ -78,7 +81,12 @@ object IgniteDataFrameExample extends App {
.filter(col("id") >= 2) //Filter clause.
.filter(col("name") like "%M%") //Another filter clause.
+ println("Data frame schema:")
+
igniteDF.printSchema() //Printing query schema to console.
+
+ println("Data frame content:")
+
igniteDF.show() //Printing query results to console.
}
@@ -90,6 +98,9 @@ object IgniteDataFrameExample extends App {
* @param spark SparkSession.
*/
def nativeSparkSqlExample(implicit spark: SparkSession): Unit = {
+ println("Querying using Spark SQL.")
+ println
+
val df = spark.read
.format(FORMAT_IGNITE) //Data source type.
.option(OPTION_TABLE, "person") //Table to read.
@@ -102,7 +113,12 @@ object IgniteDataFrameExample extends App {
//Selecting data from Ignite throw Spark SQL Engine.
val igniteDF = spark.sql("SELECT * FROM person WHERE id >= 2 AND name = 'Mary Major'")
+ println("Result schema:")
+
igniteDF.printSchema() //Printing query schema to console.
+
+ println("Result content:")
+
igniteDF.show() //Printing query results to console.
}
@@ -139,8 +155,6 @@ object IgniteDataFrameExample extends App {
cache.query(qry.setArgs(3L.asInstanceOf[JLong], "Mary Major", 1L.asInstanceOf[JLong])).getAll
cache.query(qry.setArgs(4L.asInstanceOf[JLong], "Richard Miles", 2L.asInstanceOf[JLong])).getAll
- println("Populated data.")
-
ignite
}
}