You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/12/09 00:44:24 UTC
spark git commit: [SPARK-4774] [SQL] Makes HiveFromSpark more portable
Repository: spark
Updated Branches:
refs/heads/master ab2abcb5e -> d6a972b3e
[SPARK-4774] [SQL] Makes HiveFromSpark more portable
HiveFromSpark read the kv1.txt file from SPARK_HOME/examples/src/main/resources/kv1.txt which assumed
you had a source tree checked out. Now we copy the kv1.txt file to a temporary file and delete it when
the jvm shuts down. This allows us to run this example outside of a spark source tree.
Author: Kostas Sakellis <ko...@cloudera.com>
Closes #3628 from ksakellis/kostas-spark-4774 and squashes the following commits:
6770f83 [Kostas Sakellis] [SPARK-4774] [SQL] Makes HiveFromSpark more portable
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d6a972b3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d6a972b3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d6a972b3
Branch: refs/heads/master
Commit: d6a972b3e4dc35a2d95df47d256462b325f4bda6
Parents: ab2abcb
Author: Kostas Sakellis <ko...@cloudera.com>
Authored: Mon Dec 8 15:44:18 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Dec 8 15:44:18 2014 -0800
----------------------------------------------------------------------
.../apache/spark/examples/sql/hive/HiveFromSpark.scala | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d6a972b3/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala b/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala
index 138923c..5725da1 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala
@@ -17,6 +17,10 @@
package org.apache.spark.examples.sql.hive
+import com.google.common.io.{ByteStreams, Files}
+
+import java.io.File
+
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql._
import org.apache.spark.sql.hive.HiveContext
@@ -24,10 +28,15 @@ import org.apache.spark.sql.hive.HiveContext
object HiveFromSpark {
case class Record(key: Int, value: String)
+ // Copy kv1.txt file from classpath to temporary directory
+ val kv1Stream = HiveFromSpark.getClass.getResourceAsStream("/kv1.txt")
+ val kv1File = File.createTempFile("kv1", "txt")
+ kv1File.deleteOnExit()
+ ByteStreams.copy(kv1Stream, Files.newOutputStreamSupplier(kv1File))
+
def main(args: Array[String]) {
val sparkConf = new SparkConf().setAppName("HiveFromSpark")
val sc = new SparkContext(sparkConf)
- val path = s"${System.getenv("SPARK_HOME")}/examples/src/main/resources/kv1.txt"
// A hive context adds support for finding tables in the MetaStore and writing queries
// using HiveQL. Users who do not have an existing Hive deployment can still create a
@@ -37,7 +46,7 @@ object HiveFromSpark {
import hiveContext._
sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)")
- sql(s"LOAD DATA LOCAL INPATH '$path' INTO TABLE src")
+ sql(s"LOAD DATA LOCAL INPATH '${kv1File.getAbsolutePath}' INTO TABLE src")
// Queries are expressed in HiveQL
println("Result of 'SELECT *': ")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org