You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/04/21 20:42:30 UTC
spark git commit: [SPARK-14795][SQL] Remove the use of Hive's
variable substitution
Repository: spark
Updated Branches:
refs/heads/master 79008e6cf -> 3a21e8d5e
[SPARK-14795][SQL] Remove the use of Hive's variable substitution
## What changes were proposed in this pull request?
This patch builds on #12556 and completely removes the use of Hive's variable substitution.
## How was this patch tested?
Covered by existing tests.
Author: Reynold Xin <rx...@databricks.com>
Closes #12561 from rxin/SPARK-14795.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3a21e8d5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3a21e8d5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3a21e8d5
Branch: refs/heads/master
Commit: 3a21e8d5ed640e3f82946893e24c099aa723c169
Parents: 79008e6
Author: Reynold Xin <rx...@databricks.com>
Authored: Thu Apr 21 11:42:25 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu Apr 21 11:42:25 2016 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/hive/HiveSessionState.scala | 3 +--
.../apache/spark/sql/hive/execution/HiveSqlParser.scala | 12 +++++-------
.../apache/spark/sql/hive/execution/SQLQuerySuite.scala | 4 ++--
3 files changed, 8 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3a21e8d5/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
index 171def4..6f4332c 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
@@ -21,7 +21,6 @@ import java.util.regex.Pattern
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.analysis.Analyzer
@@ -109,7 +108,7 @@ private[hive] class HiveSessionState(ctx: SQLContext) extends SessionState(ctx)
/**
* Parser for HiveQl query texts.
*/
- override lazy val sqlParser: ParserInterface = new HiveSqlParser(conf, hiveconf)
+ override lazy val sqlParser: ParserInterface = new HiveSqlParser(conf)
/**
* Planner that takes into account Hive-specific strategies.
http://git-wip-us.apache.org/repos/asf/spark/blob/3a21e8d5/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
index 1c4cda7..989da92 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
@@ -20,8 +20,6 @@ package org.apache.spark.sql.hive.execution
import scala.util.Try
import org.antlr.v4.runtime.Token
-import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution
import org.apache.hadoop.hive.serde.serdeConstants
import org.apache.spark.sql.catalyst.catalog._
@@ -29,23 +27,23 @@ import org.apache.spark.sql.catalyst.parser._
import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.SparkSqlAstBuilder
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, VariableSubstitution}
/**
* Concrete parser for HiveQl statements.
*/
-class HiveSqlParser(conf: SQLConf, hiveconf: HiveConf) extends AbstractSqlParser {
+class HiveSqlParser(conf: SQLConf) extends AbstractSqlParser {
val astBuilder = new HiveSqlAstBuilder(conf)
- lazy val substitutor = new VariableSubstitution
+ private val substitutor = new VariableSubstitution(conf)
protected override def parse[T](command: String)(toResult: SqlBaseParser => T): T = {
- super.parse(substitutor.substitute(hiveconf, command))(toResult)
+ super.parse(substitutor.substitute(command))(toResult)
}
protected override def nativeCommand(sqlText: String): LogicalPlan = {
- HiveNativeCommand(substitutor.substitute(hiveconf, sqlText))
+ HiveNativeCommand(substitutor.substitute(sqlText))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/3a21e8d5/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index b97e9fe..345ee8e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -512,13 +512,13 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
sql("SELECT key FROM ${hiveconf:tbl} ORDER BY key, value limit 1"),
sql("SELECT key FROM src ORDER BY key, value limit 1").collect().toSeq)
- sql("set hive.variable.substitute=false") // disable the substitution
+ sql("set spark.sql.variable.substitute=false") // disable the substitution
sql("set tbl2=src")
intercept[Exception] {
sql("SELECT key FROM ${hiveconf:tbl2} ORDER BY key, value limit 1").collect()
}
- sql("set hive.variable.substitute=true") // enable the substitution
+ sql("set spark.sql.variable.substitute=true") // enable the substitution
checkAnswer(
sql("SELECT key FROM ${hiveconf:tbl2} ORDER BY key, value limit 1"),
sql("SELECT key FROM src ORDER BY key, value limit 1").collect().toSeq)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org