You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by ravipesala <gi...@git.apache.org> on 2014/09/30 06:43:18 UTC
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
GitHub user ravipesala opened a pull request:
https://github.com/apache/spark/pull/2590
[SPARK-3654][SQL] Implement all extended HiveQL statements/commands with a separate parser combinator
Created separate parser for hql. It preparses the commands like cache,uncache,add jar etc.. and then parses with HiveQl
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/ravipesala/spark SPARK-3654
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/spark/pull/2590.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #2590
----
commit ba26cd1a5895fa71a028992b5c946bd8333dfdc3
Author: ravipesala <ra...@huawei.com>
Date: 2014-09-30T04:21:42Z
Created seperate parser for hql.It pre parses the commands like
cache,uncache,add jar etc.. and then parses with HiveQl
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57317726
Left some more minor styling issues. This looks great :)
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18215567
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A parser that recognizes all HiveQL constructs together with several Spark SQL specific
+ * extensions like CACHE TABLE and UNCACHE TABLE.
+ */
+class ExtendedHiveQlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
+ input.trim.drop(3).split("=", 2).map(_.trim) match {
+ case Array("") => // "set"
+ SetCommand(None, None)
+ case Array(key) => // "set key"
+ SetCommand(Some(key), None)
+ case Array(key, value) => // "set key=value"
+ SetCommand(Some(key), Some(value))
+ }
+ } else if (input.trim.startsWith("!")) {
+ ShellCommand(input.drop(1))
+ } else {
+ phrase(query)(new lexical.Scanner(input)) match {
+ case Success(r, x) => r
+ case x => sys.error(x.toString)
+ }
+ }
+ }
+
+ protected case class Keyword(str: String)
+
+ protected val CACHE = Keyword("CACHE")
+ protected val SET = Keyword("SET")
+ protected val ADD = Keyword("ADD")
+ protected val JAR = Keyword("JAR")
+ protected val TABLE = Keyword("TABLE")
+ protected val AS = Keyword("AS")
+ protected val UNCACHE = Keyword("UNCACHE")
+ protected val FILE = Keyword("FILE")
+ protected val DFS = Keyword("DFS")
+ protected val SOURCE = Keyword("SOURCE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected def allCaseConverse(k: String): Parser[String] =
+ lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected val reservedWords =
+ this.getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ protected lazy val query: Parser[LogicalPlan] = (
+ cache | uncache | addJar | addFile | dfs | source | hiveQl
+ )
--- End diff --
We can remove this redundant parenthesis.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18218312
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala ---
@@ -257,20 +239,7 @@ private[hive] object HiveQl {
|Unsupported language features in query: $sql
|${dumpTree(getAst(sql))}
""".stripMargin)
- }
- }
-
- /** Creates LogicalPlan for a given HiveQL string. */
- def createPlan(sql: String) = {
- val tree = getAst(sql)
- if (nativeCommands contains tree.getText) {
- NativeCommand(sql)
- } else {
- nodeToPlan(tree) match {
- case NativePlaceholder => NativeCommand(sql)
- case other => other
- }
- }
+ }
--- End diff --
Nit: remove trailing spaces
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18211061
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
+class HiveSqlParser extends StandardTokenParsers with PackratParsers {
--- End diff --
OK. Changed the file name.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18211107
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala ---
@@ -75,6 +75,9 @@ class LocalHiveContext(sc: SparkContext) extends HiveContext(sc) {
*/
class HiveContext(sc: SparkContext) extends SQLContext(sc) {
self =>
+
+ @transient
+ protected[sql] val hiveParser = new HiveSqlParser
--- End diff --
OK. I have moved it to HiveQl now.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18200397
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala ---
@@ -75,6 +75,9 @@ class LocalHiveContext(sc: SparkContext) extends HiveContext(sc) {
*/
class HiveContext(sc: SparkContext) extends SQLContext(sc) {
self =>
+
+ @transient
+ protected[sql] val hiveParser = new HiveSqlParser
--- End diff --
Can we move this to `HiveQl`? It seems more reasonable to put all HiveQL parsing logic together.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18211122
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
+class HiveSqlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
+ input.trim.drop(3).split("=", 2).map(_.trim) match {
+ case Array("") => // "set"
+ SetCommand(None, None)
+ case Array(key) => // "set key"
+ SetCommand(Some(key), None)
+ case Array(key, value) => // "set key=value"
+ SetCommand(Some(key), Some(value))
+ }
+ } else if (input.trim.startsWith("!")) {
+ ShellCommand(input.drop(1))
+ } else {
+ phrase(query)(new lexical.Scanner(input)) match {
+ case Success(r, x) => r
+ case x => sys.error(x.toString)
+ }
+ }
+ }
+
+ protected case class Keyword(str: String)
+
+ protected val CACHE = Keyword("CACHE")
+ protected val SET = Keyword("SET")
+ protected val ADD = Keyword("ADD")
+ protected val JAR = Keyword("JAR")
+ protected val TABLE = Keyword("TABLE")
+ protected val AS = Keyword("AS")
+ protected val UNCACHE = Keyword("UNCACHE")
+ protected val FILE = Keyword("FILE")
+ protected val DFS = Keyword("DFS")
+ protected val SOURCE = Keyword("SOURCE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected def allCaseConverse(k: String): Parser[String] =
+ lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected val reservedWords =
+ this.getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ protected lazy val query: Parser[LogicalPlan] = (
+ cache | unCache | addJar | addFile | dfs | source | hiveQl
+ )
+
+ protected lazy val hiveQl: Parser[LogicalPlan] =
+ remainingQuery ^^ {
+ case r => HiveQl.parseSql(r.trim())
+ }
+
+ /** It returns all remaining query */
+ protected lazy val remainingQuery: Parser[String] = new Parser[String] {
+ def apply(in:Input) = Success(in.source.subSequence(in.offset, in.source.length).toString,
--- End diff --
Updated
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57396021
LGTM.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57272298
Awesome! I just started working on this last weekend and you've already got done :) Left some minor comments. This generally LGTM.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by marmbrus <gi...@git.apache.org>.
Github user marmbrus commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18309810
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala ---
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+
+/**
+ * A parser that recognizes all HiveQL constructs together with several Spark SQL specific
+ * extensions like CACHE TABLE and UNCACHE TABLE.
+ */
+class ExtendedHiveQlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
+ input.trim.drop(3).split("=", 2).map(_.trim) match {
+ case Array("") => // "set"
+ SetCommand(None, None)
+ case Array(key) => // "set key"
+ SetCommand(Some(key), None)
+ case Array(key, value) => // "set key=value"
+ SetCommand(Some(key), Some(value))
+ }
+ } else if (input.trim.startsWith("!")) {
+ ShellCommand(input.drop(1))
+ } else {
+ phrase(query)(new lexical.Scanner(input)) match {
+ case Success(r, x) => r
+ case x => sys.error(x.toString)
+ }
+ }
+ }
+
+ protected case class Keyword(str: String)
+
+ protected val CACHE = Keyword("CACHE")
+ protected val SET = Keyword("SET")
+ protected val ADD = Keyword("ADD")
+ protected val JAR = Keyword("JAR")
+ protected val TABLE = Keyword("TABLE")
+ protected val AS = Keyword("AS")
+ protected val UNCACHE = Keyword("UNCACHE")
+ protected val FILE = Keyword("FILE")
+ protected val DFS = Keyword("DFS")
+ protected val SOURCE = Keyword("SOURCE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected def allCaseConverse(k: String): Parser[String] =
+ lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected val reservedWords =
+ this.getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ protected lazy val query: Parser[LogicalPlan] =
+ cache | uncache | addJar | addFile | dfs | source | hiveQl
+
+ protected lazy val hiveQl: Parser[LogicalPlan] =
+ remainingQuery ^^ {
+ case r => HiveQl.createPlan(r.trim())
+ }
+
+ /** It returns all remaining query */
+ protected lazy val remainingQuery: Parser[String] = new Parser[String] {
+ def apply(in: Input) =
+ Success(
+ in.source.subSequence(in.offset, in.source.length).toString,
+ in.drop(in.source.length()))
+ }
+
+ /** It returns all query */
+ protected lazy val allQuery: Parser[String] = new Parser[String] {
+ def apply(in: Input) =
+ Success(in.source.toString, in.drop(in.source.length()))
+ }
+
+ protected lazy val cache: Parser[LogicalPlan] =
+ CACHE ~ TABLE ~> ident ~ opt(AS ~> hiveQl) ^^ {
+ case tableName ~ None => CacheCommand(tableName, true)
+ case tableName ~ Some(plan) =>
+ CacheTableAsSelectCommand(tableName, plan)
+ }
+
+ protected lazy val uncache: Parser[LogicalPlan] =
+ UNCACHE ~ TABLE ~> ident ^^ {
+ case tableName => CacheCommand(tableName, false)
+ }
+
+ protected lazy val addJar: Parser[LogicalPlan] =
+ ADD ~ JAR ~> remainingQuery ^^ {
+ case rq => AddJar(rq.trim())
+ }
+
+ protected lazy val addFile: Parser[LogicalPlan] =
+ ADD ~ FILE ~> remainingQuery ^^ {
+ case rq => AddFile(rq.trim())
+ }
+
+ protected lazy val dfs: Parser[LogicalPlan] =
+ DFS ~> allQuery ^^ {
+ case aq => NativeCommand(aq.trim())
+ }
+
+ protected lazy val source: Parser[LogicalPlan] =
+ SOURCE ~> remainingQuery ^^ {
+ case rq => SourceCommand(rq.trim())
+ }
+
--- End diff --
Remove extra newline.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18211084
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
+class HiveSqlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
--- End diff --
OK.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18218177
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A parser that recognizes all HiveQL constructs together with several Spark SQL specific
+ * extensions like CACHE TABLE and UNCACHE TABLE.
+ */
+class ExtendedHiveQlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
+ input.trim.drop(3).split("=", 2).map(_.trim) match {
+ case Array("") => // "set"
+ SetCommand(None, None)
+ case Array(key) => // "set key"
+ SetCommand(Some(key), None)
+ case Array(key, value) => // "set key=value"
+ SetCommand(Some(key), Some(value))
+ }
+ } else if (input.trim.startsWith("!")) {
+ ShellCommand(input.drop(1))
+ } else {
+ phrase(query)(new lexical.Scanner(input)) match {
+ case Success(r, x) => r
+ case x => sys.error(x.toString)
+ }
+ }
+ }
+
+ protected case class Keyword(str: String)
+
+ protected val CACHE = Keyword("CACHE")
+ protected val SET = Keyword("SET")
+ protected val ADD = Keyword("ADD")
+ protected val JAR = Keyword("JAR")
+ protected val TABLE = Keyword("TABLE")
+ protected val AS = Keyword("AS")
+ protected val UNCACHE = Keyword("UNCACHE")
+ protected val FILE = Keyword("FILE")
+ protected val DFS = Keyword("DFS")
+ protected val SOURCE = Keyword("SOURCE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected def allCaseConverse(k: String): Parser[String] =
+ lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected val reservedWords =
+ this.getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ protected lazy val query: Parser[LogicalPlan] = (
+ cache | uncache | addJar | addFile | dfs | source | hiveQl
+ )
+
+ protected lazy val hiveQl: Parser[LogicalPlan] =
+ remainingQuery ^^ {
+ case r => HiveQl.createPlan(r.trim())
+ }
+
+ /** It returns all remaining query */
+ protected lazy val remainingQuery: Parser[String] = new Parser[String] {
+ def apply(in: Input) = Success(in.source.subSequence(in.offset, in.source.length).toString,
+ in.drop(in.source.length()))
--- End diff --
Nit: adjust indentation
```scala
def apply(in: Input) =
Success(
in.source.subSequence(in.offset, in.source.length).toString,
in.drop(in.source.length()))
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by marmbrus <gi...@git.apache.org>.
Github user marmbrus commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18309830
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala ---
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+
+/**
+ * A parser that recognizes all HiveQL constructs together with several Spark SQL specific
+ * extensions like CACHE TABLE and UNCACHE TABLE.
+ */
+class ExtendedHiveQlParser extends StandardTokenParsers with PackratParsers {
--- End diff --
This should probably be `private[hive]`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57595734
Jenkins, retest this please.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57272266
[QA tests have started](https://amplab.cs.berkeley.edu/jenkins/job/NewSparkPullRequestBuilder/209/consoleFull) for PR 2590 at commit [`ba26cd1`](https://github.com/apache/spark/commit/ba26cd1a5895fa71a028992b5c946bd8333dfdc3).
* This patch merges cleanly.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57628271
[QA tests have finished](https://amplab.cs.berkeley.edu/jenkins/job/NewSparkPullRequestBuilder/255/consoleFull) for PR 2590 at commit [`bbca7dd`](https://github.com/apache/spark/commit/bbca7dd626dcfb79230d4fa954ee3e0dee5c68e9).
* This patch **passes** unit tests.
* This patch merges cleanly.
* This patch adds the following public classes _(experimental)_:
* ` protected case class Keyword(str: String)`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18200319
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
+class HiveSqlParser extends StandardTokenParsers with PackratParsers {
--- End diff --
Maybe `ExtendedHiveQLParser` is a better name? Since HiveQL is the definitive name and we're adding extensions to it in Spark SQL.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18215437
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
--- End diff --
Please follow [this instruction](https://cwiki.apache.org/confluence/display/SPARK/Spark+Code+Style+Guide#SparkCodeStyleGuide-Imports) to organize imports, and remove unused ones.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57581369
Fixed code as per comments, please review
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57299430
Thank you. I updated the code as per your comments, Please review it.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18200306
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
--- End diff --
Since this class takes over all HiveQL parsing work (although it delegates to an underlying Hive parser), it's not accurate to call it a "pre parser", maybe this:
> A parser that recognizes all HiveQL constructs together with several Spark SQL specific extensions like `CACHE TABLE` and `UNCACHE TABLE`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57335446
Thank you for your comments. I updated code as per your comments.Please review.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18215602
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A parser that recognizes all HiveQL constructs together with several Spark SQL specific
+ * extensions like CACHE TABLE and UNCACHE TABLE.
+ */
+class ExtendedHiveQlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
+ input.trim.drop(3).split("=", 2).map(_.trim) match {
+ case Array("") => // "set"
+ SetCommand(None, None)
+ case Array(key) => // "set key"
+ SetCommand(Some(key), None)
+ case Array(key, value) => // "set key=value"
+ SetCommand(Some(key), Some(value))
+ }
+ } else if (input.trim.startsWith("!")) {
+ ShellCommand(input.drop(1))
+ } else {
+ phrase(query)(new lexical.Scanner(input)) match {
+ case Success(r, x) => r
+ case x => sys.error(x.toString)
+ }
+ }
+ }
+
+ protected case class Keyword(str: String)
+
+ protected val CACHE = Keyword("CACHE")
+ protected val SET = Keyword("SET")
+ protected val ADD = Keyword("ADD")
+ protected val JAR = Keyword("JAR")
+ protected val TABLE = Keyword("TABLE")
+ protected val AS = Keyword("AS")
+ protected val UNCACHE = Keyword("UNCACHE")
+ protected val FILE = Keyword("FILE")
+ protected val DFS = Keyword("DFS")
+ protected val SOURCE = Keyword("SOURCE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected def allCaseConverse(k: String): Parser[String] =
+ lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected val reservedWords =
+ this.getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ protected lazy val query: Parser[LogicalPlan] = (
+ cache | uncache | addJar | addFile | dfs | source | hiveQl
+ )
+
+ protected lazy val hiveQl: Parser[LogicalPlan] =
+ remainingQuery ^^ {
+ case r => HiveQl.createPlan(r.trim())
+ }
+
+ /** It returns all remaining query */
+ protected lazy val remainingQuery: Parser[String] = new Parser[String] {
+ def apply(in: Input) = Success(in.source.subSequence(in.offset, in.source.length).toString,
+ in.drop(in.source.length()))
+ }
+
+ /** It returns all query */
+ protected lazy val allQuery: Parser[String] = new Parser[String] {
+ def apply(in: Input) = Success(in.source.toString,
+ in.drop(in.source.length()))
--- End diff --
Don't break the line here since it fits in 100 columns.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18200542
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
+class HiveSqlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
+ input.trim.drop(3).split("=", 2).map(_.trim) match {
+ case Array("") => // "set"
+ SetCommand(None, None)
+ case Array(key) => // "set key"
+ SetCommand(Some(key), None)
+ case Array(key, value) => // "set key=value"
+ SetCommand(Some(key), Some(value))
+ }
+ } else if (input.trim.startsWith("!")) {
+ ShellCommand(input.drop(1))
+ } else {
+ phrase(query)(new lexical.Scanner(input)) match {
+ case Success(r, x) => r
+ case x => sys.error(x.toString)
+ }
+ }
+ }
+
+ protected case class Keyword(str: String)
+
+ protected val CACHE = Keyword("CACHE")
+ protected val SET = Keyword("SET")
+ protected val ADD = Keyword("ADD")
+ protected val JAR = Keyword("JAR")
+ protected val TABLE = Keyword("TABLE")
+ protected val AS = Keyword("AS")
+ protected val UNCACHE = Keyword("UNCACHE")
+ protected val FILE = Keyword("FILE")
+ protected val DFS = Keyword("DFS")
+ protected val SOURCE = Keyword("SOURCE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected def allCaseConverse(k: String): Parser[String] =
+ lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected val reservedWords =
+ this.getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ protected lazy val query: Parser[LogicalPlan] = (
+ cache | unCache | addJar | addFile | dfs | source | hiveQl
+ )
+
+ protected lazy val hiveQl: Parser[LogicalPlan] =
+ remainingQuery ^^ {
+ case r => HiveQl.parseSql(r.trim())
+ }
+
+ /** It returns all remaining query */
+ protected lazy val remainingQuery: Parser[String] = new Parser[String] {
+ def apply(in:Input) = Success(in.source.subSequence(in.offset, in.source.length).toString,
--- End diff --
Nit: space after `:`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57622569
[QA tests have started](https://amplab.cs.berkeley.edu/jenkins/job/NewSparkPullRequestBuilder/255/consoleFull) for PR 2590 at commit [`bbca7dd`](https://github.com/apache/spark/commit/bbca7dd626dcfb79230d4fa954ee3e0dee5c68e9).
* This patch merges cleanly.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57268296
ok to test
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18215533
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A parser that recognizes all HiveQL constructs together with several Spark SQL specific
+ * extensions like CACHE TABLE and UNCACHE TABLE.
+ */
+class ExtendedHiveQlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
+ input.trim.drop(3).split("=", 2).map(_.trim) match {
+ case Array("") => // "set"
+ SetCommand(None, None)
+ case Array(key) => // "set key"
+ SetCommand(Some(key), None)
+ case Array(key, value) => // "set key=value"
+ SetCommand(Some(key), Some(value))
+ }
+ } else if (input.trim.startsWith("!")) {
+ ShellCommand(input.drop(1))
+ } else {
+ phrase(query)(new lexical.Scanner(input)) match {
+ case Success(r, x) => r
+ case x => sys.error(x.toString)
+ }
+ }
+ }
+
+ protected case class Keyword(str: String)
+
+ protected val CACHE = Keyword("CACHE")
+ protected val SET = Keyword("SET")
+ protected val ADD = Keyword("ADD")
+ protected val JAR = Keyword("JAR")
+ protected val TABLE = Keyword("TABLE")
+ protected val AS = Keyword("AS")
+ protected val UNCACHE = Keyword("UNCACHE")
+ protected val FILE = Keyword("FILE")
+ protected val DFS = Keyword("DFS")
+ protected val SOURCE = Keyword("SOURCE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected def allCaseConverse(k: String): Parser[String] =
+ lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected val reservedWords =
+ this.getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ protected lazy val query: Parser[LogicalPlan] = (
+ cache | uncache | addJar | addFile | dfs | source | hiveQl
+ )
+
+ protected lazy val hiveQl: Parser[LogicalPlan] =
+ remainingQuery ^^ {
+ case r => HiveQl.createPlan(r.trim())
+ }
+
+ /** It returns all remaining query */
+ protected lazy val remainingQuery: Parser[String] = new Parser[String] {
+ def apply(in: Input) = Success(in.source.subSequence(in.offset, in.source.length).toString,
+ in.drop(in.source.length()))
+ }
+
+ /** It returns all query */
+ protected lazy val allQuery: Parser[String] = new Parser[String] {
+ def apply(in: Input) = Success(in.source.toString,
+ in.drop(in.source.length()))
+ }
+
+ protected lazy val cache: Parser[LogicalPlan] =
+ CACHE ~ TABLE ~> ident ~ opt(AS ~> hiveQl) ^^ {
+ case tableName ~ None => CacheCommand(tableName, true)
+ case tableName ~ Some(plan) =>
+ CacheTableAsSelectCommand(tableName, plan)
+ }
+
+ protected lazy val uncache: Parser[LogicalPlan] =
+ UNCACHE ~ TABLE ~> ident ^^ {
+ case tableName => CacheCommand(tableName, false)
+ }
+
+ protected lazy val addJar: Parser[LogicalPlan] =
+ ADD ~ JAR ~> remainingQuery ^^ {
+ case rq => AddJar(rq.trim())
+ }
+
+ protected lazy val addFile: Parser[LogicalPlan] =
+ ADD ~ FILE ~> remainingQuery ^^ {
+ case rq => AddFile(rq.trim())
+ }
+
+ protected lazy val dfs: Parser[LogicalPlan] =
+ DFS ~> allQuery ^^ {
+ case aq => NativeCommand(aq.trim())
+ }
+
+ protected lazy val source: Parser[LogicalPlan] =
+ SOURCE ~> remainingQuery ^^ {
+ case rq => SourceCommand(rq.trim())
+ }
+
+}
--- End diff --
Noticed lots of trailing spaces in this file, please remove them.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by marmbrus <gi...@git.apache.org>.
Github user marmbrus commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57746419
Thanks! I've merged this to master.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18211038
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
--- End diff --
Looks good. I updated as per your comment
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18200443
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
+class HiveSqlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
+ input.trim.drop(3).split("=", 2).map(_.trim) match {
+ case Array("") => // "set"
+ SetCommand(None, None)
+ case Array(key) => // "set key"
+ SetCommand(Some(key), None)
+ case Array(key, value) => // "set key=value"
+ SetCommand(Some(key), Some(value))
+ }
+ } else if (input.trim.startsWith("!")) {
+ ShellCommand(input.drop(1))
+ } else {
+ phrase(query)(new lexical.Scanner(input)) match {
+ case Success(r, x) => r
+ case x => sys.error(x.toString)
+ }
+ }
+ }
+
+ protected case class Keyword(str: String)
+
+ protected val CACHE = Keyword("CACHE")
+ protected val SET = Keyword("SET")
+ protected val ADD = Keyword("ADD")
+ protected val JAR = Keyword("JAR")
+ protected val TABLE = Keyword("TABLE")
+ protected val AS = Keyword("AS")
+ protected val UNCACHE = Keyword("UNCACHE")
+ protected val FILE = Keyword("FILE")
+ protected val DFS = Keyword("DFS")
+ protected val SOURCE = Keyword("SOURCE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected def allCaseConverse(k: String): Parser[String] =
+ lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected val reservedWords =
+ this.getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ protected lazy val query: Parser[LogicalPlan] = (
+ cache | unCache | addJar | addFile | dfs | source | hiveQl
--- End diff --
Nit: I'd prefer `uncache` to `unCache`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:
https://github.com/apache/spark/pull/2590
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57276077
[QA tests have finished](https://amplab.cs.berkeley.edu/jenkins/job/NewSparkPullRequestBuilder/209/consoleFull) for PR 2590 at commit [`ba26cd1`](https://github.com/apache/spark/commit/ba26cd1a5895fa71a028992b5c946bd8333dfdc3).
* This patch **passes** unit tests.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by liancheng <gi...@git.apache.org>.
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18200330
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
+class HiveSqlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
--- End diff --
I'd like to handle `SET` in parser combinator too, but we can leave this to another PR.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/2590#issuecomment-57266684
Can one of the admins verify this patch?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-3654][SQL] Implement all extended HiveQ...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/spark/pull/2590#discussion_r18211112
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSqlParser.scala ---
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.PackratParsers
+import scala.util.parsing.input.CharArrayReader.EofCh
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.SqlLexical
+import scala.util.parsing.combinator.lexical.StdLexical
+
+/**
+ * A simple Hive SQL pre parser. It parses the commands like cache,uncache etc and
+ * remaining actual query will be parsed by HiveQl.parseSql
+ */
+class HiveSqlParser extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = {
+ // Special-case out set commands since the value fields can be
+ // complex to handle without RegexParsers. Also this approach
+ // is clearer for the several possible cases of set commands.
+ if (input.trim.toLowerCase.startsWith("set")) {
+ input.trim.drop(3).split("=", 2).map(_.trim) match {
+ case Array("") => // "set"
+ SetCommand(None, None)
+ case Array(key) => // "set key"
+ SetCommand(Some(key), None)
+ case Array(key, value) => // "set key=value"
+ SetCommand(Some(key), Some(value))
+ }
+ } else if (input.trim.startsWith("!")) {
+ ShellCommand(input.drop(1))
+ } else {
+ phrase(query)(new lexical.Scanner(input)) match {
+ case Success(r, x) => r
+ case x => sys.error(x.toString)
+ }
+ }
+ }
+
+ protected case class Keyword(str: String)
+
+ protected val CACHE = Keyword("CACHE")
+ protected val SET = Keyword("SET")
+ protected val ADD = Keyword("ADD")
+ protected val JAR = Keyword("JAR")
+ protected val TABLE = Keyword("TABLE")
+ protected val AS = Keyword("AS")
+ protected val UNCACHE = Keyword("UNCACHE")
+ protected val FILE = Keyword("FILE")
+ protected val DFS = Keyword("DFS")
+ protected val SOURCE = Keyword("SOURCE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected def allCaseConverse(k: String): Parser[String] =
+ lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+
+ protected val reservedWords =
+ this.getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ protected lazy val query: Parser[LogicalPlan] = (
+ cache | unCache | addJar | addFile | dfs | source | hiveQl
--- End diff --
updated
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org