You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/04/15 22:00:24 UTC
spark git commit: [SPARK-6730][SQL] Allow using keyword as identifier
in OPTIONS
Repository: spark
Updated Branches:
refs/heads/master f11288d52 -> b75b30707
[SPARK-6730][SQL] Allow using keyword as identifier in OPTIONS
JIRA: https://issues.apache.org/jira/browse/SPARK-6730
It is very possible that keyword will be used as identifier in `OPTIONS`, this pr makes it works.
However, another approach is that we can request that `OPTIONS` can't include keywords and has to use alternative identifier (e.g. table -> cassandraTable) if needed.
If so, please let me know to close this pr. Thanks.
Author: Liang-Chi Hsieh <vi...@gmail.com>
Closes #5520 from viirya/relax_options and squashes the following commits:
339fd68 [Liang-Chi Hsieh] Use regex parser.
92be11c [Liang-Chi Hsieh] Allow using keyword as identifier in OPTIONS.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b75b3070
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b75b3070
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b75b3070
Branch: refs/heads/master
Commit: b75b3070740803480d235b0c9a86673721344f30
Parents: f11288d
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Wed Apr 15 13:00:19 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Wed Apr 15 13:00:19 2015 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/sources/ddl.scala | 15 ++++++++++++++-
.../org/apache/spark/sql/sources/DDLTestSuite.scala | 11 ++++++-----
2 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b75b3070/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
index 319de71..2e861b8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.sources
import scala.language.existentials
+import scala.util.matching.Regex
import scala.language.implicitConversions
import org.apache.spark.Logging
@@ -155,7 +156,19 @@ private[sql] class DDLParser(
protected lazy val className: Parser[String] = repsep(ident, ".") ^^ { case s => s.mkString(".")}
- protected lazy val pair: Parser[(String, String)] = ident ~ stringLit ^^ { case k ~ v => (k,v) }
+ override implicit def regexToParser(regex: Regex): Parser[String] = acceptMatch(
+ s"identifier matching regex ${regex}", {
+ case lexical.Identifier(str) if regex.unapplySeq(str).isDefined => str
+ case lexical.Keyword(str) if regex.unapplySeq(str).isDefined => str
+ }
+ )
+
+ protected lazy val optionName: Parser[String] = "[_a-zA-Z][a-zA-Z0-9]*".r ^^ {
+ case name => name
+ }
+
+ protected lazy val pair: Parser[(String, String)] =
+ optionName ~ stringLit ^^ { case k ~ v => (k,v) }
protected lazy val column: Parser[StructField] =
ident ~ dataType ~ (COMMENT ~> stringLit).? ^^ { case columnName ~ typ ~ cm =>
http://git-wip-us.apache.org/repos/asf/spark/blob/b75b3070/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala
index 3f24a49..ca25751 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala
@@ -25,17 +25,17 @@ class DDLScanSource extends RelationProvider {
override def createRelation(
sqlContext: SQLContext,
parameters: Map[String, String]): BaseRelation = {
- SimpleDDLScan(parameters("from").toInt, parameters("TO").toInt)(sqlContext)
+ SimpleDDLScan(parameters("from").toInt, parameters("TO").toInt, parameters("Table"))(sqlContext)
}
}
-case class SimpleDDLScan(from: Int, to: Int)(@transient val sqlContext: SQLContext)
+case class SimpleDDLScan(from: Int, to: Int, table: String)(@transient val sqlContext: SQLContext)
extends BaseRelation with TableScan {
override def schema: StructType =
StructType(Seq(
StructField("intType", IntegerType, nullable = false,
- new MetadataBuilder().putString("comment", "test comment").build()),
+ new MetadataBuilder().putString("comment", s"test comment $table").build()),
StructField("stringType", StringType, nullable = false),
StructField("dateType", DateType, nullable = false),
StructField("timestampType", TimestampType, nullable = false),
@@ -73,7 +73,8 @@ class DDLTestSuite extends DataSourceTest {
|USING org.apache.spark.sql.sources.DDLScanSource
|OPTIONS (
| From '1',
- | To '10'
+ | To '10',
+ | Table 'test1'
|)
""".stripMargin)
}
@@ -81,7 +82,7 @@ class DDLTestSuite extends DataSourceTest {
sqlTest(
"describe ddlPeople",
Seq(
- Row("intType", "int", "test comment"),
+ Row("intType", "int", "test comment test1"),
Row("stringType", "string", ""),
Row("dateType", "date", ""),
Row("timestampType", "timestamp", ""),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org