You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/09/20 03:22:48 UTC
spark git commit: [SPARK-10155] [SQL] Change SqlParser to object to
avoid memory leak
Repository: spark
Updated Branches:
refs/heads/master d83b6aae8 -> e789000b8
[SPARK-10155] [SQL] Change SqlParser to object to avoid memory leak
Since `scala.util.parsing.combinator.Parsers` is thread-safe since Scala 2.10 (See [SI-4929](https://issues.scala-lang.org/browse/SI-4929)), we can change SqlParser to object to avoid memory leak.
I didn't change other subclasses of `scala.util.parsing.combinator.Parsers` because there is only one instance in one SQLContext, which should not be an issue.
Author: zsxwing <zs...@gmail.com>
Closes #8357 from zsxwing/sql-memory-leak.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e789000b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e789000b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e789000b
Branch: refs/heads/master
Commit: e789000b88a6bd840f821c53f42c08b97dc02496
Parents: d83b6aa
Author: zsxwing <zs...@gmail.com>
Authored: Sat Sep 19 18:22:43 2015 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Sat Sep 19 18:22:43 2015 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/AbstractSparkSQLParser.scala | 2 +-
.../scala/org/apache/spark/sql/catalyst/ParserDialect.scala | 2 +-
.../main/scala/org/apache/spark/sql/catalyst/SqlParser.scala | 6 +++---
sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala | 6 +++---
.../src/main/scala/org/apache/spark/sql/DataFrameWriter.scala | 4 ++--
sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala | 6 +++---
sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 2 +-
.../src/main/scala/org/apache/spark/sql/hive/HiveContext.scala | 6 +++---
.../scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala | 4 ++--
9 files changed, 19 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e789000b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/AbstractSparkSQLParser.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/AbstractSparkSQLParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/AbstractSparkSQLParser.scala
index 5898a5f..2bac08e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/AbstractSparkSQLParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/AbstractSparkSQLParser.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.plans.logical._
private[sql] abstract class AbstractSparkSQLParser
extends StandardTokenParsers with PackratParsers {
- def parse(input: String): LogicalPlan = {
+ def parse(input: String): LogicalPlan = synchronized {
// Initialize the Keywords.
initLexical
phrase(start)(new lexical.Scanner(input)) match {
http://git-wip-us.apache.org/repos/asf/spark/blob/e789000b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ParserDialect.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ParserDialect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ParserDialect.scala
index 554fb4e..e21d3c0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ParserDialect.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ParserDialect.scala
@@ -61,7 +61,7 @@ abstract class ParserDialect {
*/
private[spark] class DefaultParserDialect extends ParserDialect {
@transient
- protected val sqlParser = new SqlParser
+ protected val sqlParser = SqlParser
override def parse(sqlText: String): LogicalPlan = {
sqlParser.parse(sqlText)
http://git-wip-us.apache.org/repos/asf/spark/blob/e789000b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
index f249886..dfab239 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
@@ -37,9 +37,9 @@ import org.apache.spark.unsafe.types.CalendarInterval
* This is currently included mostly for illustrative purposes. Users wanting more complete support
* for a SQL like language should checkout the HiveQL support in the sql/hive sub-project.
*/
-class SqlParser extends AbstractSparkSQLParser with DataTypeParser {
+object SqlParser extends AbstractSparkSQLParser with DataTypeParser {
- def parseExpression(input: String): Expression = {
+ def parseExpression(input: String): Expression = synchronized {
// Initialize the Keywords.
initLexical
phrase(projection)(new lexical.Scanner(input)) match {
@@ -48,7 +48,7 @@ class SqlParser extends AbstractSparkSQLParser with DataTypeParser {
}
}
- def parseTableIdentifier(input: String): TableIdentifier = {
+ def parseTableIdentifier(input: String): TableIdentifier = synchronized {
// Initialize the Keywords.
initLexical
phrase(tableIdentifier)(new lexical.Scanner(input)) match {
http://git-wip-us.apache.org/repos/asf/spark/blob/e789000b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 3e61123..8f737c2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -720,7 +720,7 @@ class DataFrame private[sql](
@scala.annotation.varargs
def selectExpr(exprs: String*): DataFrame = {
select(exprs.map { expr =>
- Column(new SqlParser().parseExpression(expr))
+ Column(SqlParser.parseExpression(expr))
}: _*)
}
@@ -745,7 +745,7 @@ class DataFrame private[sql](
* @since 1.3.0
*/
def filter(conditionExpr: String): DataFrame = {
- filter(Column(new SqlParser().parseExpression(conditionExpr)))
+ filter(Column(SqlParser.parseExpression(conditionExpr)))
}
/**
@@ -769,7 +769,7 @@ class DataFrame private[sql](
* @since 1.5.0
*/
def where(conditionExpr: String): DataFrame = {
- filter(Column(new SqlParser().parseExpression(conditionExpr)))
+ filter(Column(SqlParser.parseExpression(conditionExpr)))
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/e789000b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 745bb4e..03e9736 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -163,7 +163,7 @@ final class DataFrameWriter private[sql](df: DataFrame) {
* @since 1.4.0
*/
def insertInto(tableName: String): Unit = {
- insertInto(new SqlParser().parseTableIdentifier(tableName))
+ insertInto(SqlParser.parseTableIdentifier(tableName))
}
private def insertInto(tableIdent: TableIdentifier): Unit = {
@@ -197,7 +197,7 @@ final class DataFrameWriter private[sql](df: DataFrame) {
* @since 1.4.0
*/
def saveAsTable(tableName: String): Unit = {
- saveAsTable(new SqlParser().parseTableIdentifier(tableName))
+ saveAsTable(SqlParser.parseTableIdentifier(tableName))
}
private def saveAsTable(tableIdent: TableIdentifier): Unit = {
http://git-wip-us.apache.org/repos/asf/spark/blob/e789000b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index e3fdd78..f099940 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -590,7 +590,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
tableName: String,
source: String,
options: Map[String, String]): DataFrame = {
- val tableIdent = new SqlParser().parseTableIdentifier(tableName)
+ val tableIdent = SqlParser.parseTableIdentifier(tableName)
val cmd =
CreateTableUsing(
tableIdent,
@@ -636,7 +636,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
source: String,
schema: StructType,
options: Map[String, String]): DataFrame = {
- val tableIdent = new SqlParser().parseTableIdentifier(tableName)
+ val tableIdent = SqlParser.parseTableIdentifier(tableName)
val cmd =
CreateTableUsing(
tableIdent,
@@ -732,7 +732,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
* @since 1.3.0
*/
def table(tableName: String): DataFrame = {
- table(new SqlParser().parseTableIdentifier(tableName))
+ table(SqlParser.parseTableIdentifier(tableName))
}
private def table(tableIdent: TableIdentifier): DataFrame = {
http://git-wip-us.apache.org/repos/asf/spark/blob/e789000b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 60d9c50..2467b4e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -823,7 +823,7 @@ object functions {
*
* @group normal_funcs
*/
- def expr(expr: String): Column = Column(new SqlParser().parseExpression(expr))
+ def expr(expr: String): Column = Column(SqlParser.parseExpression(expr))
//////////////////////////////////////////////////////////////////////////////////////////////
// Math Functions
http://git-wip-us.apache.org/repos/asf/spark/blob/e789000b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index d37ba5d..c12a734 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -291,12 +291,12 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) with Logging {
* @since 1.3.0
*/
def refreshTable(tableName: String): Unit = {
- val tableIdent = new SqlParser().parseTableIdentifier(tableName)
+ val tableIdent = SqlParser.parseTableIdentifier(tableName)
catalog.refreshTable(tableIdent)
}
protected[hive] def invalidateTable(tableName: String): Unit = {
- val tableIdent = new SqlParser().parseTableIdentifier(tableName)
+ val tableIdent = SqlParser.parseTableIdentifier(tableName)
catalog.invalidateTable(tableIdent)
}
@@ -311,7 +311,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) with Logging {
*/
@Experimental
def analyze(tableName: String) {
- val tableIdent = new SqlParser().parseTableIdentifier(tableName)
+ val tableIdent = SqlParser.parseTableIdentifier(tableName)
val relation = EliminateSubQueries(catalog.lookupRelation(tableIdent.toSeq))
relation match {
http://git-wip-us.apache.org/repos/asf/spark/blob/e789000b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 0a5569b..0c1b41e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -199,7 +199,7 @@ private[hive] class HiveMetastoreCatalog(val client: ClientInterface, hive: Hive
options: Map[String, String],
isExternal: Boolean): Unit = {
createDataSourceTable(
- new SqlParser().parseTableIdentifier(tableName),
+ SqlParser.parseTableIdentifier(tableName),
userSpecifiedSchema,
partitionColumns,
provider,
@@ -375,7 +375,7 @@ private[hive] class HiveMetastoreCatalog(val client: ClientInterface, hive: Hive
}
def hiveDefaultTableFilePath(tableName: String): String = {
- hiveDefaultTableFilePath(new SqlParser().parseTableIdentifier(tableName))
+ hiveDefaultTableFilePath(SqlParser.parseTableIdentifier(tableName))
}
def hiveDefaultTableFilePath(tableIdent: TableIdentifier): String = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org