You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/03/16 09:33:59 UTC
[2/6] carbondata git commit: [CARBONDATA-2248]Fixed Memory leak in
parser/CarbonSparkSqlParser.scala
[CARBONDATA-2248]Fixed Memory leak in parser/CarbonSparkSqlParser.scala
In some scenarios where more sessions are created, there are many parser failure objects are accumulated in memory inside thread locals and causing memory leak in long run.
Solution: Remove the parser object from thread local after parsing of the query
This closes #2057
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5b48e70a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5b48e70a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5b48e70a
Branch: refs/heads/carbonfile
Commit: 5b48e70a8ab7bd662d85790aca8e3a8975e82bfe
Parents: d4f9003
Author: kumarvishal <ku...@gmail.com>
Authored: Mon Mar 12 21:00:22 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Mar 15 17:40:47 2018 +0800
----------------------------------------------------------------------
.../org/apache/spark/sql/parser/CarbonSparkSqlParser.scala | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5b48e70a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index ef4836e..e0fff08 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -36,7 +36,7 @@ import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandExcepti
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.hadoop.util.SchemaReader
import org.apache.carbondata.spark.CarbonOption
-import org.apache.carbondata.spark.util.CommonUtil
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil}
/**
* Concrete parser for Spark SQL stateENABLE_INMEMORY_MERGE_SORT_DEFAULTments and carbon specific
@@ -52,9 +52,12 @@ class CarbonSparkSqlParser(conf: SQLConf, sparkSession: SparkSession) extends Ab
override def parsePlan(sqlText: String): LogicalPlan = {
CarbonSession.updateSessionInfoToCurrentThread(sparkSession)
try {
- super.parsePlan(sqlText)
+ val parsedPlan = super.parsePlan(sqlText)
+ CarbonScalaUtil.cleanParserThreadLocals
+ parsedPlan
} catch {
case ce: MalformedCarbonCommandException =>
+ CarbonScalaUtil.cleanParserThreadLocals
throw ce
case ex =>
try {