You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2018/04/11 08:52:50 UTC
carbondata git commit: [CARBONDATA-2329] Non Serializable extra info
in session is overwritten from stale thread
Repository: carbondata
Updated Branches:
refs/heads/master df8f06739 -> 359f6e6b2
[CARBONDATA-2329] Non Serializable extra info in session is overwritten from stale thread
Problem:
1. Non Serializable extra info is copied from thread which causes stale data from old session when the thread is reused by spark.
2. CarboSessionInfo clone is not copying Non serializable info to new object which can damage session level values if local query thread updates values.
Solution: Remove logic to copy Non Serializable extra info and fix clone logic
This closes #2154
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/359f6e6b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/359f6e6b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/359f6e6b
Branch: refs/heads/master
Commit: 359f6e6b22cca44a344545470720c02d07bbb4f4
Parents: df8f067
Author: KanakaKumar <ka...@huawei.com>
Authored: Tue Apr 10 19:42:30 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Wed Apr 11 14:25:55 2018 +0530
----------------------------------------------------------------------
.../org/apache/carbondata/core/util/CarbonSessionInfo.java | 6 ++----
.../spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala | 2 --
.../src/main/scala/org/apache/spark/sql/CarbonSession.scala | 4 ----
3 files changed, 2 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/359f6e6b/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java
index 25d1c29..38f2d76 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java
@@ -60,11 +60,9 @@ public class CarbonSessionInfo implements Serializable, Cloneable {
CarbonSessionInfo newObj = new CarbonSessionInfo();
newObj.setSessionParams(sessionParams.clone());
newObj.setThreadParams(threadParams.clone());
- Map<String, Object> nonSerializableExtraInfo = getNonSerializableExtraInfo();
- for (Map.Entry<String, Object> entry : nonSerializableExtraInfo.entrySet()) {
- nonSerializableExtraInfo.put(entry.getKey(), entry.getValue());
+ for (Map.Entry<String, Object> entry : getNonSerializableExtraInfo().entrySet()) {
+ newObj.getNonSerializableExtraInfo().put(entry.getKey(), entry.getValue());
}
- newObj.setNonSerializableExtraInfo(nonSerializableExtraInfo);
return newObj;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/359f6e6b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 881a4f5..593ecce 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -82,11 +82,9 @@ class CarbonEnv {
// update carbon session parameters , preserve thread parameters
val currentThreadSesssionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
carbonSessionInfo = new CarbonSessionInfo()
- sessionParams = carbonSessionInfo.getSessionParams
if (currentThreadSesssionInfo != null) {
carbonSessionInfo.setThreadParams(currentThreadSesssionInfo.getThreadParams)
}
- carbonSessionInfo.setSessionParams(sessionParams)
ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)
val config = new CarbonSQLConf(sparkSession)
if (sparkSession.conf.getOption(CarbonCommonConstants.ENABLE_UNSAFE_SORT).isEmpty) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/359f6e6b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 1038fcf..a380308 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -292,11 +292,7 @@ object CarbonSession {
// copy all the thread parameters to apply to session parameters
currentThreadSessionInfo.getThreadParams.getAll.asScala
.foreach(entry => carbonSessionInfo.getSessionParams.addProperty(entry._1, entry._2))
- currentThreadSessionInfo.getNonSerializableExtraInfo.asScala
- .foreach(entry => carbonSessionInfo.getNonSerializableExtraInfo.put(entry._1, entry._2))
carbonSessionInfo.setThreadParams(currentThreadSessionInfo.getThreadParams)
- carbonSessionInfo
- .setNonSerializableExtraInfo(currentThreadSessionInfo.getNonSerializableExtraInfo)
}
// preserve thread parameters across call
ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)