You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/08/09 18:26:30 UTC
[46/47] carbondata git commit: [HOTFIX]Fixed reflection issue in Query
[HOTFIX]Fixed reflection issue in Query
Problem: Reflection call is taking more time while getting the task context listener.
Solution: Now instead of calling CarbonReflectionUtil.scala added code in CarbonScanRDD
This closes #2626
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/67b98408
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/67b98408
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/67b98408
Branch: refs/heads/branch-1.4
Commit: 67b98408da1134ba30c5d083446c8c105b334ee9
Parents: 3b122df
Author: kumarvishal09 <ku...@gmail.com>
Authored: Thu Aug 9 22:09:59 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Aug 9 23:51:36 2018 +0530
----------------------------------------------------------------------
.../scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala | 4 +++-
.../scala/org/apache/spark/util/CarbonReflectionUtils.scala | 6 +++---
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/67b98408/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 38062a4..c613bfc 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -464,7 +464,9 @@ class CarbonScanRDD[T: ClassTag](
// TODO: rewrite this logic to call free memory in FailureListener on failures. On success,
// TODO: no memory leak should be there, resources should be freed on success completion.
- val listeners = CarbonReflectionUtils.getField("onCompleteCallbacks", context)
+ val onCompleteCallbacksField = context.getClass.getDeclaredField("onCompleteCallbacks")
+ onCompleteCallbacksField.setAccessible(true)
+ val listeners = onCompleteCallbacksField.get(context)
.asInstanceOf[ArrayBuffer[TaskCompletionListener]]
val isAdded = listeners.exists(p => p.isInstanceOf[InsertTaskCompletionListener])
http://git-wip-us.apache.org/repos/asf/carbondata/blob/67b98408/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
index 1061e98..34eeded 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
@@ -52,9 +52,9 @@ object CarbonReflectionUtils {
* @return
*/
def getField[T: TypeTag : reflect.ClassTag](name: String, obj: T): Any = {
- val field = obj.getClass.getDeclaredField(name)
- field.setAccessible(true)
- field.get(obj)
+ val im = rm.reflect(obj)
+ im.symbol.typeSignature.members.find(_.name.toString.equals(name))
+ .map(l => im.reflectField(l.asTerm).get).getOrElse(null)
}
def getUnresolvedRelation(