You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/09 03:09:12 UTC
[1/2] incubator-carbondata git commit: Fixed IN filter expression
deserializing issue from thrift server
Repository: incubator-carbondata
Updated Branches:
refs/heads/master e0b00bfc1 -> 8e224aac9
Fixed IN filter expression deserializing issue from thrift server
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/6c11558d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/6c11558d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/6c11558d
Branch: refs/heads/master
Commit: 6c11558dffc78bf45a974ef68370959fb80ab71b
Parents: e0b00bf
Author: ravipesala <ra...@gmail.com>
Authored: Mon Aug 8 15:24:59 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Aug 8 15:27:44 2016 +0530
----------------------------------------------------------------------
.../hadoop/util/ObjectSerializationUtil.java | 2 +-
.../org/carbondata/spark/CarbonFilters.scala | 24 ++++++++++++++------
2 files changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6c11558d/hadoop/src/main/java/org/carbondata/hadoop/util/ObjectSerializationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/carbondata/hadoop/util/ObjectSerializationUtil.java b/hadoop/src/main/java/org/carbondata/hadoop/util/ObjectSerializationUtil.java
index a1d87a6..9100fe5 100644
--- a/hadoop/src/main/java/org/carbondata/hadoop/util/ObjectSerializationUtil.java
+++ b/hadoop/src/main/java/org/carbondata/hadoop/util/ObjectSerializationUtil.java
@@ -97,7 +97,7 @@ public class ObjectSerializationUtil {
ois = new ObjectInputStream(gis);
return ois.readObject();
} catch (ClassNotFoundException e) {
- throw new IOException("Could not read object");
+ throw new IOException("Could not read object", e);
} finally {
try {
if (ois != null) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6c11558d/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala b/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala
index 230bf31..b7f45ab 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala
@@ -17,7 +17,6 @@
package org.carbondata.spark
-import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.catalyst.expressions._
@@ -77,10 +76,12 @@ object CarbonFilters {
case sources.In(name, values) =>
Some(new InExpression(getCarbonExpression(name),
- new ListExpression(values.map(f => getCarbonLiteralExpression(name, f)).toList.asJava)))
+ new ListExpression(
+ convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
case sources.Not(sources.In(name, values)) =>
Some(new NotInExpression(getCarbonExpression(name),
- new ListExpression(values.map(f => getCarbonLiteralExpression(name, f)).toList.asJava)))
+ new ListExpression(
+ convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
case sources.And(lhs, rhs) =>
(createFilter(lhs) ++ createFilter(rhs)).reduceOption(new AndExpression(_, _))
@@ -259,17 +260,17 @@ object CarbonFilters {
case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
Some(new NotInExpression(transformExpression(a).get,
- new ListExpression(list.map(transformExpression(_).get).asJava)))
+ new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
Some(new InExpression(transformExpression(a).get,
- new ListExpression(list.map(transformExpression(_).get).asJava)))
+ new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
case Not(In(Cast(a: Attribute, _), list))
if !list.exists(!_.isInstanceOf[Literal]) =>
Some(new NotInExpression(transformExpression(a).get,
- new ListExpression(list.map(transformExpression(_).get).asJava)))
+ new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
case In(Cast(a: Attribute, _), list) if !list.exists(!_.isInstanceOf[Literal]) =>
Some(new InExpression(transformExpression(a).get,
- new ListExpression(list.map(transformExpression(_).get).asJava)))
+ new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
case GreaterThan(a: Attribute, l@Literal(v, t)) =>
Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
@@ -351,4 +352,13 @@ object CarbonFilters {
}
CarbonScalaUtil.convertCarbonToSparkDataType(dataType)
}
+
+ // Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is
+ // not able find the classes inside scala list and gives ClassNotFoundException.
+ private def convertToJavaList(
+ scalaList: Seq[CarbonExpression]): java.util.List[CarbonExpression] = {
+ val javaList = new java.util.ArrayList[CarbonExpression]()
+ scalaList.foreach(javaList.add)
+ javaList
+ }
}
[2/2] incubator-carbondata git commit: Fixed IN filter expression
deserializing issue from thrift server This closes #67
Posted by ch...@apache.org.
Fixed IN filter expression deserializing issue from thrift server This closes #67
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8e224aac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8e224aac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8e224aac
Branch: refs/heads/master
Commit: 8e224aac96def41743d778a38a2a8ce9c91d51fc
Parents: e0b00bf 6c11558
Author: chenliang613 <ch...@apache.org>
Authored: Tue Aug 9 11:08:55 2016 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Tue Aug 9 11:08:55 2016 +0800
----------------------------------------------------------------------
.../hadoop/util/ObjectSerializationUtil.java | 2 +-
.../org/carbondata/spark/CarbonFilters.scala | 24 ++++++++++++++------
2 files changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------