You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2021/07/13 02:49:03 UTC
[kylin] branch kylin-on-parquet-v2 updated: HOTFIX Fix some issues
for spark3
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this push:
new c51b481 HOTFIX Fix some issues for spark3
c51b481 is described below
commit c51b481e1c6006142c4b04d8957cbd3daa0e8f6b
Author: Zhichao Zhang <zh...@apache.org>
AuthorDate: Tue Jul 13 10:44:34 2021 +0800
HOTFIX Fix some issues for spark3
---
kylin-spark-project/kylin-spark-common/pom.xml | 5 +++++
.../apache/spark/sql/catalyst/expressions/KylinExpresssions.scala | 2 +-
kylin-spark-project/kylin-spark-metadata/pom.xml | 1 +
parquet-assembly/pom.xml | 3 +++
4 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/kylin-spark-project/kylin-spark-common/pom.xml b/kylin-spark-project/kylin-spark-common/pom.xml
index b027793..5e35bd4 100644
--- a/kylin-spark-project/kylin-spark-common/pom.xml
+++ b/kylin-spark-project/kylin-spark-common/pom.xml
@@ -49,6 +49,11 @@
<artifactId>kylin-spark-metadata</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/catalyst/expressions/KylinExpresssions.scala b/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/catalyst/expressions/KylinExpresssions.scala
index d25896c..7a5b965 100644
--- a/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/catalyst/expressions/KylinExpresssions.scala
+++ b/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/catalyst/expressions/KylinExpresssions.scala
@@ -492,7 +492,7 @@ case class ScatterSkewData(left: Expression, right: Expression) extends BinaryEx
val rand = ctx.addMutableState("java.util.Random", "rand")
val skewData = ctx.addMutableState("it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap",
"skewData")
- val skewDataStorage = right.simpleString
+ val skewDataStorage = ExpressionUtils.simpleString(right)
val initParamsFuncName = ctx.addNewFunction(s"initParams",
s"""
diff --git a/kylin-spark-project/kylin-spark-metadata/pom.xml b/kylin-spark-project/kylin-spark-metadata/pom.xml
index 71b8b4e..c5c842e 100644
--- a/kylin-spark-project/kylin-spark-metadata/pom.xml
+++ b/kylin-spark-project/kylin-spark-metadata/pom.xml
@@ -42,6 +42,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
</dependency>
</dependencies>
diff --git a/parquet-assembly/pom.xml b/parquet-assembly/pom.xml
index f2c795c..0c106c7 100644
--- a/parquet-assembly/pom.xml
+++ b/parquet-assembly/pom.xml
@@ -121,6 +121,9 @@
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/maven/org.slf4j/**</exclude>
+ <exclude>org/slf4j/**</exclude>
+ <exclude>javax/**</exclude>
</excludes>
</filter>
</filters>