You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/11/01 10:29:10 UTC
[kylin] branch 2.5.x updated: KYLIN-3644 Fix SparkFactDistinct step
NumberFormatException
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.5.x by this push:
new 0411360 KYLIN-3644 Fix SparkFactDistinct step NumberFormatException
0411360 is described below
commit 0411360557ec2355448fa67c3ae18a9473179afa
Author: chao long <wa...@qq.com>
AuthorDate: Thu Nov 1 16:43:11 2018 +0800
KYLIN-3644 Fix SparkFactDistinct step NumberFormatException
---
.../src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java | 2 +-
.../main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java | 5 ++++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
index 0ba4fd8..0636a5c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
@@ -56,7 +56,7 @@ public class KeyValueBuilder implements Serializable {
}
}
- protected boolean isNull(String v) {
+ public boolean isNull(String v) {
return nullStrs.contains(v);
}
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
index 213cdfd..7f05234 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
@@ -58,6 +58,7 @@ import org.apache.kylin.cube.DimensionRangeInfo;
import org.apache.kylin.cube.cuboid.CuboidUtil;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.cube.util.KeyValueBuilder;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IDictionaryBuilder;
import org.apache.kylin.engine.EngineFactory;
@@ -238,6 +239,7 @@ public class SparkFactDistinct extends AbstractApplication implements Serializab
private Map<Integer, DimensionRangeInfo> dimensionRangeInfoMap;
private ByteBuffer tmpbuf;
private LongAccumulator bytesWritten;
+ private KeyValueBuilder keyValueBuilder;
public FlatOutputFucntion(String cubeName, String segmentId, String metaurl, SerializableConfiguration conf, int samplingPercent, LongAccumulator bytesWritten) {
this.cubeName = cubeName;
@@ -259,6 +261,7 @@ public class SparkFactDistinct extends AbstractApplication implements Serializab
CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich(
EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
+ keyValueBuilder = new KeyValueBuilder(intermediateTableDesc);
reducerMapping = new FactDistinctColumnsReducerMapping(cubeInstance);
tmpbuf = ByteBuffer.allocate(4096);
@@ -304,7 +307,7 @@ public class SparkFactDistinct extends AbstractApplication implements Serializab
for (int i = 0; i < allCols.size(); i++) {
String fieldValue = row[columnIndex[i]];
- if (fieldValue == null)
+ if (fieldValue == null || keyValueBuilder.isNull(fieldValue))
continue;
final DataType type = allCols.get(i).getType();