You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/06/25 05:43:03 UTC
[kylin] branch 2.6.x updated: KYLIN-3845 solve the problem that
kylin build error in stream build task. (#663)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.6.x by this push:
new 46eb871 KYLIN-3845 solve the problem that kylin build error in stream build task. (#663)
46eb871 is described below
commit 46eb8714fbf1f82b9975c896d46383a8900a96b6
Author: zhaojintaozhao <49...@users.noreply.github.com>
AuthorDate: Tue Jun 4 13:40:45 2019 +0800
KYLIN-3845 solve the problem that kylin build error in stream build task. (#663)
* KYLIN-3845 solve the problem that kylin build error if Kafka data source lacks selected dimensions or metrics in a kylin stream build task.
---
.../main/java/org/apache/kylin/cube/util/KeyValueBuilder.java | 4 ++++
.../main/java/org/apache/kylin/dict/DictionaryGenerator.java | 8 ++++----
.../kylin/engine/mr/steps/FactDistinctColumnsMapper.java | 10 +++++++++-
3 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
index 0636a5c..fab1448 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
@@ -61,6 +61,10 @@ public class KeyValueBuilder implements Serializable {
}
private String getCell(int i, String[] flatRow) {
+ if (i >= flatRow.length) {
+ return null;
+ }
+
if (isNull(flatRow[i]))
return null;
else
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 7c33b4a..48d2b08 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -92,7 +92,9 @@ public class DictionaryGenerator {
// build
Dictionary<String> dict = builder.build();
-
+ logger.debug("Dictionary cardinality: " + dict.getSize());
+ logger.debug("Dictionary builder class: " + builder.getClass().getName());
+ logger.debug("Dictionary class: " + dict.getClass().getName());
// log a few samples
StringBuilder buf = new StringBuilder();
for (String s : samples) {
@@ -102,9 +104,7 @@ public class DictionaryGenerator {
buf.append(s.toString()).append("=>").append(dict.getIdFromValue(s));
}
logger.debug("Dictionary value samples: " + buf.toString());
- logger.debug("Dictionary cardinality: " + dict.getSize());
- logger.debug("Dictionary builder class: " + builder.getClass().getName());
- logger.debug("Dictionary class: " + dict.getClass().getName());
+
return dict;
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 7bffce7..2c3bc8d 100755
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -174,7 +174,15 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
for (String[] row : rowCollection) {
context.getCounter(RawDataCounter.BYTES).increment(countSizeInBytes(row));
for (int i = 0; i < allCols.size(); i++) {
- String fieldValue = row[columnIndex[i]];
+ int colIndex = columnIndex[i];
+ int rowSize = row.length;
+ String fieldValue = " ";
+ if (colIndex <= rowSize - 1) {
+ fieldValue = row[colIndex];
+ } else {
+ logger.debug(
+ "colIndex:" + colIndex + " is more than rowSize: " + rowSize + " -1, so set empty value.");
+ }
if (fieldValue == null)
continue;