You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/06/04 05:40:51 UTC
[kylin] branch master updated: KYLIN-3845 solve the problem that
kylin build error in stream build task. (#663)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new c3a7110 KYLIN-3845 solve the problem that kylin build error in stream build task. (#663)
c3a7110 is described below
commit c3a71101286ba994dec85aa515236b28c98b9e94
Author: zhaojintaozhao <49...@users.noreply.github.com>
AuthorDate: Tue Jun 4 13:40:45 2019 +0800
KYLIN-3845 solve the problem that kylin build error in stream build task. (#663)
* KYLIN-3845 solve the problem that kylin build error if Kafka data source lacks selected dimensions or metrics in a kylin stream build task.
---
.../main/java/org/apache/kylin/cube/util/KeyValueBuilder.java | 4 ++++
.../main/java/org/apache/kylin/dict/DictionaryGenerator.java | 8 ++++----
.../kylin/engine/mr/steps/FactDistinctColumnsMapper.java | 10 +++++++++-
3 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
index 48894d9..e9df6c0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
@@ -63,6 +63,10 @@ public class KeyValueBuilder implements Serializable {
}
private String getCell(int i, String[] flatRow) {
+ if (i >= flatRow.length) {
+ return null;
+ }
+
if (isNull(flatRow[i]))
return null;
else
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 6a501e9..f4aaa45 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -96,7 +96,9 @@ public class DictionaryGenerator {
// build
Dictionary<String> dict = builder.build();
-
+ logger.debug("Dictionary cardinality: " + dict.getSize());
+ logger.debug("Dictionary builder class: " + builder.getClass().getName());
+ logger.debug("Dictionary class: " + dict.getClass().getName());
// log a few samples
StringBuilder buf = new StringBuilder();
for (String s : samples) {
@@ -106,9 +108,7 @@ public class DictionaryGenerator {
buf.append(s.toString()).append("=>").append(dict.getIdFromValue(s));
}
logger.debug("Dictionary value samples: " + buf.toString());
- logger.debug("Dictionary cardinality: " + dict.getSize());
- logger.debug("Dictionary builder class: " + builder.getClass().getName());
- logger.debug("Dictionary class: " + dict.getClass().getName());
+
return dict;
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 7bffce7..2c3bc8d 100755
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -174,7 +174,15 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
for (String[] row : rowCollection) {
context.getCounter(RawDataCounter.BYTES).increment(countSizeInBytes(row));
for (int i = 0; i < allCols.size(); i++) {
- String fieldValue = row[columnIndex[i]];
+ int colIndex = columnIndex[i];
+ int rowSize = row.length;
+ String fieldValue = " ";
+ if (colIndex <= rowSize - 1) {
+ fieldValue = row[colIndex];
+ } else {
+ logger.debug(
+ "colIndex:" + colIndex + " is more than rowSize: " + rowSize + " -1, so set empty value.");
+ }
if (fieldValue == null)
continue;