You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/17 03:32:15 UTC
[10/21] kylin git commit: KYLIN-2357 Make ERROR_RECORD_LOG_THRESHOLD
configurable
KYLIN-2357 Make ERROR_RECORD_LOG_THRESHOLD configurable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8d374f84
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8d374f84
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8d374f84
Branch: refs/heads/master-cdh5.7
Commit: 8d374f84368194d78208ea3b7683a697e8ec4314
Parents: eb18ac5
Author: kangkaisen <ka...@live.com>
Authored: Sat Jan 7 19:16:30 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Sat Jan 14 19:14:42 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../java/org/apache/kylin/engine/mr/common/BatchConstants.java | 1 -
.../org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java | 2 +-
.../kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java | 2 +-
4 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 494c641..d6774ff 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -423,6 +423,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0"));
}
+ public Integer getErrorRecordThreshold() {
+ return Integer.parseInt(getOptional("kylin.job.error-record-threshold", "0"));
+ }
+
// ============================================================================
// SOURCE.HIVE
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 078d80f..0281539 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -80,7 +80,6 @@ public interface BatchConstants {
*/
String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder";
int NORMAL_RECORD_LOG_THRESHOLD = 100000;
- int ERROR_RECORD_LOG_THRESHOLD = 100;
/**
* dictionaries builder class
http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index d08e29a..93e413b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -87,7 +87,7 @@ abstract public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
// TODO expose errorRecordCounter as hadoop counter
errorRecordCounter++;
- if (errorRecordCounter > BatchConstants.ERROR_RECORD_LOG_THRESHOLD) {
+ if (errorRecordCounter > cubeSegment.getConfig().getErrorRecordThreshold()) {
if (ex instanceof IOException)
throw (IOException) ex;
else if (ex instanceof RuntimeException)
http://git-wip-us.apache.org/repos/asf/kylin/blob/8d374f84/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 74aebb0..34624e3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -105,7 +105,7 @@ abstract public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends Kyli
ex.printStackTrace(System.err);
errorRecordCounter++;
- if (errorRecordCounter > BatchConstants.ERROR_RECORD_LOG_THRESHOLD) {
+ if (errorRecordCounter > cubeSeg.getConfig().getErrorRecordThreshold()) {
if (ex instanceof IOException)
throw (IOException) ex;
else if (ex instanceof RuntimeException)