You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/09/12 11:51:06 UTC
[kylin] 01/02: KYLIN-4096: Make cube metadata validator rules
configuable
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit ca76625bc3b783ced7168e9ff7bd588ade2592ab
Author: Liu Shaohui <li...@xiaomi.com>
AuthorDate: Fri May 17 13:48:42 2019 +0800
KYLIN-4096: Make cube metadata validator rules configuable
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 +++
.../org/apache/kylin/cube/CubeDescManager.java | 4 +--
.../model/validation/CubeMetadataValidator.java | 30 +++++++++++++++++++++-
3 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 16755f3..7a78f88 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2247,4 +2247,8 @@ public abstract class KylinConfigBase implements Serializable {
public int getStaleJobThresholdInDays() {
return Integer.parseInt(getOptional("kylin.tool.health-check.stale-job-threshold-days", "30"));
}
+
+ public String[] getCubeMetadataExtraValidators() {
+ return getOptionalStringArray("kylin.cube.metadata.extra.validators", new String[0]);
+ }
}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index a7459c0..f95ccf0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -202,7 +202,7 @@ public class CubeDescManager {
return cubeDesc;
}
// Semantic validation
- CubeMetadataValidator validator = new CubeMetadataValidator();
+ CubeMetadataValidator validator = new CubeMetadataValidator(config);
ValidateContext context = validator.validate(cubeDesc);
if (!context.ifPass()) {
return cubeDesc;
@@ -241,7 +241,7 @@ public class CubeDescManager {
postProcessCubeDesc(desc);
// Semantic validation
- CubeMetadataValidator validator = new CubeMetadataValidator();
+ CubeMetadataValidator validator = new CubeMetadataValidator(config);
ValidateContext context = validator.validate(desc);
if (!context.ifPass()) {
return desc;
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
index c2c5f89..24fd802 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
@@ -18,12 +18,18 @@
package org.apache.kylin.cube.model.validation;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
import org.apache.kylin.cube.model.validation.rule.DictionaryRule;
import org.apache.kylin.cube.model.validation.rule.FunctionRule;
import org.apache.kylin.cube.model.validation.rule.RowKeyAttrRule;
import org.apache.kylin.cube.model.validation.rule.StreamingCubeRule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
/**
* For cube metadata validator
@@ -32,8 +38,30 @@ import org.apache.kylin.cube.model.validation.rule.StreamingCubeRule;
*
*/
public class CubeMetadataValidator {
+ protected static final Logger logger = LoggerFactory.getLogger(CubeMetadataValidator.class);
@SuppressWarnings("unchecked")
- private IValidatorRule<CubeDesc>[] rules = new IValidatorRule[] { new FunctionRule(), new AggregationGroupRule(), new RowKeyAttrRule(), new DictionaryRule(), new StreamingCubeRule() };
+ private IValidatorRule<CubeDesc>[] defaultRules = new IValidatorRule[] {
+ new FunctionRule(),
+ new AggregationGroupRule(),
+ new RowKeyAttrRule(),
+ new DictionaryRule(),
+ new StreamingCubeRule()
+ };
+
+ private List<IValidatorRule<CubeDesc>> rules;
+
+ public CubeMetadataValidator(KylinConfig config) {
+ rules = Arrays.asList(defaultRules);
+ for (String ruleName : config.getCubeMetadataExtraValidators()) {
+ try {
+ IValidatorRule<CubeDesc> rule = (IValidatorRule<CubeDesc>)
+ Class.forName(ruleName).getConstructor().newInstance();
+ rules.add(rule);
+ } catch (Exception e) {
+ logger.error("Construct cube metadata validator rule: {} failed. Ignore this rule", ruleName, e);
+ }
+ }
+ }
public ValidateContext validate(CubeDesc cube) {
ValidateContext context = new ValidateContext();