You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/12/01 05:24:31 UTC
[3/6] kylin git commit: KYLIN-2240 Add a toggle to ignore all cube
signature inconsistency temporally
KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/80018874
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/80018874
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/80018874
Branch: refs/heads/yang21-cdh5.7
Commit: 80018874c26d17f57bd288654996d3b6508d8294
Parents: 076c77a
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Dec 1 11:35:34 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Dec 1 11:35:34 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 39 ++++++---
.../org/apache/kylin/cube/model/CubeDesc.java | 90 +++++++++++---------
2 files changed, 76 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/80018874/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a91e42b..f35f969 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -18,6 +18,13 @@
package org.apache.kylin.common;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
@@ -28,14 +35,6 @@ import java.util.SortedSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
/**
* An abstract class to encapsulate access to a set of 'properties'.
* Subclass can override methods in this class to extend the content of the 'properties',
@@ -174,19 +173,25 @@ abstract public class KylinConfigBase implements Serializable {
setProperty("kylin.storage.url", storageUrl);
}
- /** was for route to hive, not used any more */
+ /**
+ * was for route to hive, not used any more
+ */
@Deprecated
public String getHiveUrl() {
return getOptional("hive.url", "");
}
- /** was for route to hive, not used any more */
+ /**
+ * was for route to hive, not used any more
+ */
@Deprecated
public String getHiveUser() {
return getOptional("hive.user", "");
}
- /** was for route to hive, not used any more */
+ /**
+ * was for route to hive, not used any more
+ */
@Deprecated
public String getHivePassword() {
return getOptional("hive.password", "");
@@ -202,7 +207,7 @@ abstract public class KylinConfigBase implements Serializable {
public String[] getRealizationProviders() {
return getOptionalStringArray("kylin.realization.providers", //
- new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
+ new String[]{"org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager"});
}
public CliCommandExecutor getCliCommandExecutor() throws IOException {
@@ -427,6 +432,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.cube.algorithm.auto.mapper.limit", "500"));
}
+ public boolean isIgnoreCubeSignatureInconsistency() {
+ return Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", "false"));
+ }
+
@Deprecated
public int getCubeAggrGroupMaxSize() {
return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", "12"));
@@ -480,10 +489,11 @@ abstract public class KylinConfigBase implements Serializable {
public float getCubeVisitTimeoutTimes() {
return Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
}
-
+
public int getDerivedInThreshold() {
return Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20"));
}
+
public int getBadQueryStackTraceDepth() {
return Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
}
@@ -569,7 +579,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int[] getQueryMetricsPercentilesIntervals() {
- String[] dft = { "60", "300", "3600" };
+ String[] dft = {"60", "300", "3600"};
return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft);
}
@@ -591,6 +601,7 @@ abstract public class KylinConfigBase implements Serializable {
/**
* HBase region cut size, in GB
+ *
* @return
*/
public float getKylinHBaseRegionCut() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/80018874/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index c914ebb..00bfbc9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -18,27 +18,16 @@
package org.apache.kylin.cube.model;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeSet;
-
-import javax.annotation.Nullable;
-
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ArrayUtils;
@@ -67,16 +56,25 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+import javax.annotation.Nullable;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
/**
*/
@@ -199,7 +197,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
/**
- * @return dimension columns excluding derived
+ * @return dimension columns excluding derived
*/
public List<TblColRef> listDimensionColumnsExcludingDerived(boolean alsoExcludeExtendedCol) {
List<TblColRef> result = new ArrayList<TblColRef>();
@@ -467,11 +465,17 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
/**
* this method is to prevent malicious metadata change by checking the saved signature
* with the calculated signature.
- *
+ * <p>
* if you're comparing two cube descs, prefer to use consistentWith()
+ *
* @return
*/
public boolean checkSignature() {
+ if (this.getConfig().isIgnoreCubeSignatureInconsistency()) {
+ logger.info("Skip checking cube signature");
+ return true;
+ }
+
if (KylinVersion.getCurrentVersion().isCompatibleWith(new KylinVersion(getVersion())) && !KylinVersion.getCurrentVersion().isSignatureCompatibleWith(new KylinVersion(getVersion()))) {
logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), getVersion());
return true;
@@ -752,7 +756,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
int find = ArrayUtils.indexOf(dimColArray, fk[i]);
if (find >= 0) {
TblColRef derivedCol = initDimensionColRef(pk[i]);
- initDerivedMap(new TblColRef[] { dimColArray[find] }, DeriveType.PK_FK, dim, new TblColRef[] { derivedCol }, null);
+ initDerivedMap(new TblColRef[]{dimColArray[find]}, DeriveType.PK_FK, dim, new TblColRef[]{derivedCol}, null);
}
}
/** disable this code as we don't need fk be derived from pk
@@ -782,7 +786,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
extra[i] = "";
}
}
- return new String[][] { cols, extra };
+ return new String[][]{cols, extra};
}
private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) {
@@ -1011,7 +1015,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
this.partitionDateEnd = partitionDateEnd;
}
- /** Get columns that have dictionary */
+ /**
+ * Get columns that have dictionary
+ */
public Set<TblColRef> getAllColumnsHaveDictionary() {
Set<TblColRef> result = Sets.newLinkedHashSet();
@@ -1040,7 +1046,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return result;
}
- /** Get columns that need dictionary built on it. Note a column could reuse dictionary of another column. */
+ /**
+ * Get columns that need dictionary built on it. Note a column could reuse dictionary of another column.
+ */
public Set<TblColRef> getAllColumnsNeedDictionaryBuilt() {
Set<TblColRef> result = getAllColumnsHaveDictionary();
@@ -1057,7 +1065,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return result;
}
- /** A column may reuse dictionary of another column, find the dict column, return same col if there's no reuse column*/
+ /**
+ * A column may reuse dictionary of another column, find the dict column, return same col if there's no reuse column
+ */
public TblColRef getDictionaryReuseColumn(TblColRef col) {
if (dictionaries == null) {
return col;
@@ -1070,7 +1080,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return col;
}
- /** Get a column which can be used in distributing the source table */
+ /**
+ * Get a column which can be used in distributing the source table
+ */
public TblColRef getDistributedByColumn() {
Set<TblColRef> shardBy = getShardByColumns();
if (shardBy != null && shardBy.size() > 0) {