You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/29 13:03:25 UTC
[1/3] kylin git commit: minor,
set default cubing algorithm to layer for stableness
Repository: kylin
Updated Branches:
refs/heads/master 4c2182147 -> fa3ee3ffb
minor, set default cubing algorithm to layer for stableness
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a776ef67
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a776ef67
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a776ef67
Branch: refs/heads/master
Commit: a776ef6747638ab757c9a5298b500dd18756f5b5
Parents: 4c21821
Author: Li Yang <li...@apache.org>
Authored: Wed Mar 29 11:51:22 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Mar 29 11:57:24 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a776ef67/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 1a55c94..d39f7ee 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -144,7 +144,7 @@ kylin.engine.mr.uhc-reducer-count=1
### CUBE | DICTIONARY ###
# 'auto', 'inmem' or 'layer'
-kylin.cube.algorithm=auto
+kylin.cube.algorithm=layer
# A smaller threshold prefers layer, a larger threshold prefers in-mem
kylin.cube.algorithm.layer-or-inmem-threshold=7
[2/3] kylin git commit: KYLIN-2525 skip signature check for older
version of cubes
Posted by li...@apache.org.
KYLIN-2525 skip signature check for older version of cubes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6b4592f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6b4592f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6b4592f
Branch: refs/heads/master
Commit: c6b4592fca047d1fc4094a0812774ddd2d8917f9
Parents: a776ef6
Author: Li Yang <li...@apache.org>
Authored: Wed Mar 29 11:57:10 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Mar 29 21:02:16 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 8 ++++----
.../main/java/org/apache/kylin/cube/model/CubeDesc.java | 11 +++++++++--
2 files changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6b4592f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 073f516..0a94fb2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -761,7 +761,7 @@ public class CubeManager implements IRealizationProvider {
CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName);
- if (!isITTestCube(cubeName))
+ if (!isSpecialTestCube(cubeName))
checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' must be same as descriptor name '%s', but it is not", cubeName, cubeDesc.getName());
if (!cubeDesc.getError().isEmpty()) {
@@ -791,9 +791,9 @@ public class CubeManager implements IRealizationProvider {
}
}
- private boolean isITTestCube(String cubeName) {
- return config.isDevEnv() //
- && (cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming"));
+ private boolean isSpecialTestCube(String cubeName) {
+ return cubeName.equals("kylin_sales_cube") //
+ || config.isDevEnv() && (cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming"));
}
private MetadataManager getMetadataManager() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6b4592f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index c1469fe..b391055 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -479,8 +479,15 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return true;
}
- if (KylinVersion.getCurrentVersion().isCompatibleWith(new KylinVersion(getVersion())) && !KylinVersion.getCurrentVersion().isSignatureCompatibleWith(new KylinVersion(getVersion()))) {
- logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), getVersion());
+ KylinVersion cubeVersion = new KylinVersion(getVersion());
+ KylinVersion kylinVersion = KylinVersion.getCurrentVersion();
+ if (!kylinVersion.isCompatibleWith(cubeVersion)) {
+ logger.info("checkSignature on {} is skipped as the its version {} is different from kylin version {}", getName(), cubeVersion, kylinVersion);
+ return true;
+ }
+
+ if (kylinVersion.isCompatibleWith(cubeVersion) && !kylinVersion.isSignatureCompatibleWith(cubeVersion)) {
+ logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), cubeVersion);
return true;
}
[3/3] kylin git commit: KYLIN-2525 tolerate job metadata exception in
StorageCleanupJob
Posted by li...@apache.org.
KYLIN-2525 tolerate job metadata exception in StorageCleanupJob
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa3ee3ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa3ee3ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa3ee3ff
Branch: refs/heads/master
Commit: fa3ee3ffbbb11ac3d4dc79dbb719f56a7e913857
Parents: c6b4592
Author: Li Yang <li...@apache.org>
Authored: Wed Mar 29 18:03:59 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Mar 29 21:02:23 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/tool/StorageCleanupJob.java | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa3ee3ff/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 477c58a..f1a3ebe 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -46,6 +46,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -57,7 +58,6 @@ import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.source.hive.HiveClientFactory;
-import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.source.hive.IHiveClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -276,9 +276,14 @@ public class StorageCleanupJob extends AbstractApplication {
sb.append(jobId).append("(").append(state).append("), ");
}
- String segmentId = getSegmentIdFromJobId(jobId);
- if (segmentId != null) {//some jobs are not cubing jobs
- segmentId2JobId.put(segmentId, jobId);
+ try {
+ String segmentId = getSegmentIdFromJobId(jobId);
+ if (segmentId != null) {//some jobs are not cubing jobs
+ segmentId2JobId.put(segmentId, jobId);
+ }
+ } catch (Exception ex) {
+ logger.warn("Failed to find segment ID from job ID " + jobId + ", ignore it");
+ // some older version job metadata may fail to read, ignore it
}
}
logger.info("Working jobIDs: " + workingJobList);