You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2016/07/20 06:21:15 UTC
[1/3] kylin git commit: Remove nonexistent task items for
EXECUTE_RESOURCE_ROOT from toDeleteResource
Repository: kylin
Updated Branches:
refs/heads/master 554874db3 -> 44ee970ca
Remove nonexistent task items for EXECUTE_RESOURCE_ROOT from toDeleteResource
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/44ee970c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/44ee970c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/44ee970c
Branch: refs/heads/master
Commit: 44ee970ca27aa8685cbef492d6e9e500f042b6f0
Parents: 07177ca
Author: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Authored: Thu Jul 14 14:59:12 2016 +0800
Committer: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Committed: Wed Jul 20 14:20:10 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/44ee970c/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
index f8d265b..dd482d6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
@@ -156,7 +156,6 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());
for (ExecutablePO task : executable.getTasks()) {
- toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid());
toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
}
}
[2/3] kylin git commit: KYLIN-1883: Consensus Problem when running
the tool, MetadataCleanupJob
Posted by nj...@apache.org.
KYLIN-1883: Consensus Problem when running the tool, MetadataCleanupJob
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07177caf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07177caf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07177caf
Branch: refs/heads/master
Commit: 07177caf7062ec3b631324924a35b6a02e1bde94
Parents: 1462371
Author: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Authored: Wed Jul 13 18:26:07 2016 +0800
Committer: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Committed: Wed Jul 20 14:20:10 2016 +0800
----------------------------------------------------------------------
.../engine/mr/steps/MetadataCleanupJob.java | 32 +++++++++-----------
1 file changed, 15 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/07177caf/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
index f300de9..f8d265b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
@@ -101,15 +101,6 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
public void cleanup() throws Exception {
CubeManager cubeManager = CubeManager.getInstance(config);
- Set<String> activeResourceList = Sets.newHashSet();
- for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
- for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
- activeResourceList.addAll(segment.getSnapshotPaths());
- activeResourceList.addAll(segment.getDictionaryPaths());
- activeResourceList.add(segment.getStatisticsResourcePath());
- }
- }
-
List<String> toDeleteResource = Lists.newArrayList();
// two level resources, snapshot tables and cube statistics
@@ -121,10 +112,8 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
if (snapshotNames != null)
for (String snapshot : snapshotNames) {
- if (!activeResourceList.contains(snapshot)) {
- if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
- toDeleteResource.add(snapshot);
- }
+ if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
+ toDeleteResource.add(snapshot);
}
}
}
@@ -140,13 +129,22 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
NavigableSet<String> dictionaries = getStore().listResources(tableCol);
if (dictionaries != null)
for (String dict : dictionaries)
- if (!activeResourceList.contains(dict)) {
- if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
- toDeleteResource.add(dict);
- }
+ if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
+ toDeleteResource.add(dict);
}
}
+ Set<String> activeResourceList = Sets.newHashSet();
+ for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
+ for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
+ activeResourceList.addAll(segment.getSnapshotPaths());
+ activeResourceList.addAll(segment.getDictionaryPaths());
+ activeResourceList.add(segment.getStatisticsResourcePath());
+ }
+ }
+
+ toDeleteResource.removeAll(activeResourceList);
+
// delete old and completed jobs
ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
List<ExecutablePO> allExecutable = executableDao.getJobs();
[3/3] kylin git commit: Change comment for hierarchy dimensions
Posted by nj...@apache.org.
Change comment for hierarchy dimensions
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1462371a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1462371a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1462371a
Branch: refs/heads/master
Commit: 1462371a59c9ff7eb3494076dcf3d7138edf62f8
Parents: 554874d
Author: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Authored: Wed Jul 13 18:04:48 2016 +0800
Committer: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Committed: Wed Jul 20 14:20:10 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/model/AggregationGroup.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1462371a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 0e25ac7..601ee0a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -36,7 +36,7 @@ public class AggregationGroup {
public static class HierarchyMask {
public long fullMask;// 00000111
public long[] allMasks;// 00000100,00000110,00000111
- public long[] dims;// 00000100,00000100,00000001
+ public long[] dims;// 00000100,00000010,00000001
}
@JsonProperty("includes")