You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/11 08:37:34 UTC
[10/14] incubator-kylin git commit: Migrate 0.6 metadata to v0.7,
exclude job history
Migrate 0.6 metadata to v0.7, exclude job history
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/00d81f9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/00d81f9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/00d81f9c
Branch: refs/heads/0.7
Commit: 00d81f9c6148897950541559df7dad5fbcf3877f
Parents: d38cbd1
Author: shaofengshi <sh...@apache.org>
Authored: Wed Jun 10 16:21:13 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Jun 10 16:21:13 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeDescUpgrader.java | 27 ++++++++++++++++++--
.../apache/kylin/cube/model/RowKeyColDesc.java | 2 +-
.../apache/kylin/job/CubeMetadataUpgrade.java | 10 +++++++-
3 files changed, 35 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/00d81f9c/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
index 139ec2f..5fb99a0 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
@@ -72,20 +72,43 @@ public class CubeDescUpgrader {
updateDimensions(oldModel, newModel);
- updateRowkeyDictionary(oldModel, newModel);
+ updateRowkeyDictionary(newModel);
return newModel;
}
- private void updateRowkeyDictionary(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ private void updateRowkeyDictionary(org.apache.kylin.cube.model.CubeDesc newModel) {
+ DataModelDesc modelDesc = newModel.getModel();
+ Map<String, String> pkToFK = Maps.newHashMap();
+ for (LookupDesc lookupDesc : modelDesc.getLookups()) {
+ if (lookupDesc.getJoin() != null) {
+ JoinDesc join = lookupDesc.getJoin();
+ for (int i=0; i< join.getForeignKey().length; i++) {
+ pkToFK.put(join.getPrimaryKey()[i], join.getForeignKey()[i]);
+ }
+ }
+ }
RowKeyDesc rowKey = newModel.getRowkey();
for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) {
if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0)
rowkeyCol.setDictionary("true");
+
+ if (pkToFK.containsKey(rowkeyCol.getColumn())) {
+ rowkeyCol.setColumn(pkToFK.get(rowkeyCol.getColumn()));
+ }
}
+ for (String[] aggregationGroup : rowKey.getAggregationGroups()) {
+ for (int i=0; i< aggregationGroup.length; i++) {
+ if (pkToFK.containsKey(aggregationGroup[i])) {
+ aggregationGroup[i] = pkToFK.get(aggregationGroup[i]);
+ }
+ }
+ }
+
+
}
private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/00d81f9c/cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 7dca61b..b3a2253 100644
--- a/cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -51,7 +51,7 @@ public class RowKeyColDesc {
return column;
}
- void setColumn(String column) {
+ public void setColumn(String column) {
this.column = column;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/00d81f9c/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
index 0b06323..8d4cdc5 100644
--- a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
+++ b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
@@ -63,6 +63,7 @@ import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
import java.io.File;
+import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
@@ -700,7 +701,14 @@ public class CubeMetadataUpgrade {
String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v2";
try {
FileUtils.deleteDirectory(new File(newMetadataUrl));
- FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl));
+ FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl), new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ if (pathname.getAbsolutePath().contains(ResourceStore.JOB_PATH_ROOT) || pathname.getAbsolutePath().contains(ResourceStore.JOB_OUTPUT_PATH_ROOT))
+ return false;
+ return true;
+ }
+ });
} catch (IOException e) {
e.printStackTrace();
}