You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/06/25 11:21:57 UTC
incubator-kylin git commit: KYLIN-540 add the step for coprocessor
upgrade, and duplicate dict for FK
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging faeb8a968 -> b08df2bce
KYLIN-540 add the step for coprocessor upgrade, and duplicate dict for FK
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b08df2bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b08df2bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b08df2bc
Branch: refs/heads/0.7-staging
Commit: b08df2bce829efd2129c35985ccebbe9fcdb7a6b
Parents: faeb8a9
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jun 25 17:21:41 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jun 25 17:21:41 2015 +0800
----------------------------------------------------------------------
.../Migrate v0.6.x metadata to v0.7.md | 7 ++
.../apache/kylin/job/CubeMetadataUpgrade.java | 93 ++++++++++++++------
2 files changed, 71 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b08df2bc/docs/Operations/Migrate v0.6.x metadata to v0.7.md
----------------------------------------------------------------------
diff --git a/docs/Operations/Migrate v0.6.x metadata to v0.7.md b/docs/Operations/Migrate v0.6.x metadata to v0.7.md
index 26140aa..badd135 100644
--- a/docs/Operations/Migrate v0.6.x metadata to v0.7.md
+++ b/docs/Operations/Migrate v0.6.x metadata to v0.7.md
@@ -38,4 +38,11 @@ Now the new format of metadata will be upload to the HBase to replace the old fo
hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar org.apache.kylin.common.persistence.ResourceTool reset
hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar org.apache.kylin.common.persistence.ResourceTool upload ./meta_dump_v2
+# Update HTables to use new coprocessor
+
+Kylin uses HBase coprocessor to do server side aggregation; When Kylin instance upgrades to V0.7, the HTables that created in V0.6 should also be updated to use the new coprocessor:
+
+ hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar org.apache.kylin.job.tools.DeployCoprocessorCLI ${KYLIN_HOME}/lib/kylin-coprocessor-x.x.x-SNAPSHOT.jar
+
+
Done; Update your v0.7 Kylin configure to point to the same metadata HBase table, then start Kylin server; Check whether all cubes and other information are kept;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b08df2bc/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
index b970775..13103ac 100644
--- a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
+++ b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
@@ -30,6 +30,7 @@ import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeDescUpgrader;
import org.apache.kylin.cube.CubeManager;
@@ -54,8 +55,7 @@ import org.apache.kylin.job.hadoop.hbase.CreateHTableJob;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.*;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
@@ -101,6 +101,7 @@ public class CubeMetadataUpgrade {
upgradeProjectInstance();
upgradeCubeInstance();
upgradeJobInstance();
+ copyDictionaryForFK();
verify();
}
@@ -395,6 +396,7 @@ public class CubeMetadataUpgrade {
newInstance.setStatus(RealizationStatusEnum.READY);
}
+
List<org.apache.kylin.cube.CubeSegment> newSegments = Lists.newArrayList();
// segment
for (CubeSegment segment : cubeInstance.getSegments()) {
@@ -449,10 +451,69 @@ public class CubeMetadataUpgrade {
}
}
+ private void copyDictionaryForFK() {
+ CubeManager cubeManager = CubeManager.getInstance(config);
+ List<org.apache.kylin.cube.CubeInstance> cubeInstances = cubeManager.listAllCubes();
+
+ Set<String> changedCubes = Sets.newHashSet();
+ for (org.apache.kylin.cube.CubeInstance newInstance : cubeInstances) {
+
+ boolean updated = false;
+ DataModelDesc dataModelDesc = null;
+ try {
+ String modelName = this.getCubeDescManager().getCubeDesc(newInstance.getDescName()).getModelName();
+ dataModelDesc = this.getMetadataManager().getDataModelDesc(modelName);
+ Map<String, String> pkToFK = Maps.newHashMap();
+ for (LookupDesc lookupDesc : dataModelDesc.getLookups()) {
+ if (lookupDesc.getJoin() != null) {
+ JoinDesc join = lookupDesc.getJoin();
+ for (int i=0; i< join.getForeignKey().length; i++) {
+ pkToFK.put(lookupDesc.getTable() + "/" +join.getPrimaryKey()[i], dataModelDesc.getFactTable() + "/" +join.getForeignKey()[i]);
+ }
+ }
+ }
+
+ List<Pair<String, String>> newDictionaries = Lists.newArrayList();
+
+ // segment
+ for (org.apache.kylin.cube.CubeSegment newSeg : newInstance.getSegments()) {
+
+ for (Map.Entry<String, String> e : newSeg.getDictionaries().entrySet()) {
+ String key = e.getKey();
+ if (pkToFK.containsKey(key) && !newSeg.getDictionaries().containsKey(pkToFK.get(key))) {
+ logger.debug("Duplicate dictionary for FK " + pkToFK.get(key) + " in cube " + newInstance.getName());
+ changedCubes.add(newInstance.getName());
+ newDictionaries.add(new Pair(pkToFK.get(key), e.getValue()));
+
+ }
+ }
+ for(Pair<String, String> dict: newDictionaries) {
+ newSeg.getDictionaries().put(dict.getFirst(), dict.getSecond());
+ updated = true;
+ }
+ }
+
+ if (updated)
+ store.putResource(newInstance.getResourcePath(), newInstance, CubeManager.CUBE_SERIALIZER);
+ } catch (Exception e) {
+ logger.error("Error during upgrade cube instance " + newInstance.getName(), e);
+ }
+ }
+
+ if (changedCubes.size() > 0)
+ logger.info("Updated these cubeInstances: " + changedCubes);
+ }
+
private MetadataManager getMetadataManager() {
return MetadataManager.getInstance(config);
}
+
+ private CubeDescManager getCubeDescManager() {
+ return CubeDescManager.getInstance(config);
+ }
+
+
private ResourceStore getStore() {
return ResourceStore.getStore(config);
}
@@ -650,33 +711,6 @@ public class CubeMetadataUpgrade {
}
- public static void main1(String[] args) {
-
- if (!(args != null && args.length == 1)) {
- System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder>; e.g, /export/kylin/meta");
- return;
- }
-
- String exportFolder = args[0];
-
- File oldMetaFolder = new File(exportFolder);
- if (!oldMetaFolder.exists()) {
- System.out.println("Provided folder doesn't exist: '" + exportFolder + "'");
- return;
- }
-
- if (!oldMetaFolder.isDirectory()) {
- System.out.println("Provided folder is not a directory: '" + exportFolder + "'");
- return;
- }
-
-
- String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v2";
- CubeMetadataUpgrade instance = new CubeMetadataUpgrade(newMetadataUrl);
- instance.verify();
- }
-
-
public static void main(String[] args) {
if (!(args != null && (args.length == 1 || args.length == 2))) {
@@ -695,6 +729,7 @@ public class CubeMetadataUpgrade {
if (verify) {
instance = new CubeMetadataUpgrade(exportFolder);
instance.verify();
+ instance.addDictionaryForFK();
} else {
File oldMetaFolder = new File(exportFolder);
if (!oldMetaFolder.exists()) {