You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/23 09:31:56 UTC
[09/50] [abbrv] incubator-kylin git commit: KYLIN-540 Data model
upgrade for legacy cube descs
KYLIN-540 Data model upgrade for legacy cube descs
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/149744d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/149744d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/149744d9
Branch: refs/heads/inverted-index
Commit: 149744d96857d01a1aaa546f42e06eed27ca0fcd
Parents: 8298424
Author: Shao Feng, Shi <sh...@hotmail.com>
Authored: Thu Jan 22 14:04:45 2015 +0800
Committer: Shao Feng, Shi <sh...@hotmail.com>
Committed: Thu Jan 22 14:04:45 2015 +0800
----------------------------------------------------------------------
.../java/com/kylinolap/cube/CubeDescUpgrader.java | 17 ++++++++++++++++-
.../kylinolap/cube/model/CubeMetadataUpgrade.java | 16 ++++++++++++----
.../com/kylinolap/cube/model/RowKeyColDesc.java | 4 ++++
.../kylinolap/metadata/MetadataUpgradeTest.java | 2 ++
...test_kylin_cube_without_slr_left_join_desc.json | 6 +++---
5 files changed, 37 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/149744d9/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java b/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java
index 27f03fb..0254a31 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java
@@ -17,6 +17,8 @@ import com.kylinolap.common.persistence.JsonSerializer;
import com.kylinolap.common.persistence.ResourceStore;
import com.kylinolap.common.persistence.Serializer;
import com.kylinolap.cube.model.HierarchyDesc;
+import com.kylinolap.cube.model.RowKeyColDesc;
+import com.kylinolap.cube.model.RowKeyDesc;
import com.kylinolap.cube.model.v1.CubePartitionDesc;
import com.kylinolap.metadata.MetadataManager;
import com.kylinolap.metadata.model.DataModelDesc;
@@ -46,12 +48,25 @@ public class CubeDescUpgrader {
DataModelDesc model = extractDataModel(oldModel, newModel);
newModel.setModel(model);
-
+
updateDimensions(oldModel, newModel);
+ updateRowkeyDictionary(oldModel, newModel);
+
return newModel;
}
+ private void updateRowkeyDictionary(com.kylinolap.cube.model.v1.CubeDesc oldModel, com.kylinolap.cube.model.CubeDesc newModel) {
+
+ RowKeyDesc rowKey = newModel.getRowkey();
+
+ for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) {
+ if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0)
+ rowkeyCol.setDictionary("true");
+ }
+
+ }
+
private void copyUnChangedProperties(com.kylinolap.cube.model.v1.CubeDesc oldModel, com.kylinolap.cube.model.CubeDesc newModel) {
newModel.setUuid(oldModel.getUuid());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/149744d9/cube/src/main/java/com/kylinolap/cube/model/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/CubeMetadataUpgrade.java b/cube/src/main/java/com/kylinolap/cube/model/CubeMetadataUpgrade.java
index bf77a08..e08e6c6 100644
--- a/cube/src/main/java/com/kylinolap/cube/model/CubeMetadataUpgrade.java
+++ b/cube/src/main/java/com/kylinolap/cube/model/CubeMetadataUpgrade.java
@@ -62,6 +62,14 @@ public class CubeMetadataUpgrade {
upgradeCubeDesc();
upgradeProjectInstance();
+ verify();
+
+ }
+
+ private void verify() {
+ MetadataManager.getInstance(config).reload();
+ CubeDescManager.clearCache();
+ CubeDescManager.getInstance(config);
}
private List<String> listResourceStore(String pathRoot) {
@@ -278,20 +286,20 @@ public class CubeMetadataUpgrade {
CubeMetadataUpgrade instance = new CubeMetadataUpgrade(newMetadataUrl);
instance.upgrade();
-
- logger.info("Run CubeMetadataUpgrade completed, check the following messages.");
- logger.info("The following resources have been successfully updated in : " + newMetadataUrl);
+ logger.info("=================================================================");
+ logger.info("Run CubeMetadataUpgrade completed; The following resources have been successfully updated in : " + newMetadataUrl);
for (String s : instance.updatedResources) {
logger.info(s);
}
+ logger.info("=================================================================");
if (instance.errorMsgs.size() > 0) {
logger.info("Here are the error/warning messages, you may need check:");
for (String s : instance.errorMsgs) {
logger.warn(s);
}
} else {
- logger.info("No error or warning messages; Looks all good.");
+ logger.info("No error or warning messages; The migration is success.");
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/149744d9/cube/src/main/java/com/kylinolap/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/RowKeyColDesc.java b/cube/src/main/java/com/kylinolap/cube/model/RowKeyColDesc.java
index d190e08..488c251 100644
--- a/cube/src/main/java/com/kylinolap/cube/model/RowKeyColDesc.java
+++ b/cube/src/main/java/com/kylinolap/cube/model/RowKeyColDesc.java
@@ -77,6 +77,10 @@ public class RowKeyColDesc {
this.colRef = colRef;
}
+ public void setDictionary(String dictionary) {
+ this.dictionary = dictionary;
+ }
+
@Override
public String toString() {
return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/149744d9/cube/src/test/java/com/kylinolap/metadata/MetadataUpgradeTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/metadata/MetadataUpgradeTest.java b/cube/src/test/java/com/kylinolap/metadata/MetadataUpgradeTest.java
index f49a317..5c97f25 100644
--- a/cube/src/test/java/com/kylinolap/metadata/MetadataUpgradeTest.java
+++ b/cube/src/test/java/com/kylinolap/metadata/MetadataUpgradeTest.java
@@ -5,6 +5,7 @@ import java.util.List;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import com.kylinolap.common.KylinConfig;
@@ -23,6 +24,7 @@ import com.kylinolap.metadata.model.TableDesc;
* @author shaoshi
*
*/
+@Ignore("Not needed, the migrate and test has been moved to CubeMetadataUpgrade.java")
public class MetadataUpgradeTest extends LocalFileMetadataTestCase {
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/149744d9/examples/test_case_data/localmeta_v1/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta_v1/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta_v1/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index d79ae72..b7c275b 100644
--- a/examples/test_case_data/localmeta_v1/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta_v1/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -181,9 +181,9 @@
"parameter": {
"type": "column",
"value": "seller_id"
- },
- "dependent_measure_ref" : "seller_cnt_hll"
- }
+ }
+ },
+ "dependent_measure_ref" : "seller_cnt_hll"
},
{
"id": "7",