You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/11 09:13:14 UTC
[3/9] kylin git commit: KYLIN-2531 convert old tableExt jackson to
new one instead of removing it
KYLIN-2531 convert old tableExt jackson to new one instead of removing it
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/266a62a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/266a62a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/266a62a5
Branch: refs/heads/master
Commit: 266a62a5ab6b0b39b08b79d3f3d4d1948e111247
Parents: 38fe432
Author: Cheng Wang <ch...@kyligence.io>
Authored: Wed Apr 5 14:43:52 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Wed Apr 5 14:46:12 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/MetadataManager.java | 64 +++++++++++++++-----
.../kylin/metadata/MetadataManagerTest.java | 41 +++++++++++--
2 files changed, 85 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/266a62a5/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 9427ace..f17983f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -19,8 +19,10 @@
package org.apache.kylin.metadata;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -30,8 +32,10 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
@@ -47,6 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* Serves (and caches) metadata for Kylin instance.
@@ -207,11 +212,7 @@ public class MetadataManager {
result.setUuid(UUID.randomUUID().toString());
result.setLastModified(0);
result.init();
- try {
- saveTableExt(result);
- } catch (IOException ex) {
- logger.warn("Failed to save TableExt", ex);
- }
+ srcTableExdMap.put(result.getName(), result);
}
return result;
}
@@ -224,9 +225,14 @@ public class MetadataManager {
tableExt.init();
String path = tableExt.getResourcePath();
- getStore().putResource(path, tableExt, TABLE_EXT_SERIALIZER);
- srcTableExdMap.put(tableExt.getName(), tableExt);
+ ResourceStore store = getStore();
+
+ TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER);
+ if (t != null && t.getName() == null)
+ store.deleteResource(path);
+
+ store.putResource(path, tableExt, TABLE_EXT_SERIALIZER);
}
public void removeTableExt(String tableName) throws IOException {
@@ -387,22 +393,52 @@ public class MetadataManager {
private TableExtDesc reloadTableExtAt(String path) throws IOException {
ResourceStore store = getStore();
TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER);
+
if (t == null) {
return null;
}
+
+ // convert old tableExt json to new one
+ if (t.getName() == null) {
+ t = convertOldTableExtToNewer(path);
+ }
+
t.init();
- String name = t.getName();
+ srcTableExdMap.putLocal(t.getName(), t);
+ return t;
+ }
- // remove old json
- if (name == null) {
- getStore().deleteResource(path);
- return null;
+ private TableExtDesc convertOldTableExtToNewer(String path) throws IOException {
+ Map<String, String> attrs = Maps.newHashMap();
+
+ ResourceStore store = getStore();
+ RawResource res = store.getResource(path);
+
+ InputStream is = res.inputStream;
+
+ try {
+ attrs.putAll(JsonUtil.readValue(is, HashMap.class));
+ } finally {
+ if (is != null)
+ is.close();
}
- srcTableExdMap.putLocal(name, t);
+ String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY);
- return t;
+ // parse table identity from file name
+ String file = path;
+ if (file.indexOf("/") > -1) {
+ file = file.substring(file.lastIndexOf("/") + 1);
+ }
+ String tableIdentity = file.substring(0, file.length() - MetadataConstants.FILE_SURFIX.length()).toUpperCase();
+ TableExtDesc result = new TableExtDesc();
+ result.setName(tableIdentity);
+ result.setUuid(UUID.randomUUID().toString());
+ result.setLastModified(0);
+ result.init();
+ result.setCardinality(cardinality);
+ return result;
}
private void reloadAllExternalFilter() throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/266a62a5/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index a861ee1..eb04287 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -23,10 +23,16 @@ import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
@@ -51,7 +57,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
public void after() throws Exception {
this.cleanupTestMetadata();
}
-
+
@Test
public void testCiModel() {
MetadataManager mgr = getInstance(getTestConfig());
@@ -59,7 +65,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
DataModelDesc im = mgr.getDataModelDesc("ci_inner_join_model");
assertSnowflakeQuality(lm);
assertSnowflakeQuality(im);
-
+
// check inner/left models are identical apart from the left/inner difference
assertEquals(lm.getJoinTables().length, im.getJoinTables().length);
for (int i = 0, n = im.getJoinTables().length; i < n; i++) {
@@ -73,7 +79,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
assertTrue(lt.getJoin().isLeftJoin());
assertTrue(it.getJoin().isInnerJoin());
}
-
+
assertEquals(lm.getDimensions().size(), im.getDimensions().size());
for (int i = 0, n = im.getDimensions().size(); i < n; i++) {
ModelDimensionDesc ld = lm.getDimensions().get(i);
@@ -81,10 +87,10 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
assertEquals(ld.getTable(), id.getTable());
assertArrayEquals(ld.getColumns(), id.getColumns());
}
-
+
assertArrayEquals(lm.getMetrics(), im.getMetrics());
}
-
+
private void assertSnowflakeQuality(DataModelDesc model) {
Assert.assertNotNull(model);
try {
@@ -93,7 +99,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
} catch (IllegalArgumentException ex) {
// excepted
}
-
+
Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
@@ -148,4 +154,27 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
getInstance(getTestConfig()).removeTableExt("TEST.TEST_TABLE");
}
+
+ @Test
+ public void testTableExtCompatibility() throws IOException {
+ String tableName = "TEST.TEST_TABLE";
+ Map<String, String> oldTableExt = new HashMap<>();
+ oldTableExt.put(MetadataConstants.TABLE_EXD_CARDINALITY, "1,2,3,4");
+ mockUpOldTableExtJson(tableName, oldTableExt);
+ TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt(tableName);
+ Assert.assertEquals("1,2,3,4,", tableExtDesc.getCardinality());
+ getInstance(getTestConfig()).removeTableExt(tableName);
+ }
+
+ private void mockUpOldTableExtJson(String tableId, Map<String, String> tableExdProperties) throws IOException {
+ String path = TableDesc.concatExdResourcePath(tableId);
+
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ JsonUtil.writeValueIndent(os, tableExdProperties);
+ os.flush();
+ InputStream is = new ByteArrayInputStream(os.toByteArray());
+ getStore().putResource(path, is, System.currentTimeMillis());
+ os.close();
+ is.close();
+ }
}