You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by qh...@apache.org on 2015/08/05 11:14:02 UTC
incubator-kylin git commit: KYLIN-928
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 e6cf318a5 -> 849480a1f
KYLIN-928
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/849480a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/849480a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/849480a1
Branch: refs/heads/0.8
Commit: 849480a1f2d41d5703bc35f0c404dd787c871253
Parents: e6cf318
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Aug 5 17:15:49 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Aug 5 17:15:57 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/DateStrDictionary.java | 2 +-
.../java/org/apache/kylin/dict/Dictionary.java | 2 +-
.../apache/kylin/dict/DictionaryManager.java | 78 ++++++++++++--------
.../apache/kylin/dict/TimeStrDictionary.java | 2 +-
.../org/apache/kylin/dict/TrieDictionary.java | 10 +--
.../apache/kylin/dict/TrieDictionaryTest.java | 4 +-
.../engine/mr/steps/MergeCuboidMapperTest.java | 2 -
7 files changed, 57 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/849480a1/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index 6ff7cd8..1801cf2 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -174,7 +174,7 @@ public class DateStrDictionary extends Dictionary<String> {
}
@Override
- public boolean containedBy(Dictionary<?> other) {
+ public boolean contains(Dictionary<?> other) {
return this.equals(other);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/849480a1/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
index 45cd361..7c484ff 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
@@ -67,7 +67,7 @@ abstract public class Dictionary<T> implements Writable {
/**
* @return true if each entry of this dict is contained by the dict in param
*/
- abstract public boolean containedBy(Dictionary<?> another);
+ abstract public boolean contains(Dictionary<?> another);
/**
* Convenient form of <code>getIdFromValue(value, 0)</code>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/849480a1/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 881c9b2..4d241f5 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -18,12 +18,10 @@
package org.apache.kylin.dict;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.metadata.MetadataManager;
@@ -37,7 +35,13 @@ import org.apache.kylin.source.TableSourceFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
public class DictionaryManager {
@@ -67,14 +71,24 @@ public class DictionaryManager {
// ============================================================================
private KylinConfig config;
- private ConcurrentHashMap<String, DictionaryInfo> dictCache; // resource
+ private LoadingCache<String, DictionaryInfo> dictCache; // resource
// path ==>
// DictionaryInfo
private DictionaryManager(KylinConfig config) {
this.config = config;
- dictCache = new ConcurrentHashMap<String, DictionaryInfo>();
+ this.dictCache = CacheBuilder.newBuilder().weakValues().expireAfterWrite(10, TimeUnit.MINUTES).build(new CacheLoader<String, DictionaryInfo>() {
+ @Override
+ public DictionaryInfo load(String key) throws Exception {
+ DictionaryInfo dictInfo = DictionaryManager.this.load(key, true);
+ if (dictInfo == null) {
+ return NONE_INDICATOR;
+ } else {
+ return dictInfo;
+ }
+ }
+ });
}
public Dictionary<?> getDictionary(String resourcePath) throws IOException {
@@ -82,15 +96,17 @@ public class DictionaryManager {
return dictInfo == null ? null : dictInfo.getDictionaryObject();
}
- public DictionaryInfo getDictionaryInfo(String resourcePath) throws IOException {
- DictionaryInfo dictInfo = dictCache.get(resourcePath);
- if (dictInfo == null) {
- dictInfo = load(resourcePath, true);
- if (dictInfo == null)
- dictInfo = NONE_INDICATOR;
- dictCache.put(resourcePath, dictInfo);
+ public DictionaryInfo getDictionaryInfo(final String resourcePath) throws IOException {
+ try {
+ DictionaryInfo result = dictCache.get(resourcePath);
+ if (result == NONE_INDICATOR) {
+ return null;
+ } else {
+ return result;
+ }
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e.getCause());
}
- return dictInfo == NONE_INDICATOR ? null : dictInfo;
}
/**
@@ -99,20 +115,23 @@ public class DictionaryManager {
*/
public DictionaryInfo trySaveNewDict(Dictionary<?> newDict, DictionaryInfo newDictInfo) throws IOException {
+ newDictInfo.setCardinality(newDict.getSize());
newDictInfo.setDictionaryObject(newDict);
newDictInfo.setDictionaryClass(newDict.getClass().getName());
- DictionaryInfo largestDict = findLargestDict(newDictInfo);
- if (largestDict != null) {
- if (newDict.containedBy(largestDict.getDictionaryObject())) {
- logger.info("dictionary content " + newDict + ", is contained by dictionary at " + largestDict.getResourcePath());
- return largestDict;
- } else if (largestDict.getDictionaryObject().containedBy(newDict)) {
+ DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
+ if (largestDictInfo != null) {
+ largestDictInfo = getDictionaryInfo(largestDictInfo.getResourcePath());
+ Dictionary<?> largestDictObject = largestDictInfo.getDictionaryObject();
+ if (newDict.contains(largestDictObject)) {
logger.info("dictionary content " + newDict + " is by far the largest, save it");
return saveNewDict(newDictInfo);
+ } else if (largestDictObject.contains(newDict)) {
+ logger.info("dictionary content " + newDict + ", is contained by dictionary at " + largestDictInfo.getResourcePath());
+ return largestDictInfo;
} else {
logger.info("merge dict and save...");
- return mergeDictionary(Lists.newArrayList(newDictInfo, largestDict));
+ return mergeDictionary(Lists.newArrayList(newDictInfo, largestDictInfo));
}
} else {
logger.info("first dict of this column, save it directly");
@@ -182,7 +201,6 @@ public class DictionaryManager {
return dicts.get(0);
} else {
Dictionary<?> newDict = DictionaryGenerator.mergeDictionaries(DataType.getInstance(newDictInfo.getDataType()), dicts);
- newDictInfo.setCardinality(newDict.getSize());
return trySaveNewDict(newDict, newDictInfo);
}
}
@@ -217,8 +235,6 @@ public class DictionaryManager {
}
Dictionary<?> dictionary = DictionaryGenerator.buildDictionary(dictInfo, inpTable);
- dictInfo.setCardinality(dictionary.getSize());
-
return trySaveNewDict(dictionary, dictInfo);
}
@@ -260,7 +276,7 @@ public class DictionaryManager {
return null;
}
- private DictionaryInfo findLargestDict(DictionaryInfo dictInfo) throws IOException {
+ private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException {
ResourceStore store = MetadataManager.getInstance(config).getStore();
ArrayList<String> dictInfos = store.listResources(dictInfo.getResourceDir());
if (dictInfos == null || dictInfos.isEmpty()) {
@@ -268,7 +284,7 @@ public class DictionaryManager {
}
//Collections.sort(dictInfos);
- final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(dictInfos.get(0), dictInfos.get(dictInfos.size() - 1), DictionaryInfo.class, DictionaryInfoSerializer.FULL_SERIALIZER);
+ final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(dictInfos.get(0), dictInfos.get(dictInfos.size() - 1), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
DictionaryInfo largestDict = null;
for (DictionaryInfo dictionaryInfo : allResources) {
@@ -277,7 +293,7 @@ public class DictionaryManager {
continue;
}
- if (largestDict.getDictionaryObject().getSize() < dictionaryInfo.getDictionaryObject().getSize()) {
+ if (largestDict.getCardinality() < dictionaryInfo.getCardinality()) {
largestDict = dictionaryInfo;
}
}
@@ -287,7 +303,7 @@ public class DictionaryManager {
public void removeDictionary(String resourcePath) throws IOException {
ResourceStore store = MetadataManager.getInstance(config).getStore();
store.deleteResource(resourcePath);
- dictCache.remove(resourcePath);
+ dictCache.invalidate(resourcePath);
}
public void removeDictionaries(String srcTable, String srcCol) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/849480a1/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
index 6b58354..302247a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -106,7 +106,7 @@ public class TimeStrDictionary extends Dictionary<String> {
}
@Override
- public boolean containedBy(Dictionary<?> other) {
+ public boolean contains(Dictionary<?> other) {
return this.equals(other);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/849480a1/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index ab27b4c..de5e2f4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -481,14 +481,14 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
@Override
- public boolean containedBy(Dictionary other) {
- if (this.getSize() > other.getSize()) {
+ public boolean contains(Dictionary other) {
+ if (other.getSize() > this.getSize()) {
return false;
}
- for (int i = getMinId(); i <= getMaxId(); ++i) {
- T v = this.getValueFromId(i);
- if (!other.containsValue(v)) {
+ for (int i = other.getMinId(); i <= other.getMaxId(); ++i) {
+ T v = (T) other.getValueFromId(i);
+ if (!this.containsValue(v)) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/849480a1/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index f6031e8..5953339 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -139,8 +139,8 @@ public class TrieDictionaryTest {
baseId = new Random().nextInt(100);
TrieDictionary<String> dict2 = b.build(baseId);
- assertEquals(true,dict.containedBy(dict2));
- assertEquals(false,dict2.containedBy(dict));
+ assertEquals(true, dict2.contains(dict));
+ assertEquals(false, dict.contains(dict2));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/849480a1/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
index a8d9670..c3e7212 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
@@ -80,7 +80,6 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
values.add(new byte[] { 101, 101, 101 });
values.add(new byte[] { 102, 102, 102 });
Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(DataType.getInstance(newDictInfo.getDataType()), values);
- newDictInfo.setCardinality(dict.getSize());
dictionaryManager.trySaveNewDict(dict, newDictInfo);
((TrieDictionary) dict).dump(System.out);
@@ -133,7 +132,6 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
else
values.add(new byte[] { 98, 98, 98 });
Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(DataType.getInstance(newDictInfo.getDataType()), values);
- newDictInfo.setCardinality(dict.getSize());
dictionaryManager.trySaveNewDict(dict, newDictInfo);
((TrieDictionary) dict).dump(System.out);