You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/17 03:32:14 UTC
[09/21] kylin git commit: KYLIN-2304 Only copy latest version dict
for global dict
KYLIN-2304 Only copy latest version dict for global dict
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb18ac59
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb18ac59
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb18ac59
Branch: refs/heads/master-cdh5.7
Commit: eb18ac59fa74b3f2f1f0163f7728739a1027a233
Parents: 4b00ec2
Author: kangkaisen <ka...@live.com>
Authored: Sun Dec 18 20:09:03 2016 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Sat Jan 14 19:07:59 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/AppendTrieDictionary.java | 19 +++++++++++++------
.../org/apache/kylin/dict/CachedTreeMap.java | 2 +-
2 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb18ac59/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 5029bc4..80403cc 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -1173,13 +1173,15 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
@Override
public AppendTrieDictionary copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
+ //copy appendDict
Configuration conf = new Configuration();
- AppendTrieDictionary newDict = new AppendTrieDictionary();
- newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues, bytesConvert);
- newDict.initDictSliceMap((CachedTreeMap)dictSliceMap);
- logger.info("Copy AppendDict from {} to {}", this.baseDir, newDict.baseDir);
- Path srcPath = new Path(this.baseDir);
- Path dstPath = new Path(newDict.baseDir);
+
+ Path base = new Path(baseDir);
+ FileSystem srcFs = FileSystem.get(base.toUri(), conf);
+ Path srcPath = CachedTreeMap.getLatestVersion(conf, srcFs, base);
+ Path dstPath = new Path(srcPath.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()));
+ logger.info("Copy appendDict from {} to {}", srcPath, dstPath);
+
FileSystem dstFs = FileSystem.get(dstPath.toUri(), conf);
if (dstFs.exists(dstPath)) {
logger.info("Delete existing AppendDict {}", dstPath);
@@ -1187,6 +1189,11 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath, FileSystem.get(dstPath.toUri(), conf), dstPath, false, true, conf);
+ // init new AppendTrieDictionary
+ AppendTrieDictionary newDict = new AppendTrieDictionary();
+ newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues, bytesConverter);
+ newDict.initDictSliceMap((CachedTreeMap) dictSliceMap);
+
return newDict;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb18ac59/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
index 3be4fd7..cc23261 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
@@ -225,7 +225,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
return getLatestVersion(conf, fs, baseDir).toUri().getPath();
}
- private static Path getLatestVersion(Configuration conf, FileSystem fs, Path baseDir) throws IOException {
+ public static Path getLatestVersion(Configuration conf, FileSystem fs, Path baseDir) throws IOException {
String[] versions = listAllVersions(fs, baseDir);
if (versions.length > 0) {
return new Path(versions[versions.length - 1]);