You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/01/18 08:44:05 UTC
[1/2] kylin git commit: KYLIN-2404 Add "hive.merge.mapfiles" and
"hive.merge.mapredfiles" to kylin_hive_conf.xml
Repository: kylin
Updated Branches:
refs/heads/master 837bd8200 -> 131a3f3f0
KYLIN-2404 Add "hive.merge.mapfiles" and "hive.merge.mapredfiles" to kylin_hive_conf.xml
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/131a3f3f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/131a3f3f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/131a3f3f
Branch: refs/heads/master
Commit: 131a3f3f0ca3fbcc8000188b9973c27406bd9a90
Parents: 6e30376
Author: shaofengshi <sh...@apache.org>
Authored: Wed Jan 18 09:33:19 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Jan 18 16:28:05 2017 +0800
----------------------------------------------------------------------
build/conf/kylin_hive_conf.xml | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/131a3f3f/build/conf/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_hive_conf.xml b/build/conf/kylin_hive_conf.xml
index 2fc6dab..f01d08e 100644
--- a/build/conf/kylin_hive_conf.xml
+++ b/build/conf/kylin_hive_conf.xml
@@ -88,4 +88,15 @@
<description>Collect statistics for newly created intermediate table</description>
</property>
+ <property>
+ <name>hive.merge.mapfiles</name>
+ <value>false</value>
+ <description>Disable Hive's auto merge</description>
+ </property>
+
+ <property>
+ <name>hive.merge.mapredfiles</name>
+ <value>false</value>
+ <description>Disable Hive's auto merge</description>
+ </property>
</configuration>
[2/2] kylin git commit: Opt some constant value
Posted by sh...@apache.org.
Opt some constant value
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e303767
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e303767
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e303767
Branch: refs/heads/master
Commit: 6e30376752d95bafa4a1773d39c2f798f75c35a7
Parents: 837bd82
Author: xiefan46 <95...@qq.com>
Authored: Tue Jan 17 10:34:20 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Jan 18 16:28:05 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/TrieDictionaryForest.java | 77 +++++++++++++++-----
1 file changed, 57 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e303767/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 04292d2..1023892 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -51,6 +51,14 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
private ArrayList<ByteArray> maxValue;
+ private int minId;
+
+ private int maxId;
+
+ private int sizeOfId;
+
+ private int sizeOfValue;
+
public TrieDictionaryForest() { // default constructor for Writable interface
}
@@ -65,42 +73,28 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
this.accuOffset = accuOffset;
this.bytesConvert = bytesConverter;
this.baseId = baseId;
- initMaxValue();
+ initConstantValue();
initForestCache();
}
@Override
public int getMinId() {
- if (trees.isEmpty())
- return baseId;
- return trees.get(0).getMinId() + baseId;
+ return this.minId;
}
@Override
public int getMaxId() {
- if (trees.isEmpty())
- return baseId - 1;
- int index = trees.size() - 1;
- int id = accuOffset.get(index) + trees.get(index).getMaxId() + baseId;
- return id;
+ return this.maxId;
}
@Override
public int getSizeOfId() {
- if (trees.isEmpty())
- return 1;
- int maxOffset = accuOffset.get(accuOffset.size() - 1);
- TrieDictionary<T> lastTree = trees.get(trees.size() - 1);
- int sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1L);
- return sizeOfId;
+ return this.sizeOfId;
}
@Override
public int getSizeOfValue() {
- int maxValue = 0;
- for (TrieDictionary<T> tree : trees)
- maxValue = Math.max(maxValue, tree.getSizeOfValue());
- return maxValue;
+ return this.sizeOfValue;
}
@Override
@@ -340,7 +334,16 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
}
- private void initMaxValue() throws IllegalStateException {
+ private void initConstantValue() throws IllegalStateException {
+ initMaxValueForEachTrie();
+ initMaxId();
+ initMinId();
+ initSizeOfId();
+ initSizeOfValue();
+ }
+
+ private void initMaxValueForEachTrie(){
+ //init max value
this.maxValue = new ArrayList<>();
if (this.trees == null || trees.isEmpty()) {
return;
@@ -353,6 +356,40 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
}
+ private void initMaxId(){
+ if (trees.isEmpty()) {
+ this.maxId = baseId - 1;
+ return;
+ }
+ int index = trees.size() - 1;
+ this.maxId = accuOffset.get(index) + trees.get(index).getMaxId() + baseId;
+ }
+
+ private void initMinId(){
+ if (trees.isEmpty()) {
+ this.minId = baseId;
+ return;
+ }
+ this.minId = trees.get(0).getMinId() + baseId;
+ }
+
+ private void initSizeOfId(){
+ if (trees.isEmpty()){
+ this.sizeOfId = 1;
+ return;
+ }
+ int maxOffset = accuOffset.get(accuOffset.size() - 1);
+ TrieDictionary<T> lastTree = trees.get(trees.size() - 1);
+ this.sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1L);
+ }
+
+ private void initSizeOfValue(){
+ int maxValue = 0;
+ for (TrieDictionary<T> tree : trees)
+ maxValue = Math.max(maxValue, tree.getSizeOfValue());
+ this.sizeOfValue = maxValue;
+ }
+
private void initForestCache() {
enableCache();
for (TrieDictionary<T> tree : trees) { //disable duplicate cache