You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/07/11 02:52:04 UTC
[1/5] kylin git commit: KYLIN-1862 "table not found" in "Build
Dimension Dictionary" step
Repository: kylin
Updated Branches:
refs/heads/master e89af6c22 -> 4528ff630
KYLIN-1862 "table not found" in "Build Dimension Dictionary" step
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47feb29d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47feb29d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47feb29d
Branch: refs/heads/master
Commit: 47feb29d681a68663a78c3a189337740a6775268
Parents: e89af6c
Author: shaofengshi <sh...@apache.org>
Authored: Fri Jul 8 22:30:52 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 11 10:50:52 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/dict/DictionaryManager.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/47feb29d/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 74120dc..bd00f10 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -290,9 +290,9 @@ public class DictionaryManager {
MetadataManager metadataManager = MetadataManager.getInstance(config);
TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable));
if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
- tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable());
- String tableName = tableDesc.getMaterializedName();
- tableDesc.setName(tableName);
+ TableDesc materializedTbl = new TableDesc();
+ materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
+ materializedTbl.setName(tableDesc.getMaterializedName());
inpTable = SourceFactory.createReadableTable(tableDesc);
} else {
inpTable = SourceFactory.createReadableTable(tableDesc);
[4/5] kylin git commit: KYLIN-1656 set default ‘kylin.job.mapreduce.mapper.input.rows’ to 1million
Posted by sh...@apache.org.
KYLIN-1656 set default \u2018kylin.job.mapreduce.mapper.input.rows\u2019 to 1million
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5bce12cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5bce12cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5bce12cd
Branch: refs/heads/master
Commit: 5bce12cd885c3b9130c64274db9057a130307951
Parents: ccc83cd
Author: shaofengshi <sh...@apache.org>
Authored: Sat Jul 9 10:26:04 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 11 10:51:36 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../kylin/source/hive/CreateFlatHiveTableStep.java | 17 +++++++++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bce12cd/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4a017c4..312ecbf 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -320,7 +320,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getHadoopJobMapperInputRows() {
- return Integer.parseInt(getOptional("kylin.job.mapreduce.mapper.input.rows", "500000"));
+ return Integer.parseInt(getOptional("kylin.job.mapreduce.mapper.input.rows", "1000000"));
}
public boolean getRunAsRemoteCommand() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bce12cd/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index d8e2754..0297bfa 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.source.hive;
import java.io.IOException;
[5/5] kylin git commit: minor, use upper in hql
Posted by sh...@apache.org.
minor, use upper in hql
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4528ff63
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4528ff63
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4528ff63
Branch: refs/heads/master
Commit: 4528ff630415b2834c60d0b40713e6dbe2d80cff
Parents: 5bce12c
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jul 11 09:58:08 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 11 10:51:53 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/job/JoinedFlatTable.java | 12 +++---------
1 file changed, 3 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4528ff63/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 41c301a..31c92b6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -42,8 +42,7 @@ import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
/**
- * @author George Song (ysong1)
- *
+ *
*/
public class JoinedFlatTable {
@@ -115,11 +114,6 @@ public class JoinedFlatTable {
return sql.toString();
}
- public static String generateRedistributeDataStatement(IJoinedFlatTableDesc intermediateTableDesc) {
- final String tableName = intermediateTableDesc.getTableName();
- return "INSERT OVERWRITE TABLE " + tableName + " SELECT * FROM " + tableName + " distribute by rand();\n";
- }
-
public static String generateSelectDataStatement(IJoinedFlatTableDesc intermediateTableDesc) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT" + "\n");
@@ -222,9 +216,9 @@ public class JoinedFlatTable {
if (distDcol != null) {
String tblAlias = tableAliasMap.get(distDcol.getTable());
- sql.append(" distribute by ").append(tblAlias).append(".").append(distDcol.getName());
+ sql.append(" DISTRIBUTE BY ").append(tblAlias).append(".").append(distDcol.getName());
} else {
- sql.append(" distribute by rand()");
+ sql.append(" DISTRIBUTE BY RAND()");
}
}
[3/5] kylin git commit: KYLIN-1862 'table not found' in 'Build
Dimension Dictionary' step
Posted by sh...@apache.org.
KYLIN-1862 'table not found' in 'Build Dimension Dictionary' step
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ccc83cd1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ccc83cd1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ccc83cd1
Branch: refs/heads/master
Commit: ccc83cd1d2c27af0cf9dacd7fbb7f64ea1857b28
Parents: 41c3726
Author: shaofengshi <sh...@apache.org>
Authored: Sat Jul 9 09:50:29 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 11 10:51:18 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/dict/DictionaryManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ccc83cd1/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index bd00f10..ff088b9 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -293,7 +293,7 @@ public class DictionaryManager {
TableDesc materializedTbl = new TableDesc();
materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
materializedTbl.setName(tableDesc.getMaterializedName());
- inpTable = SourceFactory.createReadableTable(tableDesc);
+ inpTable = SourceFactory.createReadableTable(materializedTbl);
} else {
inpTable = SourceFactory.createReadableTable(tableDesc);
}
[2/5] kylin git commit: KYLIN-1861 change default “hive.auto.convert.join.noconditionaltask.size” to 100Mb
Posted by sh...@apache.org.
KYLIN-1861 change default \u201chive.auto.convert.join.noconditionaltask.size\u201d to 100Mb
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41c37265
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41c37265
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41c37265
Branch: refs/heads/master
Commit: 41c372659e916cef0125246a58b8d18b7d51de37
Parents: 47feb29
Author: shaofengshi <sh...@apache.org>
Authored: Sat Jul 9 09:14:55 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 11 10:51:06 2016 +0800
----------------------------------------------------------------------
build/conf/kylin_hive_conf.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/41c37265/build/conf/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_hive_conf.xml b/build/conf/kylin_hive_conf.xml
index b95c077..3d6109b 100644
--- a/build/conf/kylin_hive_conf.xml
+++ b/build/conf/kylin_hive_conf.xml
@@ -35,7 +35,7 @@
<property>
<name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>300000000</value>
+ <value>100000000</value>
<description>enable map-side join</description>
</property>