You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/13 10:37:25 UTC
[19/22] kylin git commit: Merge commit
'a36288142f30cad931881107872adb4ba0a574a3'
Merge commit 'a36288142f30cad931881107872adb4ba0a574a3'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e9ec116c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e9ec116c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e9ec116c
Branch: refs/heads/master-hbase0.98
Commit: e9ec116c247072603a5405ddba425fd349ba670b
Parents: f44f828 a362881
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Apr 11 17:12:46 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Apr 11 17:12:46 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 1 +
.../kylin/common/util/HiveCmdBuilder.java | 16 +++-
.../kylin/common/util/HiveCmdBuilderTest.java | 1 +
.../cube/model/CubeJoinedFlatTableDesc.java | 4 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 34 +++----
.../kylin/job/engine/JobEngineConfig.java | 2 +-
.../apache/kylin/job/JoinedFlatTableTest.java | 4 +-
.../metadata/filter/CompareTupleFilter.java | 14 +++
.../filter/FilterOptimizeTransformer.java | 89 ++++++++++++++++++
.../kylin/metadata/model/FunctionDesc.java | 8 +-
.../kylin/metadata/model/ParameterDesc.java | 58 ++++++------
.../apache/kylin/metadata/model/TblColRef.java | 10 ++
.../gtrecord/GTCubeStorageQueryBase.java | 2 +-
.../gtrecord/StorageResponseGTScatter.java | 22 +++--
.../engine/mr/common/HadoopStatusGetter.java | 15 +--
examples/test_case_data/sandbox/hdfs-site.xml | 4 +
.../test_case_data/sandbox/kylin.properties | 1 +
.../src/test/resources/query/sql/query104.sql | 31 +++++++
.../resources/query/sql_casewhen/query55.sql | 22 +++++
.../query/sql_casewhen/query55.sql.disable | 22 -----
.../resources/query/sql_subquery/query15.sql | 29 ++++++
.../resources/query/sql_subquery/query16.sql | 38 ++++++++
.../kylin/query/relnode/OLAPAggregateRel.java | 13 ++-
.../kylin/query/relnode/OLAPFilterRel.java | 25 ++---
.../kylin/query/relnode/OLAPProjectRel.java | 12 +--
.../kylin/query/relnode/OLAPTableScan.java | 2 +-
.../org/apache/kylin/query/udf/ConcatUDF.java | 28 ++++++
.../org/apache/kylin/rest/util/QueryUtil.java | 15 ---
.../apache/kylin/rest/util/QueryUtilTest.java | 6 --
.../apache/kylin/source/hive/HiveMRInput.java | 97 +++++++++-----------
.../kylin/storage/hbase/HBaseConnection.java | 7 +-
.../hbase/util/DeployCoprocessorCLI.java | 1 +
32 files changed, 425 insertions(+), 208 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --cc core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 7313630,5553d34..54c1e61
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@@ -19,8 -19,8 +19,9 @@@
package org.apache.kylin.job;
import java.io.File;
+import java.lang.reflect.Method;
import java.util.HashSet;
+ import java.util.Map;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
@@@ -49,18 -46,18 +50,17 @@@ public class JoinedFlatTable
return storageDfsDir + "/" + flatDesc.getTableName();
}
- public static String generateHiveSetStatements(JobEngineConfig engineConfig) {
- public static String generateHiveInitStatements(
- String flatTableDatabase, String kylinHiveFile, Map<String, String> cubeOverrides) {
++ public static String generateHiveInitStatements(String flatTableDatabase, String kylinHiveFile, Map<String, String> cubeOverrides) {
+
StringBuilder buffer = new StringBuilder();
+ buffer.append("USE ").append(flatTableDatabase).append(";\n");
try {
- File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath());
-
- if (hadoopPropertiesFile.exists()) {
+ File file = new File(kylinHiveFile);
+ if (file.exists()) {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- DocumentBuilder builder;
- Document doc;
- builder = factory.newDocumentBuilder();
- doc = builder.parse(hadoopPropertiesFile);
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document doc = builder.parse(file);
NodeList nl = doc.getElementsByTagName("property");
for (int i = 0; i < nl.getLength(); i++) {
String name = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
@@@ -101,24 -103,8 +106,19 @@@
return ddl.toString();
}
- public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc, JobEngineConfig engineConfig) {
+ public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) {
+ final KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
- StringBuilder sql = new StringBuilder();
-
+ if (kylinConfig.isAdvancedFlatTableUsed()) {
+ try {
+ Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass());
+ Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class, JobEngineConfig.class);
- return (String) method.invoke(null, flatDesc, engineConfig);
++ return (String) method.invoke(null, flatDesc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
- sql.append(generateHiveSetStatements(engineConfig));
- sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n");
-
- return sql.toString();
+ return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n";
}
public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------