You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/03/21 03:11:00 UTC
[kylin] branch 2.3.x updated: KYLIN-3278 Revert "KYLIN-3084,
fix issue by using distribute by"
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.3.x by this push:
new aa2f8e7 KYLIN-3278 Revert "KYLIN-3084, fix issue by using distribute by"
aa2f8e7 is described below
commit aa2f8e796f387c1227c8fa216b9af69d0ee9de42
Author: shaofengshi <sh...@apache.org>
AuthorDate: Wed Mar 21 11:03:13 2018 +0800
KYLIN-3278 Revert "KYLIN-3084, fix issue by using distribute by"
This reverts commit 849faec7d63b59af58eadc331f00c96343f18cc8.
Remove unnecessary UT
---
.../apache/kylin/common/util/HiveCmdBuilder.java | 16 ---
.../java/org/apache/kylin/job/JoinedFlatTable.java | 6 +-
.../kylin/source/hive/ITHiveTezUnionAllTest.java | 108 ---------------------
.../kylin/source/hive/CreateFlatHiveTableStep.java | 2 +-
.../org/apache/kylin/source/hive/HiveMRInput.java | 4 +-
5 files changed, 7 insertions(+), 129 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index c33a7fb..fcfc598 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -142,22 +142,6 @@ public class HiveCmdBuilder {
statements.add(statement);
}
- public void addStatementWithRedistributeBy(String statement) {
- StringBuilder builder = new StringBuilder();
- builder.append(statement);
- addStatementWithRedistributeBy(builder);
- }
-
- public void addStatementWithRedistributeBy(StringBuilder statement) {
- /**
- * When hive.execution.engine is tez and table is a view of union-all struct, it generates
- * subdirectories in output, which causes file not found exception.
- * Use "DISTRIBUTE BY RAND()" to workaround this issue.
- */
- statement.append("DISTRIBUTE BY RAND()").append(";\n");
- statements.add(statement.toString());
- }
-
public void addStatements(String[] stats) {
for (String s : stats) {
statements.add(s);
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 7b8e413..9612c7e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -116,11 +116,13 @@ public class JoinedFlatTable {
}
}
- return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + "\n";
+ return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc)
+ + ";\n";
}
public static String generateInsertPartialDataStatement(IJoinedFlatTableDesc flatDesc) {
- return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + "\n";
+ return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc)
+ + ";\n";
}
public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc) {
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTezUnionAllTest.java b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTezUnionAllTest.java
deleted file mode 100644
index 742c322..0000000
--- a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTezUnionAllTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.HiveCmdBuilder;
-import org.apache.kylin.engine.mr.JobBuilderSupport;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ITHiveTezUnionAllTest extends HBaseMetadataTestCase {
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void doTests() throws Exception {
- testMaterializeView(true);
- testMaterializeView(false);
- }
-
- private void testMaterializeView(boolean isDistributeBy) throws Exception {
- KylinConfig config = getTestConfig();
-
- /**
- * For UT debug
- * config.setProperty("kylin.job.use-remote-cli", "true");
- */
-
- String viewName = "test_union_all_view";
- String tableName = "test_union_all_table";
-
- HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
- JobEngineConfig jobConf = new JobEngineConfig(config);
- String storagePath = JobBuilderSupport.getJobWorkingDir(jobConf, "it-test") + "/" + tableName;
-
- StringBuilder testCmd = new StringBuilder();
- testCmd.append("USE " + config.getHiveDatabaseForIntermediateTable() + ";").append("\n");
- testCmd.append("SET hive.execution.engine=tez;");
- testCmd.append("DROP VIEW IF EXISTS " + viewName + ";\n");
- testCmd.append(
- "CREATE VIEW " + viewName + " AS SELECT * FROM test_kylin_fact UNION ALL SELECT * FROM test_kylin_fact")
- .append(";\n");
- testCmd.append("DROP TABLE IF EXISTS " + tableName + ";\n");
- testCmd.append("CREATE TABLE IF NOT EXISTS " + tableName + "\n");
- testCmd.append("LOCATION '" + storagePath + "'\n");
- testCmd.append("AS SELECT * FROM " + viewName + "\n");
- if (isDistributeBy)
- hiveCmdBuilder.addStatementWithRedistributeBy(testCmd);
- else
- hiveCmdBuilder.addStatement(testCmd.toString());
-
- Path rootPath = new Path(storagePath);
- FileSystem fs = HadoopUtil.getFileSystem(storagePath);
-
- fs.delete(rootPath, true);
- fs.mkdirs(rootPath);
-
- config.getCliCommandExecutor().execute(hiveCmdBuilder.build());
-
- rootPath = fs.makeQualified(rootPath);
- for (FileStatus statsFolder : fs.listStatus(rootPath)) {
- if (isDistributeBy)
- Assert.assertTrue(!statsFolder.isDirectory());
- else
- Assert.assertTrue(statsFolder.isDirectory());
- }
-
- HiveCmdBuilder cleanupCmdBuilder = new HiveCmdBuilder();
- StringBuilder cleanupCmd = new StringBuilder();
- cleanupCmd.append("USE " + config.getHiveDatabaseForIntermediateTable() + ";").append("\n");
- cleanupCmd.append("DROP VIEW IF EXISTS " + viewName + ";\n");
- cleanupCmd.append("DROP TABLE IF EXISTS " + tableName + ";\n");
- cleanupCmdBuilder.addStatement(cleanupCmd.toString());
- config.getCliCommandExecutor().execute(cleanupCmdBuilder.build());
- fs.delete(rootPath, true);
- }
-}
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index 46bca96..21ef940 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -53,7 +53,7 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.overwriteHiveProps(config.getHiveConfigOverride());
hiveCmdBuilder.addStatement(getInitStatement());
- hiveCmdBuilder.addStatementWithRedistributeBy(getCreateTableStatement());
+ hiveCmdBuilder.addStatement(getCreateTableStatement());
final String cmd = hiveCmdBuilder.toString();
stepLogger.log("Create and distribute table, cmd: ");
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index f02d2ea..38656cf 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -246,8 +246,8 @@ public class HiveMRInput implements IMRInput {
createIntermediateTableHql
.append("ALTER TABLE " + intermediate + " SET TBLPROPERTIES('auto.purge'='true');\n");
createIntermediateTableHql
- .append("INSERT OVERWRITE TABLE " + intermediate + " SELECT * FROM " + identity + "\n");
- hiveCmdBuilder.addStatementWithRedistributeBy(createIntermediateTableHql);
+ .append("INSERT OVERWRITE TABLE " + intermediate + " SELECT * FROM " + identity + ";\n");
+ hiveCmdBuilder.addStatement(createIntermediateTableHql.toString());
hiveViewIntermediateTables = hiveViewIntermediateTables + intermediate + ";";
}
}
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.