You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/01/26 14:56:11 UTC
[03/34] kylin git commit: KYLIN-3172 No such file or directory error
with CreateLookupHiveViewMaterializationStep
KYLIN-3172 No such file or directory error with CreateLookupHiveViewMaterializationStep
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/35979ff0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/35979ff0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/35979ff0
Branch: refs/heads/master
Commit: 35979ff0875d06cb4b1cc4b151efe6af37d1318d
Parents: 195fdef
Author: yiming.xu <10...@qq.com>
Authored: Wed Jan 17 18:37:08 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../kylin/common/util/HiveCmdBuilder.java | 45 ++++++---------
.../kylin/common/util/HiveCmdBuilderTest.java | 33 +++++++----
.../apache/kylin/source/hive/HiveMRInput.java | 36 ++++++------
.../kylin/source/hive/HiveMRInputTest.java | 59 ++++++++++++++++++++
4 files changed, 115 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index add53db..c33a7fb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -18,11 +18,7 @@
package org.apache.kylin.common.util;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
@@ -30,8 +26,6 @@ import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
@@ -42,9 +36,10 @@ import org.w3c.dom.NodeList;
import com.google.common.collect.Lists;
public class HiveCmdBuilder {
- private static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class);
+ public static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class);
public static final String HIVE_CONF_FILENAME = "kylin_hive_conf";
+ static final String CREATE_HQL_TMP_FILE_TEMPLATE = "cat >%s<<EOL\n%sEOL";
public enum HiveClientMode {
CLI, BEELINE
@@ -68,10 +63,11 @@ public class HiveCmdBuilder {
beelineShell = kylinConfig.getSparkSqlBeelineShell();
beelineParams = kylinConfig.getSparkSqlBeelineParams();
if (StringUtils.isBlank(beelineShell)) {
- throw new IllegalStateException("Missing config 'kylin.source.hive.sparksql-beeline-shell', please check kylin.properties");
+ throw new IllegalStateException(
+ "Missing config 'kylin.source.hive.sparksql-beeline-shell', please check kylin.properties");
}
}
-
+
StringBuffer buf = new StringBuffer();
switch (clientMode) {
@@ -84,37 +80,28 @@ public class HiveCmdBuilder {
buf.append(parseProps());
break;
case BEELINE:
- BufferedWriter bw = null;
- File tmpHql = null;
+ String tmpHqlPath = null;
+ StringBuilder hql = new StringBuilder();
try {
- tmpHql = File.createTempFile("beeline_", ".hql");
- bw = new BufferedWriter(new FileWriter(tmpHql));
+ tmpHqlPath = "/tmp/" + System.currentTimeMillis() + ".hql";
for (String statement : statements) {
- bw.write(statement);
- bw.newLine();
+ hql.append(statement);
+ hql.append("\n");
}
+ String createFileCmd = String.format(CREATE_HQL_TMP_FILE_TEMPLATE, tmpHqlPath, hql);
+ buf.append(createFileCmd);
+ buf.append("\n");
buf.append(beelineShell);
buf.append(" ");
buf.append(beelineParams);
buf.append(parseProps());
buf.append(" -f ");
- buf.append(tmpHql.getAbsolutePath());
+ buf.append(tmpHqlPath);
buf.append(";ret_code=$?;rm -f ");
- buf.append(tmpHql.getAbsolutePath());
+ buf.append(tmpHqlPath);
buf.append(";exit $ret_code");
-
- } catch (IOException e) {
- throw new RuntimeException(e);
} finally {
- IOUtils.closeQuietly(bw);
-
- if (tmpHql != null && logger.isDebugEnabled()) {
- String hql = null;
- try {
- hql = FileUtils.readFileToString(tmpHql, Charset.defaultCharset());
- } catch (IOException e) {
- // ignore
- }
+ if (tmpHqlPath != null && logger.isDebugEnabled()) {
logger.debug("The SQL to execute in beeline: \n" + hql);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index 4262722..ecc8961 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -45,7 +45,7 @@ public class HiveCmdBuilderTest {
System.clearProperty("kylin.source.hive.client");
System.clearProperty("kylin.source.hive.beeline-shell");
System.clearProperty("kylin.source.hive.beeline-params");
-
+
System.clearProperty("kylin.source.hive.enable-sparksql-for-table-ops");
System.clearProperty("kylin.source.hive.sparksql-beeline-shell");
System.clearProperty("kylin.source.hive.sparksql-beeline-params");
@@ -65,12 +65,15 @@ public class HiveCmdBuilderTest {
hiveCmdBuilder.addStatement("SHOW\n TABLES;");
hiveCmdBuilder.setHiveConfProps(hiveProps);
hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite);
- assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
+ assertEquals(
+ "hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez",
+ hiveCmdBuilder.build());
}
@Test
public void testBeeline() throws IOException {
- String lineSeparator = java.security.AccessController.doPrivileged(new sun.security.action.GetPropertyAction("line.separator"));
+ String lineSeparator = java.security.AccessController
+ .doPrivileged(new sun.security.action.GetPropertyAction("line.separator"));
System.setProperty("kylin.source.hive.client", "beeline");
System.setProperty("kylin.source.hive.beeline-shell", "/spark-client/bin/beeline");
System.setProperty("kylin.source.hive.beeline-params", "-u jdbc_url");
@@ -78,17 +81,19 @@ public class HiveCmdBuilderTest {
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
- hiveCmdBuilder.addStatement("SHOW\n TABLES;");
+ hiveCmdBuilder.addStatement("SHOW TABLES;");
String cmd = hiveCmdBuilder.build();
- assertTrue(cmd.startsWith("/spark-client/bin/beeline -u jdbc_url"));
-
String hqlFile = cmd.substring(cmd.lastIndexOf("-f ") + 3).trim();
hqlFile = hqlFile.substring(0, hqlFile.length() - ";exit $ret_code".length());
-
+ String createFileCmd = cmd.substring(0, cmd.indexOf("EOL\n", cmd.indexOf("EOL\n") + 1) + 3);
+ CliCommandExecutor cliCommandExecutor = new CliCommandExecutor();
+ Pair<Integer, String> execute = cliCommandExecutor.execute(createFileCmd);
String hqlStatement = FileUtils.readFileToString(new File(hqlFile), Charset.defaultCharset());
- assertEquals("USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW\n TABLES;" + lineSeparator, hqlStatement);
-
+ assertEquals(
+ "USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW TABLES;" + lineSeparator,
+ hqlStatement);
+ assertBeelineCmd(cmd);
FileUtils.forceDelete(new File(hqlFile));
}
@@ -101,9 +106,13 @@ public class HiveCmdBuilderTest {
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
- hiveCmdBuilder.addStatement("SHOW\n TABLES;");
-
+ hiveCmdBuilder.addStatement("SHOW TABLES;");
String cmd = hiveCmdBuilder.build();
- assertTrue(cmd.startsWith("/spark-client/bin/beeline -u jdbc_url"));
+ assertBeelineCmd(cmd);
+ }
+
+ private void assertBeelineCmd(String cmd) {
+ String beelineCmd = cmd.substring(cmd.indexOf("EOL\n", cmd.indexOf("EOL\n") + 1) + 4);
+ assertTrue(beelineCmd.startsWith("/spark-client/bin/beeline -u jdbc_url"));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 6f42961..31a99cd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -177,7 +177,25 @@ public class HiveMRInput implements IMRInput {
}
protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
- return JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
+
+ String jobWorkingDir = JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
+ return jobWorkingDir;
+ }
+
+ private void checkAndCreateWorkDir(String jobWorkingDir) {
+ try {
+ Path path = new Path(jobWorkingDir);
+ FileSystem fileSystem = HadoopUtil.getFileSystem(path);
+ if (!fileSystem.exists(path)) {
+ logger.info("Create jobWorkDir : " + jobWorkingDir);
+ fileSystem.mkdirs(path);
+ }
+ } catch (IOException e) {
+ logger.error("Could not create lookUp table dir : " + jobWorkingDir);
+ }
}
private AbstractExecutable createRedistributeFlatHiveTableStep(String hiveInitStatements, String cubeName) {
@@ -210,9 +228,6 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
- // Create work dir to avoid hive create it,
- // the difference is that the owners are different.
- checkAndCreateWorkDir(jobWorkingDir);
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
@@ -242,19 +257,6 @@ public class HiveMRInput implements IMRInput {
return step;
}
- private void checkAndCreateWorkDir(String jobWorkingDir) {
- try {
- Path path = new Path(jobWorkingDir);
- FileSystem fileSystem = HadoopUtil.getFileSystem(path);
- if (!fileSystem.exists(path)) {
- logger.info("Create jobWorkDir : " + jobWorkingDir);
- fileSystem.mkdirs(path);
- }
- } catch (IOException e) {
- logger.error("Could not create lookUp table dir : " + jobWorkingDir);
- }
- }
-
private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
String cubeName) {
//from hive to hive
http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
new file mode 100644
index 0000000..6d0737d
--- /dev/null
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class HiveMRInputTest {
+
+ @Test
+ public void TestGetJobWorkingDir() throws IOException {
+ FileSystem fileSystem = FileSystem.get(new Configuration());
+ Path jobWorkDirPath = null;
+ try {
+ KylinConfig kylinConfig = mock(KylinConfig.class);
+ KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+ when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
+
+ DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class);
+ defaultChainedExecutable.setId(UUID.randomUUID().toString());
+
+ HiveMRInput.BatchCubingInputSide batchCubingInputSide = new HiveMRInput.BatchCubingInputSide(null);
+ String jobWorkingDir = batchCubingInputSide.getJobWorkingDir(defaultChainedExecutable);
+ jobWorkDirPath = new Path(jobWorkingDir);
+ Assert.assertTrue(fileSystem.exists(jobWorkDirPath));
+ } finally {
+ if (jobWorkDirPath != null)
+ fileSystem.deleteOnExit(jobWorkDirPath);
+ }
+ }
+
+}
\ No newline at end of file