You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ab...@apache.org on 2019/05/03 08:21:30 UTC
[hive] branch master updated: HIVE-21661: Add option to check q
file syntax before execution (Bruno Pusztahazi via Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 7ae14d1 HIVE-21661: Add option to check q file syntax before execution (Bruno Pusztahazi via Laszlo Bodor)
7ae14d1 is described below
commit 7ae14d1090f22f656c3955aab59396aeeae232ae
Author: Bruno Pusztahazi <bp...@hortonworks.com>
AuthorDate: Fri May 3 10:01:56 2019 +0200
HIVE-21661: Add option to check q file syntax before execution (Bruno Pusztahazi via Laszlo Bodor)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
.../org/apache/hadoop/hive/ql/QTestSyntaxUtil.java | 185 +++++++++++++++++++++
.../java/org/apache/hadoop/hive/ql/QTestUtil.java | 8 +-
2 files changed, 190 insertions(+), 3 deletions(-)
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestSyntaxUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestSyntaxUtil.java
new file mode 100644
index 0000000..3630da2
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestSyntaxUtil.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql;
+
+import static org.junit.Assert.assertTrue;
+
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.ParseDriver;
+import org.apache.hadoop.hive.ql.processors.AddResourceProcessor;
+import org.apache.hadoop.hive.ql.processors.CommandProcessor;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
+import org.apache.hadoop.hive.ql.processors.CompileProcessor;
+import org.apache.hadoop.hive.ql.processors.CryptoProcessor;
+import org.apache.hadoop.hive.ql.processors.DfsProcessor;
+import org.apache.hadoop.hive.ql.processors.ListResourceProcessor;
+import org.apache.hadoop.hive.ql.processors.LlapCacheResourceProcessor;
+import org.apache.hadoop.hive.ql.processors.LlapClusterResourceProcessor;
+import org.apache.hadoop.hive.ql.processors.ReloadProcessor;
+import org.apache.hadoop.hive.ql.processors.ResetProcessor;
+import org.apache.hadoop.hive.ql.processors.SetProcessor;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.common.util.HiveStringUtils;
+
+/**
+ * Q File Syntax Checker Utility.
+ *
+ */
+public class QTestSyntaxUtil {
+
+ private QTestUtil qUtil;
+ private HiveConf conf;
+ private ParseDriver pd;
+
+ public QTestSyntaxUtil(QTestUtil qTestUtil, HiveConf conf, ParseDriver pd) {
+ qUtil = qTestUtil;
+ this.conf = conf;
+ this.pd = pd;
+ }
+
+ public void checkQFileSyntax(List<String> cmds) {
+ String command = "";
+ if (shouldCheckSyntax()) {
+ //check syntax first
+ for (String oneCmd : cmds) {
+ if (StringUtils.endsWith(oneCmd, "\\")) {
+ command += StringUtils.chop(oneCmd) + "\\;";
+ continue;
+ } else {
+ if (qUtil.isHiveCommand(oneCmd)) {
+ command = oneCmd;
+ } else {
+ command += oneCmd;
+ }
+ }
+ if (StringUtils.isBlank(command)) {
+ continue;
+ }
+ assertTrue("Syntax error in command: " + command, checkSyntax(command));
+ command = "";
+ }
+ }
+ }
+
+ private boolean shouldCheckSyntax() {
+ return "true".equalsIgnoreCase(System.getProperty("test.check.syntax"));
+ }
+
+ private boolean checkSyntax(String cmd) {
+ ASTNode tree;
+ int ret = 0;
+ CliSessionState ss = (CliSessionState) SessionState.get();
+
+ String cmdTrimmed = HiveStringUtils.removeComments(cmd).trim();
+ String[] tokens = cmdTrimmed.split("\\s+");
+ if (tokens[0].equalsIgnoreCase("source")) {
+ return true;
+ }
+ if (cmdTrimmed.toLowerCase().equals("quit") || cmdTrimmed.toLowerCase().equals("exit")) {
+ return true;
+ }
+ if (cmdTrimmed.startsWith("!")) {
+ return true;
+ }
+ try {
+ CommandProcessor proc = CommandProcessorFactory.get(tokens, (HiveConf) conf);
+ if (proc instanceof IDriver) {
+ try {
+ Context ctx = new Context(conf);
+ HiveTxnManager queryTxnMgr = SessionState.get().initTxnMgr(conf);
+ ctx.setHiveTxnManager(queryTxnMgr);
+ ctx.setCmd(cmd);
+ ctx.setHDFSCleanup(true);
+ tree = pd.parse(cmd, ctx);
+ qUtil.analyzeAST(tree);
+ } catch (Exception e) {
+ return false;
+ }
+ } else {
+ ret = processLocalCmd(cmdTrimmed, proc, ss);
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ return false;
+ }
+ if (ret != 0) {
+ return false;
+ }
+ return true;
+ }
+
+ private static int processLocalCmd(String cmd, CommandProcessor proc, CliSessionState ss) {
+ int ret = 0;
+ if (proc != null) {
+ String firstToken = cmd.trim().split("\\s+")[0];
+ String cmd1 = cmd.trim().substring(firstToken.length()).trim();
+ //CommandProcessorResponse res = proc.run(cmd_1);
+ if (proc instanceof ResetProcessor ||
+ proc instanceof CompileProcessor ||
+ proc instanceof ReloadProcessor ||
+ proc instanceof CryptoProcessor ||
+ proc instanceof AddResourceProcessor ||
+ proc instanceof ListResourceProcessor ||
+ proc instanceof LlapClusterResourceProcessor ||
+ proc instanceof LlapCacheResourceProcessor) {
+ if (cmd1.trim().split("\\s+").length < 1) {
+ ret = -1;
+ }
+ }
+ if (proc instanceof SetProcessor) {
+ if (!cmd1.contains("=")) {
+ ret = -1;
+ }
+ }
+ if (proc instanceof DfsProcessor) {
+ String[] argv = cmd1.trim().split("\\s+");
+ if ("-put".equals(firstToken) || "-test".equals(firstToken) ||
+ "-copyFromLocal".equals(firstToken) || "-moveFromLocal".equals(firstToken)) {
+ if (argv.length < 3) {
+ ret = -1;
+ }
+ } else if ("-get".equals(firstToken) ||
+ "-copyToLocal".equals(firstToken) || "-moveToLocal".equals(firstToken)) {
+ if (argv.length < 3) {
+ ret = -1;
+ }
+ } else if ("-mv".equals(firstToken) || "-cp".equals(firstToken)) {
+ if (argv.length < 3) {
+ ret = -1;
+ }
+ } else if ("-rm".equals(firstToken) || "-rmr".equals(firstToken) ||
+ "-cat".equals(firstToken) || "-mkdir".equals(firstToken) ||
+ "-touchz".equals(firstToken) || "-stat".equals(firstToken) ||
+ "-text".equals(firstToken)) {
+ if (argv.length < 2) {
+ ret = -1;
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
+}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 7538d18..5da654a 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
-
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
@@ -1261,6 +1260,9 @@ public class QTestUtil {
int rc = 0;
String command = "";
+ QTestSyntaxUtil qtsu = new QTestSyntaxUtil(this, conf, pd);
+ qtsu.checkQFileSyntax(cmds);
+
for (String oneCmd : cmds) {
if (StringUtils.endsWith(oneCmd, "\\")) {
command += StringUtils.chop(oneCmd) + "\\;";
@@ -1293,7 +1295,7 @@ public class QTestUtil {
return rc;
}
- /**
+/**
* This allows a .q file to continue executing after a statement runs into an error which is convenient
* if you want to use another hive cmd after the failure to sanity check the state of the system.
*/
@@ -1301,7 +1303,7 @@ public class QTestUtil {
return conf.getBoolVar(HiveConf.ConfVars.CLIIGNOREERRORS);
}
- private boolean isHiveCommand(String command) {
+ boolean isHiveCommand(String command) {
String[] cmd = command.trim().split("\\s+");
if (HiveCommand.find(cmd) != null) {
return true;