You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/05/12 22:09:49 UTC
[14/50] [abbrv] incubator-impala git commit: IMPALA-3397: Source
query files from shell.
IMPALA-3397: Source query files from shell.
This patch allows you to write SOURCE <file> or SRC <file>, and have the
shell read the file and execute all the queries in it.
Change-Id: Ib05df3e755cd12e9e9562de6b353857940eace03
Reviewed-on: http://gerrit.cloudera.org:8080/2663
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a805e100
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a805e100
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a805e100
Branch: refs/heads/master
Commit: a805e100b2c89c23c44f4f9eac6410b79375c561
Parents: e1c5959
Author: Henry Robinson <he...@cloudera.com>
Authored: Tue Mar 29 16:58:56 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Thu May 12 14:17:54 2016 -0700
----------------------------------------------------------------------
shell/impala_shell.py | 60 ++++++++++++++++++------------
tests/shell/shell.cmds | 3 ++
tests/shell/shell2.cmds | 1 +
tests/shell/shell_error.cmds | 6 +++
tests/shell/test_shell_interactive.py | 35 +++++++++++++++++
5 files changed, 82 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index 2e246e0..1f82186 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -163,6 +163,8 @@ class ImpalaShell(cmd.Cmd):
self.print_summary = options.print_summary
self.print_progress = options.print_progress
+ self.ignore_query_failure = options.ignore_query_failure
+
# Due to a readline bug in centos/rhel7, importing it causes control characters to be
# printed. This breaks any scripting against the shell in non-interactive mode. Since
# the non-interactive mode does not need readline - do not import it.
@@ -1021,6 +1023,20 @@ class ImpalaShell(cmd.Cmd):
"""Print a random tip"""
print_to_stderr(random.choice(TIPS))
+ def do_src(self, args):
+ return self.do_source(args)
+
+ def do_source(self, args):
+ try:
+ cmd_file = open(args, "r")
+ except Exception, e:
+ print_to_stderr("Error opening file '%s': %s" % (args, e))
+ return CmdStatus.ERROR
+ if self.execute_query_list(parse_query_text(cmd_file.read())):
+ return CmdStatus.SUCCESS
+ else:
+ return CmdStatus.ERROR
+
def preloop(self):
"""Load the history file if it exists"""
if self.readline:
@@ -1088,6 +1104,18 @@ class ImpalaShell(cmd.Cmd):
# If the user input is lower case or mixed case, return lower case commands.
return cmd_names
+ def execute_query_list(self, queries):
+ if not self.imp_client.connected:
+ print_to_stderr('Not connected to Impala, could not execute queries.')
+ return False
+ queries = [ self.sanitise_input(q) for q in self.cmdqueue + queries ]
+ for q in queries:
+ if self.onecmd(q) is CmdStatus.ERROR:
+ print_to_stderr('Could not execute command: %s' % q)
+ if not self.ignore_query_failure: return False
+ return True
+
+
TIPS=[
"Press TAB twice to see a list of available commands.",
"After running a query, type SUMMARY to see a summary of where time was spent.",
@@ -1159,7 +1187,6 @@ def parse_variables(keyvals):
def execute_queries_non_interactive_mode(options):
"""Run queries in non-interactive mode."""
- queries = []
if options.query_file:
try:
# "-" here signifies input from STDIN
@@ -1167,31 +1194,18 @@ def execute_queries_non_interactive_mode(options):
query_file_handle = sys.stdin
else:
query_file_handle = open(options.query_file, 'r')
-
- queries = parse_query_text(query_file_handle.read())
- if query_file_handle != sys.stdin:
- query_file_handle.close()
except Exception, e:
- print_to_stderr('Error: %s' % e)
- sys.exit(1)
+ print_to_stderr("Could not open file '%s': %s", options.query_file, e)
+
+ query_text = query_file_handle.read()
elif options.query:
- queries = parse_query_text(options.query)
- shell = ImpalaShell(options)
- # The impalad was specified on the command line and the connection failed.
- # Return with an error, no need to process the query.
- if options.impalad and shell.imp_client.connected == False:
+ query_text = options.query
+ else:
+ return
+
+ queries = parse_query_text(query_text)
+ if not ImpalaShell(options).execute_query_list(queries):
sys.exit(1)
- queries = shell.cmdqueue + queries
- # Deal with case.
- sanitized_queries = []
- for query in queries:
- sanitized_queries.append(shell.sanitise_input(query))
- for query in sanitized_queries:
- # check if an error was encountered
- if shell.onecmd(query) is CmdStatus.ERROR:
- print_to_stderr('Could not execute command: %s' % query)
- if not options.ignore_query_failure:
- sys.exit(1)
if __name__ == "__main__":
# pass defaults into option parser
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/tests/shell/shell.cmds
----------------------------------------------------------------------
diff --git a/tests/shell/shell.cmds b/tests/shell/shell.cmds
new file mode 100644
index 0000000..fd39bd8
--- /dev/null
+++ b/tests/shell/shell.cmds
@@ -0,0 +1,3 @@
+USE FUNCTIONAL;
+SHOW TABLES;
+SOURCE shell2.cmds;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/tests/shell/shell2.cmds
----------------------------------------------------------------------
diff --git a/tests/shell/shell2.cmds b/tests/shell/shell2.cmds
new file mode 100644
index 0000000..8ea9481
--- /dev/null
+++ b/tests/shell/shell2.cmds
@@ -0,0 +1 @@
+SELECT VERSION();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/tests/shell/shell_error.cmds
----------------------------------------------------------------------
diff --git a/tests/shell/shell_error.cmds b/tests/shell/shell_error.cmds
new file mode 100644
index 0000000..04dfecd
--- /dev/null
+++ b/tests/shell/shell_error.cmds
@@ -0,0 +1,6 @@
+USE UNKNOWN_DATABASE;
+NOT A SQL QUERY;
+USE FUNCTIONAL;
+SHOW TABLES;
+# Note missing semi-colon
+SHOW DATABASES
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a805e100/tests/shell/test_shell_interactive.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
old mode 100644
new mode 100755
index f43a97e..7b54d61
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -266,6 +266,41 @@ class TestImpalaShellInteractive(object):
result = run_impala_shell_interactive(cmds, shell_args=args)
assert_var_substitution(result)
+ @pytest.mark.execute_serially
+ def test_source_file(self):
+ cwd = os.getcwd()
+ try:
+ # Change working dir so that SOURCE command in shell.cmds can find shell2.cmds.
+ os.chdir("%s/tests/shell/" % os.environ['IMPALA_HOME'])
+ result = run_impala_shell_interactive("source shell.cmds;")
+ assert "Query: use FUNCTIONAL" in result.stderr
+ assert "Query: show TABLES" in result.stderr
+ assert "alltypes" in result.stdout
+
+ # This is from shell2.cmds, the result of sourcing a file from a sourced file.
+ assert "select VERSION()" in result.stderr
+ assert "version()" in result.stdout
+ finally:
+ os.chdir(cwd)
+
+ @pytest.mark.execute_serially
+ def test_source_file_with_errors(self):
+ full_path = "%s/tests/shell/shell_error.cmds" % os.environ['IMPALA_HOME']
+ result = run_impala_shell_interactive("source %s;" % full_path)
+ assert "Could not execute command: use UNKNOWN_DATABASE" in result.stderr
+ assert "Query: use FUNCTIONAL" not in result.stderr
+
+ result = run_impala_shell_interactive("source %s;" % full_path, '-c')
+ assert "Could not execute command: use UNKNOWN_DATABASE" in result.stderr
+ assert "Query: use FUNCTIONAL" in result.stderr
+ assert "Query: show TABLES" in result.stderr
+ assert "alltypes" in result.stdout
+
+ @pytest.mark.execute_serially
+ def test_source_missing_file(self):
+ full_path = "%s/tests/shell/doesntexist.cmds" % os.environ['IMPALA_HOME']
+ result = run_impala_shell_interactive("source %s;" % full_path)
+ assert "No such file or directory" in result.stderr
def run_impala_shell_interactive(input_lines, shell_args=''):
"""Runs a command in the Impala shell interactively."""