You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/08/01 17:24:27 UTC
[2/6] impala git commit: IMPALA-1624: Allow toggling and unsetting
some command-line options inside impala-shell
IMPALA-1624: Allow toggling and unsetting some command-line options inside impala-shell
This change provides a way to modify command-line options like -B,
--output_file and --delimiter inside impala-shell without quitting
the shell and then restarting again.
Also fixed IMPALA-7286: command "unset" does not work for shell options
Testing:
Added tests for all new options in test_shell_interactive.py
Tested on Python 2.6 and Python 2.7
Change-Id: Id8d4487c24f24806223bfd5c54336914e3afd763
Reviewed-on: http://gerrit.cloudera.org:8080/10900
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/de4bdb0b
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/de4bdb0b
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/de4bdb0b
Branch: refs/heads/master
Commit: de4bdb0bbfd9a8ef17e020cc4904e3a66d2ac298
Parents: b951110
Author: nghia le <mi...@gmail.com>
Authored: Tue Jul 10 16:19:19 2018 +0200
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Jul 31 20:35:27 2018 +0000
----------------------------------------------------------------------
shell/impala_shell.py | 25 +++++++++++---
tests/shell/test_shell_interactive.py | 54 ++++++++++++++++++++++++++++++
2 files changed, 75 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/de4bdb0b/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index 95ff8ab..aab478d 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -140,10 +140,16 @@ class ImpalaShell(object, cmd.Cmd):
DML_REGEX = re.compile("^(insert|upsert|update|delete)$", re.I)
# Seperator for queries in the history file.
HISTORY_FILE_QUERY_DELIM = '_IMP_DELIM_'
+ # Strings that are interpreted as True for some shell options.
+ TRUE_STRINGS = ("true", "TRUE", "True", "1")
VALID_SHELL_OPTIONS = {
- 'LIVE_PROGRESS' : (lambda x: x in ("true", "TRUE", "True", "1"), "print_progress"),
- 'LIVE_SUMMARY' : (lambda x: x in ("true", "TRUE", "True", "1"), "print_summary")
+ 'LIVE_PROGRESS' : (lambda x: x in ImpalaShell.TRUE_STRINGS, "print_progress"),
+ 'LIVE_SUMMARY' : (lambda x: x in ImpalaShell.TRUE_STRINGS, "print_summary"),
+ 'WRITE_DELIMITED' : (lambda x: x in ImpalaShell.TRUE_STRINGS, "write_delimited"),
+ 'VERBOSE' : (lambda x: x in ImpalaShell.TRUE_STRINGS, "verbose"),
+ 'DELIMITER' : (lambda x: " " if x == '\\s' else x, "output_delimiter"),
+ 'OUTPUT_FILE' : (lambda x: None if x == '' else x, "output_file"),
}
# Minimum time in seconds between two calls to get the exec summary.
@@ -180,7 +186,8 @@ class ImpalaShell(object, cmd.Cmd):
# Output formatting flags/options
self.output_file = options.output_file
- self.output_delimiter = options.output_delimiter
+ self.output_delimiter = " " if options.output_delimiter == "\\s" \
+ else options.output_delimiter
self.write_delimited = options.write_delimited
self.print_header = options.print_header
@@ -655,6 +662,14 @@ class ImpalaShell(object, cmd.Cmd):
except KeyError:
return False
+ def _handle_unset_shell_options(self, token):
+ try:
+ handle = self.VALID_SHELL_OPTIONS[token]
+ self.__dict__[handle[1]] = impala_shell_defaults[handle[1]]
+ return True
+ except KeyError:
+ return False
+
def _get_var_name(self, name):
"""Look for a namespace:var_name pattern in an option name.
Return the variable name if it's a match or None otherwise.
@@ -737,6 +752,8 @@ class ImpalaShell(object, cmd.Cmd):
elif self.set_query_options.get(option):
print 'Unsetting option %s' % option
del self.set_query_options[option]
+ elif self._handle_unset_shell_options(option):
+ print 'Unsetting shell option %s' % option
else:
print "No option called %s is set" % option
@@ -1447,7 +1464,7 @@ LIVE_PROGRESS=1;'.",
to remove formatting from results you want to save for later, or to benchmark Impala.",
"You can run a single query from the command line using the '-q' option.",
"When pretty-printing is disabled, you can use the '--output_delimiter' flag to set \
-the delimiter for fields in the same row. The default is ','.",
+the delimiter for fields in the same row. The default is '\\t'.",
"Run the PROFILE command after a query has finished to see a comprehensive summary of \
all the performance and diagnostic information that Impala gathered for that query. Be \
warned, it can be very long!",
http://git-wip-us.apache.org/repos/asf/impala/blob/de4bdb0b/tests/shell/test_shell_interactive.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index bf4923d..1d81663 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -77,6 +77,60 @@ class TestImpalaShellInteractive(object):
self._expect_with_cmd(proc, "set", ("LIVE_PROGRESS: True", "LIVE_SUMMARY: False"))
self._expect_with_cmd(proc, "set live_summary=1")
self._expect_with_cmd(proc, "set", ("LIVE_PROGRESS: True", "LIVE_SUMMARY: True"))
+ self._expect_with_cmd(proc, "set", ("WRITE_DELIMITED: False", "VERBOSE: True"))
+ self._expect_with_cmd(proc, "set", ("DELIMITER: \\t", "OUTPUT_FILE: None"))
+ self._expect_with_cmd(proc, "set write_delimited=true")
+ self._expect_with_cmd(proc, "set", ("WRITE_DELIMITED: True", "VERBOSE: True"))
+ self._expect_with_cmd(proc, "set DELIMITER=,")
+ self._expect_with_cmd(proc, "set", ("DELIMITER: ,", "OUTPUT_FILE: None"))
+ self._expect_with_cmd(proc, "set output_file=/tmp/clmn.txt")
+ self._expect_with_cmd(proc, "set", ("DELIMITER: ,", "OUTPUT_FILE: /tmp/clmn.txt"))
+
+ @pytest.mark.execute_serially
+ def test_write_delimited(self):
+ """Test output rows in delimited mode"""
+ p = ImpalaShell()
+ p.send_cmd("use tpch")
+ p.send_cmd("set write_delimited=true")
+ p.send_cmd("select * from nation")
+ result = p.get_result()
+ assert "+----------------+" not in result.stdout
+ assert "21\tVIETNAM\t2" in result.stdout
+
+ @pytest.mark.execute_serially
+ def test_change_delimiter(self):
+ """Test change output delimiter if delimited mode is enabled"""
+ p = ImpalaShell()
+ p.send_cmd("use tpch")
+ p.send_cmd("set write_delimited=true")
+ p.send_cmd("set delimiter=,")
+ p.send_cmd("select * from nation")
+ result = p.get_result()
+ assert "21,VIETNAM,2" in result.stdout
+
+ @pytest.mark.execute_serially
+ def test_print_to_file(self):
+ """Test print to output file and unset"""
+ # test print to file
+ p1 = ImpalaShell()
+ p1.send_cmd("use tpch")
+ local_file = tempfile.NamedTemporaryFile(delete=True)
+ p1.send_cmd("set output_file=%s" % local_file.name)
+ p1.send_cmd("select * from nation")
+ result = p1.get_result()
+ assert "VIETNAM" not in result.stdout
+ with open(local_file.name, "r") as fi:
+ # check if the results were written to the file successfully
+ result = fi.read()
+ assert "VIETNAM" in result
+ # test unset to print back to stdout
+ p2 = ImpalaShell()
+ p2.send_cmd("use tpch")
+ p2.send_cmd("set output_file=%s" % local_file.name)
+ p2.send_cmd("unset output_file")
+ p2.send_cmd("select * from nation")
+ result = p2.get_result()
+ assert "VIETNAM" in result.stdout
@pytest.mark.execute_serially
def test_compute_stats_with_live_progress_options(self):