[2/3] impala git commit: IMPALA-2751: Matching quotes are not requirerd in comments

[tm...@apache.org]

IMPALA-2751: Matching quotes are not requirerd in comments

This patch fixes the issue where non-matching quotes inside comments
will cause the shell to not terminate.

The fix is to strip any SQL comments before sending to shlex since shlex
does not understand SQL comments and will raise an exception when it sees
unmatched quotes regardless whether the quotes are in the comments or
not.

Testing:
- Added new shell tests
- Ran all end-to-end shell tests

Change-Id: Ic899fdddc182947f73101ddbc2e3c8caf97d9085
Reviewed-on: http://gerrit.cloudera.org:8080/10474
Reviewed-by: Michael Brown <mi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/433ae34f
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/433ae34f
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/433ae34f

Branch: refs/heads/2.x
Commit: 433ae34f3483246aa043b2b963079d4a5670fb61
Parents: f76cf25
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Mon May 21 20:48:39 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed May 30 01:16:18 2018 +0000

----------------------------------------------------------------------
 shell/impala_shell.py                 | 12 +++++++-----
 tests/shell/test_shell_interactive.py | 18 ++++++++++++++++++
 2 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/impala/blob/433ae34f/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index a2dd89f..f7f572b 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -398,7 +398,8 @@ class ImpalaShell(object, cmd.Cmd):
       try:
         # Look for an open quotation in the entire command, and not just the
         # current line.
-        if self.partial_cmd: line = '%s %s' % (self.partial_cmd, line)
+        if self.partial_cmd:
+          line = sqlparse.format('%s %s' % (self.partial_cmd, line), strip_comments=True)
         self._shlex_split(line)
         return True
       # If the command ends with a delimiter, check if it has an open quotation.
@@ -416,8 +417,8 @@ class ImpalaShell(object, cmd.Cmd):
     # Iterate through the line and switch the state if a single or double quote is found
     # and ignore escaped single and double quotes if the line is considered open (meaning
     # a previous single or double quote has not been closed yet)
-      state_closed = True;
-      opener = None;
+      state_closed = True
+      opener = None
       for i, char in enumerate(line):
         if state_closed and (char in ['\'', '\"']):
           state_closed = False
@@ -425,7 +426,7 @@ class ImpalaShell(object, cmd.Cmd):
         elif not state_closed and opener == char:
           if line[i - 1] != '\\':
             state_closed = True
-            opener = None;
+            opener = None
 
       return state_closed
 
@@ -1129,7 +1130,8 @@ class ImpalaShell(object, cmd.Cmd):
     query = self._create_beeswax_query(args)
     # Set posix=True and add "'" to escaped quotes
     # to deal with escaped quotes in string literals
-    lexer = shlex.shlex(query.query.lstrip(), posix=True)
+    lexer = shlex.shlex(sqlparse.format(query.query.lstrip(), strip_comments=True),
+                        posix=True)
     lexer.escapedquotes += "'"
     # Because the WITH clause may precede DML or SELECT queries,
     # just checking the first token is insufficient.

http://git-wip-us.apache.org/repos/asf/impala/blob/433ae34f/tests/shell/test_shell_interactive.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index e112e3e..fe41e36 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -512,6 +512,24 @@ class TestImpalaShellInteractive(object):
     assert '| 1"23"4 |' in result.stdout
 
   @pytest.mark.execute_serially
+  def test_comment_with_quotes(self):
+    # IMPALA-2751: Comment does not need to have matching quotes
+    queries = [
+      "select -- '\n1;",
+      'select -- "\n1;',
+      "select -- \"'\n 1;",
+      "select /*'\n*/ 1;",
+      'select /*"\n*/ 1;',
+      "select /*\"'\n*/ 1;",
+      "with a as (\nselect 1\n-- '\n) select * from a",
+      'with a as (\nselect 1\n-- "\n) select * from a',
+      "with a as (\nselect 1\n-- '\"\n) select * from a",
+    ]
+    for query in queries:
+      result = run_impala_shell_interactive(query)
+      assert '| 1 |' in result.stdout
+
+  @pytest.mark.execute_serially
   def test_shell_prompt(self):
     proc = pexpect.spawn(SHELL_CMD)
     proc.expect(":21000] default>")