You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by tm...@apache.org on 2018/05/30 18:09:23 UTC
[1/3] impala git commit: IMPALA-6953: part 1: clean up DiskIoMgr
Repository: impala
Updated Branches:
refs/heads/2.x 564687265 -> b1ac8b95a
IMPALA-6953: part 1: clean up DiskIoMgr
There should be no behavioural changes as a result of
this refactoring.
Make DiskQueue an encapsulated class.
Remove friend classes where possible, either by using public
methods or moving code between classes.
Move method into protected in some cases.
Split GetNextRequestRange() into two methods that
operate on DiskQueue and RequestContext state. The methods
belong to the respective classes.
Reduce transitive #include dependencies to hopefully help
with build time.
Testing:
Ran core tests.
Change-Id: I5a6e393f8c01d10143cbac91108af37f6498c1b1
Reviewed-on: http://gerrit.cloudera.org:8080/10245
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-on: http://gerrit.cloudera.org:8080/10532
Reviewed-by: Thomas Marshall <th...@cmu.edu>
Tested-by: Thomas Marshall <th...@cmu.edu>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/f76cf257
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/f76cf257
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/f76cf257
Branch: refs/heads/2.x
Commit: f76cf25725238f601544b8d660a5aa32a8e5f7cf
Parents: 5646872
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Sat Apr 28 11:27:53 2018 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed May 30 01:11:09 2018 +0000
----------------------------------------------------------------------
----------------------------------------------------------------------
[2/3] impala git commit: IMPALA-2751: Matching quotes are not
requirerd in comments
Posted by tm...@apache.org.
IMPALA-2751: Matching quotes are not requirerd in comments
This patch fixes the issue where non-matching quotes inside comments
will cause the shell to not terminate.
The fix is to strip any SQL comments before sending to shlex since shlex
does not understand SQL comments and will raise an exception when it sees
unmatched quotes regardless whether the quotes are in the comments or
not.
Testing:
- Added new shell tests
- Ran all end-to-end shell tests
Change-Id: Ic899fdddc182947f73101ddbc2e3c8caf97d9085
Reviewed-on: http://gerrit.cloudera.org:8080/10474
Reviewed-by: Michael Brown <mi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/433ae34f
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/433ae34f
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/433ae34f
Branch: refs/heads/2.x
Commit: 433ae34f3483246aa043b2b963079d4a5670fb61
Parents: f76cf25
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Mon May 21 20:48:39 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed May 30 01:16:18 2018 +0000
----------------------------------------------------------------------
shell/impala_shell.py | 12 +++++++-----
tests/shell/test_shell_interactive.py | 18 ++++++++++++++++++
2 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/433ae34f/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index a2dd89f..f7f572b 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -398,7 +398,8 @@ class ImpalaShell(object, cmd.Cmd):
try:
# Look for an open quotation in the entire command, and not just the
# current line.
- if self.partial_cmd: line = '%s %s' % (self.partial_cmd, line)
+ if self.partial_cmd:
+ line = sqlparse.format('%s %s' % (self.partial_cmd, line), strip_comments=True)
self._shlex_split(line)
return True
# If the command ends with a delimiter, check if it has an open quotation.
@@ -416,8 +417,8 @@ class ImpalaShell(object, cmd.Cmd):
# Iterate through the line and switch the state if a single or double quote is found
# and ignore escaped single and double quotes if the line is considered open (meaning
# a previous single or double quote has not been closed yet)
- state_closed = True;
- opener = None;
+ state_closed = True
+ opener = None
for i, char in enumerate(line):
if state_closed and (char in ['\'', '\"']):
state_closed = False
@@ -425,7 +426,7 @@ class ImpalaShell(object, cmd.Cmd):
elif not state_closed and opener == char:
if line[i - 1] != '\\':
state_closed = True
- opener = None;
+ opener = None
return state_closed
@@ -1129,7 +1130,8 @@ class ImpalaShell(object, cmd.Cmd):
query = self._create_beeswax_query(args)
# Set posix=True and add "'" to escaped quotes
# to deal with escaped quotes in string literals
- lexer = shlex.shlex(query.query.lstrip(), posix=True)
+ lexer = shlex.shlex(sqlparse.format(query.query.lstrip(), strip_comments=True),
+ posix=True)
lexer.escapedquotes += "'"
# Because the WITH clause may precede DML or SELECT queries,
# just checking the first token is insufficient.
http://git-wip-us.apache.org/repos/asf/impala/blob/433ae34f/tests/shell/test_shell_interactive.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index e112e3e..fe41e36 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -512,6 +512,24 @@ class TestImpalaShellInteractive(object):
assert '| 1"23"4 |' in result.stdout
@pytest.mark.execute_serially
+ def test_comment_with_quotes(self):
+ # IMPALA-2751: Comment does not need to have matching quotes
+ queries = [
+ "select -- '\n1;",
+ 'select -- "\n1;',
+ "select -- \"'\n 1;",
+ "select /*'\n*/ 1;",
+ 'select /*"\n*/ 1;',
+ "select /*\"'\n*/ 1;",
+ "with a as (\nselect 1\n-- '\n) select * from a",
+ 'with a as (\nselect 1\n-- "\n) select * from a',
+ "with a as (\nselect 1\n-- '\"\n) select * from a",
+ ]
+ for query in queries:
+ result = run_impala_shell_interactive(query)
+ assert '| 1 |' in result.stdout
+
+ @pytest.mark.execute_serially
def test_shell_prompt(self):
proc = pexpect.spawn(SHELL_CMD)
proc.expect(":21000] default>")
[3/3] impala git commit: IMPALA-7068: fix
test_unsupported_text_compression table location
Posted by tm...@apache.org.
IMPALA-7068: fix test_unsupported_text_compression table location
WAREHOUSE is defined as get_fs_path('/test-warehouse') so we
shouldn't run that through get_fs_path again. Instead, I copied
the TBL_LOCATION format from line 59.
Change-Id: I73f21866d5c2bd7d3f4a5110d4d088a718b56960
Reviewed-on: http://gerrit.cloudera.org:8080/10504
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/b1ac8b95
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/b1ac8b95
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/b1ac8b95
Branch: refs/heads/2.x
Commit: b1ac8b95a4590021e4843e6cafa01dac4d660ec6
Parents: 433ae34
Author: Dan Hecht <dh...@cloudera.com>
Authored: Thu May 24 13:21:39 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed May 30 01:16:18 2018 +0000
----------------------------------------------------------------------
tests/metadata/test_partition_metadata.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/b1ac8b95/tests/metadata/test_partition_metadata.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_partition_metadata.py b/tests/metadata/test_partition_metadata.py
index 99eff1e..e3cc2c1 100644
--- a/tests/metadata/test_partition_metadata.py
+++ b/tests/metadata/test_partition_metadata.py
@@ -165,8 +165,7 @@ class TestPartitionMetadataUncompressedTextOnly(ImpalaTestSuite):
Should be able to query partitions with supported codecs."""
TBL_NAME = "multi_text_compression"
FQ_TBL_NAME = unique_database + "." + TBL_NAME
- TBL_LOCATION = get_fs_path(
- '{0}/{1}.db/{2}'.format(WAREHOUSE, unique_database, TBL_NAME))
+ TBL_LOCATION = '%s/%s.db/%s' % (WAREHOUSE, unique_database, TBL_NAME)
file_format = vector.get_value('table_format').file_format
# Clean up any existing data in the table directory.