You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/05/12 22:10:13 UTC
[38/50] [abbrv] incubator-impala git commit: IMPALA-3491: Use
unique_database fixture in test_recover_partitions.py.
IMPALA-3491: Use unique_database fixture in test_recover_partitions.py.
Testing: I ran the test 10 times in a loop locally and ran
a private core/hdfs run.
Change-Id: I5be5fa5d20bc6ed5b7830e0ce90201431d6aa008
Reviewed-on: http://gerrit.cloudera.org:8080/3003
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c0ee93bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c0ee93bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c0ee93bb
Branch: refs/heads/master
Commit: c0ee93bbbe250507b524a3404e8092c5c83b0225
Parents: 2781581
Author: Alex Behm <al...@cloudera.com>
Authored: Fri May 6 22:39:05 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Thu May 12 14:18:00 2016 -0700
----------------------------------------------------------------------
tests/metadata/test_recover_partitions.py | 271 +++++++++++++------------
1 file changed, 136 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c0ee93bb/tests/metadata/test_recover_partitions.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_recover_partitions.py b/tests/metadata/test_recover_partitions.py
index a219433..4997756 100644
--- a/tests/metadata/test_recover_partitions.py
+++ b/tests/metadata/test_recover_partitions.py
@@ -22,11 +22,6 @@ from tests.util.filesystem_utils import WAREHOUSE, IS_DEFAULT_FS
# Validates ALTER TABLE RECOVER PARTITIONS statement
class TestRecoverPartitions(ImpalaTestSuite):
- TEST_DB = "recover_parts_db"
- TEST_TBL = "alter_recover_partitions"
- TEST_TBL2 = "alter_recover_partitions_all_types"
- BASE_DIR = 'test-warehouse/%s.db/%s/' % (TEST_DB, TEST_TBL)
- BASE_DIR2 = 'test-warehouse/%s.db/%s/' % (TEST_DB, TEST_TBL2)
DEF_NULL_PART_KEY = "__HIVE_DEFAULT_PARTITION__"
@classmethod
@@ -38,7 +33,7 @@ class TestRecoverPartitions(ImpalaTestSuite):
super(TestRecoverPartitions, cls).add_test_dimensions()
sync_ddl_opts = [0, 1]
if cls.exploration_strategy() != 'exhaustive':
- # Only run with sync_ddl on exhaustive since it increases test runtime.
+ # Only run without sync_ddl on exhaustive since it increases test runtime.
sync_ddl_opts = [0]
cls.TestMatrix.add_dimension(create_exec_option_dimension(
@@ -50,213 +45,218 @@ class TestRecoverPartitions(ImpalaTestSuite):
# There is no reason to run these tests using all dimensions.
cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
- def setup_method(self, method):
- self.cleanup_db(self.TEST_DB)
- self.client.execute("create database {0} location '{1}/{0}.db'".format(self.TEST_DB,
- WAREHOUSE))
- self.client.execute("use %s" % self.TEST_DB)
-
- def teardown_method(self, method):
- self.cleanup_db(self.TEST_DB)
+ def __get_fs_location(self, db_name, table_name):
+ return 'test-warehouse/%s.db/%s/' % (db_name, table_name)
@SkipIfLocal.hdfs_client
- @pytest.mark.execute_serially
- def test_recover_partitions(self, vector):
+ def test_recover_partitions(self, vector, unique_database):
"""Test that RECOVER PARTITIONS correctly discovers new partitions added externally
by the hdfs client.
"""
- part_name = "p2"
- leaf_dir = "i=0001/p=%s/" % part_name
- malformed_dir = "i=fish/p=%s/" % part_name
- file_path = "test"
- inserted_value = "2"
- null_dir = "i=1/p=%s/" % self.DEF_NULL_PART_KEY
- null_inserted_value = "4"
+ TBL_NAME = "test_recover_partitions"
+ FQ_TBL_NAME = unique_database + "." + TBL_NAME
+ TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME)
+ PART_NAME = "p2"
+ LEAF_DIR = "i=0001/p=%s/" % PART_NAME
+ MALFORMED_DIR = "i=fish/p=%s/" % PART_NAME
+ FILE_PATH = "test"
+ INSERTED_VALUE = "2"
+ NULL_DIR = "i=1/p=%s/" % self.DEF_NULL_PART_KEY
+ NULL_INSERTED_VALUE = "4"
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % (self.TEST_TBL))
+ "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % (FQ_TBL_NAME))
self.execute_query_expect_success(self.client,
- "INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % (self.TEST_TBL))
+ "INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % (FQ_TBL_NAME))
# Create a path for a new partition using hdfs client and add a file with some values.
# Test that the partition can be recovered and that the inserted data are accessible.
- self.filesystem_client.make_dir(self.BASE_DIR + leaf_dir)
- self.filesystem_client.create_file(self.BASE_DIR + leaf_dir + file_path,
- inserted_value)
+ self.filesystem_client.make_dir(TBL_LOCATION + LEAF_DIR)
+ self.filesystem_client.create_file(TBL_LOCATION + LEAF_DIR + FILE_PATH,
+ INSERTED_VALUE)
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL))
- assert self.has_value(part_name, result.data) == False
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
+ assert not self.has_value(PART_NAME, result.data)
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s RECOVER PARTITIONS" % (self.TEST_TBL))
+ "ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL))
- assert (self.has_value(part_name, result.data) == True,
- "ALTER TABLE %s RECOVER PARTITIONS failed." % (self.TEST_TBL))
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
+ assert (self.has_value(PART_NAME, result.data),
+ "ALTER TABLE %s RECOVER PARTITIONS failed." % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "select c from %s" % self.TEST_TBL)
- assert (self.has_value(inserted_value, result.data) == True,
+ "select c from %s" % FQ_TBL_NAME)
+ assert (self.has_value(INSERTED_VALUE, result.data),
"Failed to load tables after ALTER TABLE %s RECOVER PARTITIONS."
- % (self.TEST_TBL))
+ % FQ_TBL_NAME)
# Test that invalid partition values are ignored during partition recovery.
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL))
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
old_length = len(result.data)
- self.filesystem_client.make_dir(self.BASE_DIR + malformed_dir)
+ self.filesystem_client.make_dir(TBL_LOCATION + MALFORMED_DIR)
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s RECOVER PARTITIONS" % (self.TEST_TBL))
+ "ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL))
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
assert (len(result.data) == old_length,
"ALTER TABLE %s RECOVER PARTITIONS failed to handle invalid partition values."
- % (self.TEST_TBL))
+ % FQ_TBL_NAME)
# Create a directory whose subdirectory names contain __HIVE_DEFAULT_PARTITION__
# and check that is recovered as a NULL partition.
- self.filesystem_client.make_dir(self.BASE_DIR + null_dir)
+ self.filesystem_client.make_dir(TBL_LOCATION + NULL_DIR)
self.filesystem_client.create_file(
- self.BASE_DIR + null_dir + file_path, null_inserted_value)
+ TBL_LOCATION + NULL_DIR + FILE_PATH, NULL_INSERTED_VALUE)
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL))
- assert self.has_value(self.DEF_NULL_PART_KEY, result.data) == False
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
+ assert not self.has_value(self.DEF_NULL_PART_KEY, result.data)
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s RECOVER PARTITIONS" % (self.TEST_TBL))
+ "ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL))
- assert (self.has_value("NULL", result.data) == True,
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
+ assert (self.has_value("NULL", result.data),
"ALTER TABLE %s RECOVER PARTITIONS failed to handle null partition values."
- % (self.TEST_TBL))
+ % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "select c from %s" % self.TEST_TBL)
- assert self.has_value(null_inserted_value, result.data) == True
+ "select c from %s" % FQ_TBL_NAME)
+ assert self.has_value(NULL_INSERTED_VALUE, result.data)
@SkipIfLocal.hdfs_client
- @pytest.mark.execute_serially
- def test_nondefault_location_partitions(self, vector):
+ def test_nondefault_location_partitions(self, vector, unique_database):
"""If the location of data files in one partition is changed, test that data files
in the default location will not be loaded after partition recovery."""
- file_path = "test"
- leaf_dir = "i=1/p=p3/"
- inserted_value = "4"
+ TBL_NAME = "test_recover_partitions"
+ FQ_TBL_NAME = unique_database + "." + TBL_NAME
+ TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME)
+ FILE_PATH = "test"
+ LEAF_DIR = "i=1/p=p3/"
+ INSERTED_VALUE = "4"
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % (self.TEST_TBL))
+ "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
- "INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % (self.TEST_TBL))
+ "INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s ADD PARTITION(i=1, p='p3')" % (self.TEST_TBL))
+ "ALTER TABLE %s ADD PARTITION(i=1, p='p3')" % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
"ALTER TABLE %s PARTITION (i=1, p='p3') SET LOCATION '%s/%s.db/tmp' "
- % (self.TEST_TBL, WAREHOUSE, self.TEST_DB))
- self.filesystem_client.delete_file_dir(self.BASE_DIR + leaf_dir, recursive=True)
- self.filesystem_client.make_dir(self.BASE_DIR + leaf_dir);
- self.filesystem_client.create_file(self.BASE_DIR + leaf_dir + file_path,
- inserted_value)
+ % (FQ_TBL_NAME, WAREHOUSE, unique_database))
+ self.filesystem_client.delete_file_dir(TBL_LOCATION + LEAF_DIR, recursive=True)
+ self.filesystem_client.make_dir(TBL_LOCATION + LEAF_DIR);
+ self.filesystem_client.create_file(TBL_LOCATION + LEAF_DIR + FILE_PATH,
+ INSERTED_VALUE)
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s RECOVER PARTITIONS" % (self.TEST_TBL))
+ "ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
# Ensure that no duplicate partitions are recovered.
result = self.execute_query_expect_success(self.client,
- "select c from %s" % self.TEST_TBL)
- assert (self.has_value(inserted_value, result.data) == False,
- "ALTER TABLE %s RECOVER PARTITIONS failed to handle non-default partition location."
- % (self.TEST_TBL))
+ "select c from %s" % FQ_TBL_NAME)
+ assert not self.has_value(INSERTED_VALUE, result.data),\
+ "ALTER TABLE %s RECOVER PARTITIONS failed to handle "\
+ "non-default partition location." % FQ_TBL_NAME
self.execute_query_expect_success(self.client,
- "INSERT INTO TABLE %s PARTITION(i=1, p='p3') VALUES(4)" % (self.TEST_TBL))
+ "INSERT INTO TABLE %s PARTITION(i=1, p='p3') VALUES(4)" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "select c from %s" % self.TEST_TBL)
- assert self.has_value(inserted_value, result.data) == True
+ "select c from %s" % FQ_TBL_NAME)
+ assert self.has_value(INSERTED_VALUE, result.data)
@SkipIfLocal.hdfs_client
- @pytest.mark.execute_serially
- def test_duplicate_partitions(self, vector):
+ def test_duplicate_partitions(self, vector, unique_database):
"""Test that RECOVER PARTITIONS does not recover equivalent partitions. Two partitions
are considered equivalent if they correspond to distinct paths but can be converted
to the same partition key values (e.g. "i=0005/p=p2" and "i=05/p=p2")."""
- same_value_dir1 = "i=0004/p=p2/"
- same_value_dir2 = "i=000004/p=p2/"
- file_path = "test"
+ TBL_NAME = "test_recover_partitions"
+ FQ_TBL_NAME = unique_database + "." + TBL_NAME
+ TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME)
+ SAME_VALUE_DIR1 = "i=0004/p=p2/"
+ SAME_VALUE_DIR2 = "i=000004/p=p2/"
+ FILE_PATH = "test"
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % (self.TEST_TBL))
+ "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
- "INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % (self.TEST_TBL))
+ "INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % FQ_TBL_NAME)
# Create a partition with path "/i=1/p=p4".
# Create a path "/i=0001/p=p4" using hdfs client, and add a file with some values.
# Test that no new partition will be recovered and the inserted data are not
# accessible.
- leaf_dir = "i=0001/p=p4/"
- inserted_value = "5"
+ LEAF_DIR = "i=0001/p=p4/"
+ INSERTED_VALUE = "5"
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s ADD PARTITION(i=1, p='p4')" % (self.TEST_TBL))
- self.filesystem_client.make_dir(self.BASE_DIR + leaf_dir);
- self.filesystem_client.create_file(self.BASE_DIR + leaf_dir + file_path, inserted_value)
+ "ALTER TABLE %s ADD PARTITION(i=1, p='p4')" % FQ_TBL_NAME)
+ self.filesystem_client.make_dir(TBL_LOCATION + LEAF_DIR);
+ self.filesystem_client.create_file(TBL_LOCATION + LEAF_DIR + FILE_PATH, INSERTED_VALUE)
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s RECOVER PARTITIONS" % (self.TEST_TBL))
+ "ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "select c from %s" % self.TEST_TBL)
- assert (self.has_value(inserted_value, result.data) == False,
- "ALTER TABLE %s RECOVER PARTITIONS failed to handle duplicate partition key values."
- % (self.TEST_TBL))
+ "select c from %s" % FQ_TBL_NAME)
+ assert not self.has_value(INSERTED_VALUE, result.data),\
+ "ALTER TABLE %s RECOVER PARTITIONS failed to handle "\
+ "duplicate partition key values." % FQ_TBL_NAME
# Create two paths '/i=0004/p=p2/' and "i=000004/p=p2/" using hdfs client.
# Test that only one partition will be added.
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL))
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
old_length = len(result.data)
- self.filesystem_client.make_dir(self.BASE_DIR + same_value_dir1)
- self.filesystem_client.make_dir(self.BASE_DIR + same_value_dir2)
+ self.filesystem_client.make_dir(TBL_LOCATION + SAME_VALUE_DIR1)
+ self.filesystem_client.make_dir(TBL_LOCATION + SAME_VALUE_DIR2)
# Only one partition will be added.
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s RECOVER PARTITIONS" % (self.TEST_TBL))
+ "ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL))
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
assert ((old_length + 1) == len(result.data),
"ALTER TABLE %s RECOVER PARTITIONS failed to handle duplicate partition key values."
- % (self.TEST_TBL))
+ % FQ_TBL_NAME)
@SkipIfLocal.hdfs_client
- @pytest.mark.execute_serially
- def test_post_invalidate(self, vector):
+ def test_post_invalidate(self, vector, unique_database):
"""Test that RECOVER PARTITIONS works correctly after invalidate."""
- leaf_dir = "i=002/p=p2/"
- file_path = "test"
- inserted_value = "2"
+ TBL_NAME = "test_recover_partitions"
+ FQ_TBL_NAME = unique_database + "." + TBL_NAME
+ TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME)
+ LEAF_DIR = "i=002/p=p2/"
+ FILE_PATH = "test"
+ INSERTED_VALUE = "2"
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % (self.TEST_TBL))
+ "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
- "INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % (self.TEST_TBL))
+ "INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % FQ_TBL_NAME)
# Test that the recovered partitions are properly stored in Hive MetaStore.
# Invalidate the table metadata and then check if the recovered partitions
# are accessible.
- self.filesystem_client.make_dir(self.BASE_DIR + leaf_dir);
- self.filesystem_client.create_file(self.BASE_DIR + leaf_dir + file_path,
- inserted_value)
+ self.filesystem_client.make_dir(TBL_LOCATION + LEAF_DIR)
+ self.filesystem_client.create_file(TBL_LOCATION + LEAF_DIR + FILE_PATH,
+ INSERTED_VALUE)
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s RECOVER PARTITIONS" % (self.TEST_TBL))
+ "ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "select c from %s" % self.TEST_TBL)
- assert self.has_value(inserted_value, result.data) == True
- self.client.execute("INVALIDATE METADATA %s" % (self.TEST_TBL))
+ "select c from %s" % FQ_TBL_NAME)
+ assert self.has_value(INSERTED_VALUE, result.data)
+ self.client.execute("INVALIDATE METADATA %s" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "select c from %s" % self.TEST_TBL)
- assert (self.has_value(inserted_value, result.data) == True,
+ "select c from %s" % FQ_TBL_NAME)
+ assert (self.has_value(INSERTED_VALUE, result.data),
"INVALIDATE can't work on partitions recovered by ALTER TABLE %s RECOVER PARTITIONS."
- % (self.TEST_TBL))
+ % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
- "INSERT INTO TABLE %s PARTITION(i=002, p='p2') VALUES(4)" % (self.TEST_TBL))
+ "INSERT INTO TABLE %s PARTITION(i=002, p='p2') VALUES(4)" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "select c from %s" % self.TEST_TBL)
- assert self.has_value('4', result.data) == True
+ "select c from %s" % FQ_TBL_NAME)
+ assert self.has_value('4', result.data)
@SkipIfLocal.hdfs_client
- @pytest.mark.execute_serially
- def test_support_all_types(self, vector):
+ def test_support_all_types(self, vector, unique_database):
"""Test that RECOVER PARTITIONS works correctly on all supported data types."""
+ TBL_NAME = "test_recover_partitions"
+ FQ_TBL_NAME = unique_database + "." + TBL_NAME
+ TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME)
+
normal_values = ["a=1", "b=128", "c=32768", "d=2147483648", "e=11.11",
"f=22.22", "g=33.33", "j=tchar", "k=tvchar", "s=recover"]
malformed_values = ["a=a", "b=b", "c=c", "d=d", "e=e", "f=f", "g=g"]
@@ -267,38 +267,40 @@ class TestRecoverPartitions(ImpalaTestSuite):
self.execute_query_expect_success(self.client,
"CREATE TABLE %s (i INT) PARTITIONED BY (a TINYINT, b SMALLINT, c INT, d BIGINT,"
" e DECIMAL(4,2), f FLOAT, g DOUBLE, j CHAR(5), k VARCHAR(6), s STRING)"
- % (self.TEST_TBL2))
+ % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
"INSERT INTO TABLE %s PARTITION(a=1, b=2, c=3, d=4, e=55.55, f=6.6, g=7.7, "
"j=cast('j' as CHAR(5)), k=cast('k' as VARCHAR(6)), s='s') VALUES(1)"
- % (self.TEST_TBL2))
+ % FQ_TBL_NAME)
# Test valid partition values.
normal_dir = ""
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL2))
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
old_length = len(result.data)
normal_dir = '/'.join(normal_values)
- self.filesystem_client.make_dir(self.BASE_DIR2 + normal_dir)
+ self.filesystem_client.make_dir(TBL_LOCATION + normal_dir)
# One partition will be added.
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s RECOVER PARTITIONS" % (self.TEST_TBL2))
+ "ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL2))
+ "SHOW PARTITIONS %s" % FQ_TBL_NAME)
assert (len(result.data) == (old_length + 1),
"ALTER TABLE %s RECOVER PARTITIONS failed to handle some data types."
- % (self.TEST_TBL))
+ % FQ_TBL_NAME)
# Test malformed partition values.
- self.check_invalid_partition_values(normal_values, malformed_values)
+ self.check_invalid_partition_values(FQ_TBL_NAME, TBL_LOCATION,
+ normal_values, malformed_values)
# Test overflow partition values.
- self.check_invalid_partition_values(normal_values, overflow_values)
+ self.check_invalid_partition_values(FQ_TBL_NAME, TBL_LOCATION,
+ normal_values, overflow_values)
- @SkipIfLocal.hdfs_client
- def check_invalid_partition_values(self, normal_values, invalid_values):
+ def check_invalid_partition_values(self, fq_tbl_name, tbl_location,
+ normal_values, invalid_values):
""""Check that RECOVER PARTITIONS ignores partitions with invalid partition values."""
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL2))
+ "SHOW PARTITIONS %s" % fq_tbl_name)
old_length = len(result.data)
for i in range(len(invalid_values)):
@@ -308,17 +310,16 @@ class TestRecoverPartitions(ImpalaTestSuite):
invalid_dir += (normal_values[j] + "/")
else:
invalid_dir += (invalid_values[j] + "/")
- self.filesystem_client.make_dir(self.BASE_DIR2 + invalid_dir)
+ self.filesystem_client.make_dir(tbl_location + invalid_dir)
# No partition will be added.
self.execute_query_expect_success(self.client,
- "ALTER TABLE %s RECOVER PARTITIONS" % (self.TEST_TBL2))
+ "ALTER TABLE %s RECOVER PARTITIONS" % fq_tbl_name)
result = self.execute_query_expect_success(self.client,
- "SHOW PARTITIONS %s" % (self.TEST_TBL2))
+ "SHOW PARTITIONS %s" % fq_tbl_name)
assert (len(result.data) == old_length,
"ALTER TABLE %s RECOVER PARTITIONS failed to handle invalid partition key values."
- % (self.TEST_TBL))
+ % fq_tbl_name)
def has_value(self, value, lines):
"""Check if lines contain value."""
return any([line.find(value) != -1 for line in lines])
-