You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/05/14 08:30:10 UTC
[4/5] incubator-impala git commit: IMPALA-3532: S3:
test_truncate_cleans_hdfs_files fails because we skip INSERT staging
IMPALA-3532: S3: test_truncate_cleans_hdfs_files fails because we skip INSERT staging
On the introduction of IMPALA-3452, we defaulted to skipping the
INSERT staging for S3. test_truncate_cleans_hdfs_files assumes that
it will always see the _impala_insert_staging folder but we will not
see that on S3 runs.
This patch deletes the staging folder if it exists and continues the
test without taking into account the staging folder.
Change-Id: I3580f03690e29fe99f441b26bc9baa4c0964d79c
Reviewed-on: http://gerrit.cloudera.org:8080/3049
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4c9c74dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4c9c74dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4c9c74dd
Branch: refs/heads/master
Commit: 4c9c74dd33e0b84c3a3328df0a1721d627267192
Parents: b4558d3
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Thu May 12 13:07:58 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Sat May 14 01:30:01 2016 -0700
----------------------------------------------------------------------
tests/metadata/test_ddl.py | 54 +++++++++++++++++++++++------------------
1 file changed, 30 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4c9c74dd/tests/metadata/test_ddl.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 0a1900c..6179d1a 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -183,30 +183,36 @@ class TestDdlStatements(ImpalaTestSuite):
# Verify the table directory exists
assert self.filesystem_client.exists("test-warehouse/truncate_table_test_db.db/t1/")
- # Should have created one file in the table's dir
- self.client.execute("insert into %s.t1 values (1)" % TRUNCATE_DB)
- assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 2
-
- # Truncating the table removes the data files and preserves the table's directory
- self.client.execute("truncate table %s.t1" % TRUNCATE_DB)
- assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 1
-
- self.client.execute(
- "create table %s.t2(i int) partitioned by (p int)" % TRUNCATE_DB)
- # Verify the table directory exists
- assert self.filesystem_client.exists("test-warehouse/%s.db/t2/" % TRUNCATE_DB)
-
- # Should have created the partition dir, which should contain exactly one file
- self.client.execute(
- "insert into %s.t2 partition(p=1) values (1)" % TRUNCATE_DB)
- assert len(self.filesystem_client.ls(
- "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 1
-
- # Truncating the table removes the data files and preserves the partition's directory
- self.client.execute("truncate table %s.t2" % TRUNCATE_DB)
- assert self.filesystem_client.exists("test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)
- assert len(self.filesystem_client.ls(
- "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 0
+ try:
+ # If we're testing S3, we want the staging directory to be created.
+ self.client.execute("set s3_skip_insert_staging=false")
+ # Should have created one file in the table's dir
+ self.client.execute("insert into %s.t1 values (1)" % TRUNCATE_DB)
+ assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 2
+
+ # Truncating the table removes the data files and preserves the table's directory
+ self.client.execute("truncate table %s.t1" % TRUNCATE_DB)
+ assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 1
+
+ self.client.execute(
+ "create table %s.t2(i int) partitioned by (p int)" % TRUNCATE_DB)
+ # Verify the table directory exists
+ assert self.filesystem_client.exists("test-warehouse/%s.db/t2/" % TRUNCATE_DB)
+
+ # Should have created the partition dir, which should contain exactly one file
+ self.client.execute(
+ "insert into %s.t2 partition(p=1) values (1)" % TRUNCATE_DB)
+ assert len(self.filesystem_client.ls(
+ "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 1
+
+ # Truncating the table removes the data files and preserves the partition's directory
+ self.client.execute("truncate table %s.t2" % TRUNCATE_DB)
+ assert self.filesystem_client.exists("test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)
+ assert len(self.filesystem_client.ls(
+ "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 0
+ finally:
+ # Reset to its default value.
+ self.client.execute("set s3_skip_insert_staging=true")
@pytest.mark.execute_serially
def test_truncate_table(self, vector):