[impala] 01/05: IMPALA-8638: Fix flaky TestLineage::test_create_table_timestamp

[lv...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

lv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit d4c05b96e2f1166572b01cae31b20ecd22fa4094
Author: Fredy Wijaya <fw...@cloudera.com>
AuthorDate: Fri Jun 7 10:29:55 2019 -0700

    IMPALA-8638: Fix flaky TestLineage::test_create_table_timestamp
    
    This patch fixes the bug TestLineage::test_create_table_timestamp where
    it uses the same lineage log directory created by
    TestLineage::test_start_end_timestamp, which could fail the query ID
    assertion. The fix is to use a different lineage log directory than the
    one used by TestLineage::test_start_end_timestamp.
    
    Testing:
    - Ran test_lineage.py multiple times and it still passed.
    
    Change-Id: I5813e4a570c181ba196b9ddf0210c8a0d92e21e8
    Reviewed-on: http://gerrit.cloudera.org:8080/13560
    Reviewed-by: Fredy Wijaya <fw...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 tests/custom_cluster/test_lineage.py | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/tests/custom_cluster/test_lineage.py b/tests/custom_cluster/test_lineage.py
index 35469aa..62746cb 100644
--- a/tests/custom_cluster/test_lineage.py
+++ b/tests/custom_cluster/test_lineage.py
@@ -16,15 +16,12 @@
 # under the License.
 #
 # Tests for column lineage.
-# TODO: add verification for more fields.
 
 import json
 import logging
 import os
 import pytest
 import re
-import shutil
-import stat
 import tempfile
 import time
 
@@ -34,22 +31,20 @@ LOG = logging.getLogger(__name__)
 
 
 class TestLineage(CustomClusterTestSuite):
-  lineage_log_dir = tempfile.mkdtemp()
+  START_END_TIME_LINEAGE_LOG_DIR = tempfile.mkdtemp(prefix="start_end_time")
+  CREATE_TABLE_TIME_LINEAGE_LOG_DIR = tempfile.mkdtemp(prefix="create_table_time")
 
   @classmethod
   def setup_class(cls):
     super(TestLineage, cls).setup_class()
 
-  @classmethod
-  def teardown_class(cls):
-    shutil.rmtree(cls.lineage_log_dir)
-
   @pytest.mark.execute_serially
-  @CustomClusterTestSuite.with_args("--lineage_event_log_dir=%s" % lineage_log_dir)
+  @CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}"
+                                    .format(START_END_TIME_LINEAGE_LOG_DIR))
   def test_start_end_timestamp(self, vector):
     """Test that 'timestamp' and 'endTime' in the lineage graph are populated with valid
        UNIX times."""
-    LOG.info("lineage_event_log_dir is " + self.lineage_log_dir)
+    LOG.info("lineage_event_log_dir is {0}".format(self.START_END_TIME_LINEAGE_LOG_DIR))
     before_time = int(time.time())
     query = "select count(*) from functional.alltypes"
     result = self.execute_query_expect_success(self.client, query)
@@ -60,8 +55,8 @@ class TestLineage(CustomClusterTestSuite):
     # Stop the cluster in order to flush the lineage log files.
     self._stop_impala_cluster()
 
-    for log_filename in os.listdir(self.lineage_log_dir):
-      log_path = os.path.join(self.lineage_log_dir, log_filename)
+    for log_filename in os.listdir(self.START_END_TIME_LINEAGE_LOG_DIR):
+      log_path = os.path.join(self.START_END_TIME_LINEAGE_LOG_DIR, log_filename)
       # Only the coordinator's log file will be populated.
       if os.path.getsize(log_path) > 0:
         LOG.info("examining file: " + log_path)
@@ -77,7 +72,8 @@ class TestLineage(CustomClusterTestSuite):
         LOG.info("empty file: " + log_path)
 
   @pytest.mark.execute_serially
-  @CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}".format(lineage_log_dir))
+  @CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}"
+                                    .format(CREATE_TABLE_TIME_LINEAGE_LOG_DIR))
   def test_create_table_timestamp(self, vector, unique_database):
     """Test that 'createTableTime' in the lineage graph are populated with valid value
        from HMS."""
@@ -89,8 +85,8 @@ class TestLineage(CustomClusterTestSuite):
     # Wait to flush the lineage log files.
     time.sleep(3)
 
-    for log_filename in os.listdir(self.lineage_log_dir):
-      log_path = os.path.join(self.lineage_log_dir, log_filename)
+    for log_filename in os.listdir(self.CREATE_TABLE_TIME_LINEAGE_LOG_DIR):
+      log_path = os.path.join(self.CREATE_TABLE_TIME_LINEAGE_LOG_DIR, log_filename)
       # Only the coordinator's log file will be populated.
       if os.path.getsize(log_path) > 0:
         with open(log_path) as log_file: