[impala] 02/02: IMPALA-8931: Fix fe trigger for lineage events

[bh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

bharathv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 4327cc351317878de51003c2dcf4a71c9183eb44
Author: Bharath Vissapragada <bh...@cloudera.com>
AuthorDate: Sat Sep 7 15:46:15 2019 -0700

    IMPALA-8931: Fix fe trigger for lineage events
    
    Currently, fe generates lineage objects only when --lineage_event_log_dir
    is configured. This is a legacy startup param. Lineages should also be
    generated when event hooks are configured since they can potentially
    consume them.
    
    Added a test that confirms that the hook is invoked when a lineage is
    created.
    
    Change-Id: I2d8deb05883cc3ecab27fe4afd031a1e7ccb0829
    Reviewed-on: http://gerrit.cloudera.org:8080/14194
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 fe/src/main/java/org/apache/impala/service/BackendConfig.java |  6 +++++-
 tests/custom_cluster/test_query_event_hooks.py                | 11 ++++++++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index 534dc70..b528435 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -51,7 +51,11 @@ public class BackendConfig {
   public TBackendGflags getBackendCfg() { return backendCfg_; }
   public long getReadSize() { return backendCfg_.read_size; }
   public boolean getComputeLineage() {
-    return !Strings.isNullOrEmpty(backendCfg_.lineage_event_log_dir);
+    // Lineage is computed in the fe if --lineage_event_log_dir is configured or
+    // a query event hook is configured with --query_event_hook_classes. The hook
+    // may or may not consume the lineage but we still include it.
+    return !Strings.isNullOrEmpty(backendCfg_.lineage_event_log_dir) ||
+        !Strings.isNullOrEmpty(getQueryExecHookClasses());
   }
   public long getIncStatsMaxSize() { return backendCfg_.inc_stats_size_limit_bytes; }
   public boolean isStatsExtrapolationEnabled() {
diff --git a/tests/custom_cluster/test_query_event_hooks.py b/tests/custom_cluster/test_query_event_hooks.py
index d0d22ac..a884a22 100644
--- a/tests/custom_cluster/test_query_event_hooks.py
+++ b/tests/custom_cluster/test_query_event_hooks.py
@@ -35,7 +35,7 @@ class TestHooks(CustomClusterTestSuite):
   @CustomClusterTestSuite.with_args(
       impala_log_dir=tempfile.mkdtemp(prefix="test_hooks_", dir=os.getenv("LOG_DIR")),
       impalad_args="--query_event_hook_classes={0} "
-                   "--minidump_path={1} "
+                   "--minidump_path={1} -logbuflevel=-1"
                    .format(DUMMY_HOOK, MINIDUMP_PATH),
       catalogd_args="--minidump_path={0}".format(MINIDUMP_PATH))
   def test_query_event_hooks_execute(self, unique_database):
@@ -45,8 +45,13 @@ class TestHooks(CustomClusterTestSuite):
     """
     # Dummy hook should log something (See org.apache.impala.testutil.DummyQueryEventHook)
     self.assert_impalad_log_contains("INFO",
-                                     "{0}.onImpalaStartup".format(self.DUMMY_HOOK),
-                                     expected_count=-1)
+        "{0}.onImpalaStartup".format(self.DUMMY_HOOK), expected_count=-1)
+    # Run a test query that triggers a lineage event.
+    self.execute_query_expect_success(
+        self.client, "select count(*) from functional.alltypes")
+    # onQueryComplete() is invoked by the lineage logger.
+    self.assert_impalad_log_contains("INFO",
+        "{0}.onQueryComplete".format(self.DUMMY_HOOK), expected_count=-1)
 
 
 class TestHooksStartupFail(CustomClusterTestSuite):