You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by he...@apache.org on 2017/06/08 05:26:00 UTC

[1/3] incubator-impala git commit: IMPALA-4164: Avoid overly aggressive inlining in LLVM IR

Repository: incubator-impala
Updated Branches:
  refs/heads/master 26a3d04b8 -> 541180105


IMPALA-4164: Avoid overly aggressive inlining in LLVM IR

When generating IR functions during codegen, we used to
always tag the functions with the "AlwaysInline" attribute.
That potentially leads to excessive inlining, causing very
long optimization / compilation time with marginal performance
benefit at runtime. One of the reasons for doing it was that
the "target-cpu" and "target-features" attributes were
missing in the generated IR functions so the LLVM inliner
considers them incompatible with the cross-compiled functions.
As a result, the inliner will not inline the generated IR
functions into cross-compiled functions and vice versa unless
the "AlwaysInline" attributes exist.

This change fixes the problem above by setting the "target-cpu"
and "target-features" attributes of all IR functions to match
that of of the host's CPUs so both generated IR functions and
cross-compiled functions will have the same values for those
attributes. With these attributes set, we now rely on the
inliner of LLVM to determine whether a function is worth being
inlined. With this change, the codegen time of a query with very
long predicate went from 15s to 4s and the overall runtime went
from 19s to 8s.

Change-Id: I2d87ae8d222b415587e7320cb9072e4a8d6615ce
Reviewed-on: http://gerrit.cloudera.org:8080/6941
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/91237051
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/91237051
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/91237051

Branch: refs/heads/master
Commit: 91237051af9fd02f8aba5987e25f21ca98b33e52
Parents: 26a3d04
Author: Michael Ho <kw...@cloudera.com>
Authored: Fri May 19 00:12:45 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Jun 8 00:16:12 2017 +0000

----------------------------------------------------------------------
 be/src/codegen/llvm-codegen.cc                  |  30 ++-
 be/src/codegen/llvm-codegen.h                   |  14 +-
 .../queries/primitive_long_predicate.test       | 182 +++++++++++++++++++
 3 files changed, 214 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/91237051/be/src/codegen/llvm-codegen.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/llvm-codegen.cc b/be/src/codegen/llvm-codegen.cc
index cdc0c53..1a4fd8b 100644
--- a/be/src/codegen/llvm-codegen.cc
+++ b/be/src/codegen/llvm-codegen.cc
@@ -104,6 +104,7 @@ namespace impala {
 bool LlvmCodeGen::llvm_initialized_ = false;
 string LlvmCodeGen::cpu_name_;
 vector<string> LlvmCodeGen::cpu_attrs_;
+string LlvmCodeGen::target_features_attr_;
 CodegenCallGraph LlvmCodeGen::shared_call_graph_;
 
 [[noreturn]] static void LlvmCodegenHandleError(
@@ -134,8 +135,8 @@ Status LlvmCodeGen::InitializeLlvm(bool load_backend) {
   cpu_name_ = llvm::sys::getHostCPUName().str();
   LOG(INFO) << "CPU class for runtime code generation: " << cpu_name_;
   GetHostCPUAttrs(&cpu_attrs_);
-  LOG(INFO) << "CPU flags for runtime code generation: "
-            << boost::algorithm::join(cpu_attrs_, ",");
+  target_features_attr_ = boost::algorithm::join(cpu_attrs_, ",");
+  LOG(INFO) << "CPU flags for runtime code generation: " << target_features_attr_;
 
   // Write an empty map file for perf to find.
   if (FLAGS_perf_map) CodegenSymbolEmitter::WritePerfMap();
@@ -150,7 +151,7 @@ Status LlvmCodeGen::InitializeLlvm(bool load_backend) {
   // Validate the module by verifying that functions for all IRFunction::Type
   // can be found.
   for (int i = IRFunction::FN_START; i < IRFunction::FN_END; ++i) {
-    DCHECK(FN_MAPPINGS[i].fn == i);
+    DCHECK_EQ(FN_MAPPINGS[i].fn, i);
     const string& fn_name = FN_MAPPINGS[i].fn_name;
     if (init_codegen->module_->getFunction(fn_name) == nullptr) {
       return Status(Substitute("Failed to find function $0", fn_name));
@@ -589,6 +590,7 @@ Status LlvmCodeGen::MaterializeFunctionHelper(Function *fn) {
 
   // Materialized functions are marked as not materializable by LLVM.
   DCHECK(!fn->isMaterializable());
+  SetCPUAttrs(fn);
   const unordered_set<string>* callees = shared_call_graph_.GetCallees(fn->getName());
   if (callees != nullptr) {
     for (const string& callee : *callees) {
@@ -628,8 +630,6 @@ Function* LlvmCodeGen::GetFunction(IRFunction::Type ir_type, bool clone) {
       LOG(ERROR) << "Unable to locate function " << fn_name;
       return NULL;
     }
-    // Mixing "NoInline" with "AlwaysInline" will lead to compilation failure.
-    if (!fn->hasFnAttribute(Attribute::NoInline)) fn->addFnAttr(Attribute::AlwaysInline);
     loaded_functions_[ir_type] = fn;
   }
   Status status = MaterializeFunction(fn);
@@ -692,9 +692,20 @@ bool LlvmCodeGen::VerifyFunction(Function* fn) {
 
 void LlvmCodeGen::SetNoInline(llvm::Function* function) const {
   function->removeFnAttr(llvm::Attribute::AlwaysInline);
+  function->removeFnAttr(llvm::Attribute::InlineHint);
   function->addFnAttr(llvm::Attribute::NoInline);
 }
 
+void LlvmCodeGen::SetCPUAttrs(llvm::Function* function) {
+  // Set all functions' "target-cpu" and "target-features" to match the
+  // host's CPU's features. It's assumed that the functions don't use CPU
+  // features which the host doesn't support. CreateFromMemory() checks
+  // the features of the host's CPU and loads the module compatible with
+  // the host's CPU.
+  function->addFnAttr("target-cpu", cpu_name_);
+  function->addFnAttr("target-features", target_features_attr_);
+}
+
 LlvmCodeGen::FnPrototype::FnPrototype(
     LlvmCodeGen* codegen, const string& name, Type* ret_type)
   : codegen_(codegen), name_(name), ret_type_(ret_type) {
@@ -951,10 +962,7 @@ Function* LlvmCodeGen::CloneFunction(Function* fn) {
 }
 
 Function* LlvmCodeGen::FinalizeFunction(Function* function) {
-  if (LIKELY(!function->hasFnAttribute(llvm::Attribute::NoInline))) {
-    function->addFnAttr(llvm::Attribute::AlwaysInline);
-  }
-
+  SetCPUAttrs(function);
   if (!VerifyFunction(function)) {
     function->eraseFromParent(); // deletes function
     return NULL;
@@ -1080,7 +1088,9 @@ Status LlvmCodeGen::OptimizeModule() {
   pass_builder.OptLevel = 2;
   // Don't optimize for code size (this corresponds to -O2/-O3)
   pass_builder.SizeLevel = 0;
-  pass_builder.Inliner = createFunctionInliningPass();
+  // Use a threshold equivalent to adding InlineHint on all functions.
+  // This results in slightly better performance than the default threshold (225).
+  pass_builder.Inliner = createFunctionInliningPass(325);
 
   // The TargetIRAnalysis pass is required to provide information about the target
   // machine to optimisation passes, e.g. the cost model.

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/91237051/be/src/codegen/llvm-codegen.h
----------------------------------------------------------------------
diff --git a/be/src/codegen/llvm-codegen.h b/be/src/codegen/llvm-codegen.h
index 5c26c14..07aae55 100644
--- a/be/src/codegen/llvm-codegen.h
+++ b/be/src/codegen/llvm-codegen.h
@@ -407,8 +407,8 @@ class LlvmCodeGen {
   llvm::Function* GetFnvHashFunction(int num_bytes = -1);
   llvm::Function* GetMurmurHashFunction(int num_bytes = -1);
 
-  /// Set the NoInline attribute on 'function' and remove the AlwaysInline attribute if
-  /// present.
+  /// Set the NoInline attribute on 'function' and remove the AlwaysInline and InlineHint
+  /// attributes if present.
   void SetNoInline(llvm::Function* function) const;
 
   /// Allocate stack storage for local variables.  This is similar to traditional c, where
@@ -570,6 +570,11 @@ class LlvmCodeGen {
   /// anyway (they must be explicitly invoked) so it is dead code.
   static void StripGlobalCtorsDtors(llvm::Module* module);
 
+  /// Set the "target-cpu" and "target-features" of 'function' to match the host's CPU's
+  /// features. Having consistent attributes for all materialized functions allows
+  /// generated IR to be inlined into cross-compiled functions' IR and vice versa.
+  static void SetCPUAttrs(llvm::Function* function);
+
   // Setup any JIT listeners to process generated machine code object, e.g. to generate
   // perf symbol map or disassembly.
   void SetupJITListeners();
@@ -640,6 +645,11 @@ class LlvmCodeGen {
   static std::string cpu_name_;
   static std::vector<std::string> cpu_attrs_;
 
+  /// Value of "target-features" attribute to be set on all IR functions. Derived from
+  /// 'cpu_attrs_'. Using a consistent value for this attribute among hand-crafted IR
+  /// and cross-compiled functions allow them to be inlined into each other.
+  static std::string target_features_attr_;
+
   /// A global shared call graph for all IR functions in the main module.
   /// Used for determining dependencies when materializing IR functions.
   static CodegenCallGraph shared_call_graph_;

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/91237051/testdata/workloads/targeted-perf/queries/primitive_long_predicate.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/targeted-perf/queries/primitive_long_predicate.test b/testdata/workloads/targeted-perf/queries/primitive_long_predicate.test
new file mode 100644
index 0000000..b7ae8c8
--- /dev/null
+++ b/testdata/workloads/targeted-perf/queries/primitive_long_predicate.test
@@ -0,0 +1,182 @@
+====
+---- QUERY: primitive_long_predicate
+-- Description: A simple scan with extremely long predicate.
+-- Target test case: Query generated by BI tools with complex expressions. Aim to
+--  exercise long codegen time due to inlining. Intentionally disabled query rewrite
+--  in the FE to expose the long expressions to the BE.
+SELECT COUNT(*)
+FROM lineitem
+WHERE ((l_orderkey = 520 AND  l_shipdate between "2017-01-01 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 534 AND  l_shipdate between "2017-01-02 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 518 AND  l_shipdate between "2017-01-03 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 535 AND  l_shipdate between "2017-01-04 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 496 AND  l_shipdate between "2017-01-05 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 497 AND  l_shipdate between "2017-01-06 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 516 AND  l_shipdate between "2017-01-07 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 531 AND  l_shipdate between "2017-01-08 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 533 AND  l_shipdate between "2017-01-09 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 532 AND  l_shipdate between "2017-01-10 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 546 AND  l_shipdate between "2017-01-11 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 549 AND  l_shipdate between "2017-01-12 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 548 AND  l_shipdate between "2017-01-13 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 547 AND  l_shipdate between "2017-01-14 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 513 AND  l_shipdate between "2017-01-15 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 515 AND  l_shipdate between "2017-01-16 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 507 AND  l_shipdate between "2017-01-17 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 509 AND  l_shipdate between "2017-01-18 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 519 AND  l_shipdate between "2017-01-19 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 527 AND  l_shipdate between "2017-01-20 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 501 AND  l_shipdate between "2017-01-21 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 544 AND  l_shipdate between "2017-01-22 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 545 AND  l_shipdate between "2017-01-23 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 541 AND  l_shipdate between "2017-01-24 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 550 AND  l_shipdate between "2017-01-25 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 529 AND  l_shipdate between "2017-01-26 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 528 AND  l_shipdate between "2017-01-27 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 537 AND  l_shipdate between "2017-01-28 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 538 AND  l_shipdate between "2017-01-29 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 506 AND  l_shipdate between "2017-01-30 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 502 AND  l_shipdate between "2017-01-31 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 543 AND  l_shipdate between "2017-03-01 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 390 AND  l_shipdate between "2017-03-02 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 389 AND  l_shipdate between "2017-03-03 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 0   AND  l_shipdate between "2017-03-04 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 852 AND  l_shipdate between "2017-03-05 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 853 AND  l_shipdate between "2017-03-06 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 854 AND  l_shipdate between "2017-03-07 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 844 AND  l_shipdate between "2017-03-08 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 846 AND  l_shipdate between "2017-03-09 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 843 AND  l_shipdate between "2017-03-10 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 849 AND  l_shipdate between "2017-03-11 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 850 AND  l_shipdate between "2017-03-12 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 847 AND  l_shipdate between "2017-03-13 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 746 AND  l_shipdate between "2017-03-14 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 745 AND  l_shipdate between "2017-03-15 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 901 AND  l_shipdate between "2017-03-16 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 899 AND  l_shipdate between "2017-03-17 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 895 AND  l_shipdate between "2017-03-18 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 878 AND  l_shipdate between "2017-03-19 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 884 AND  l_shipdate between "2017-03-20 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 871 AND  l_shipdate between "2017-03-21 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 872 AND  l_shipdate between "2017-03-22 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 886 AND  l_shipdate between "2017-03-23 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 879 AND  l_shipdate between "2017-03-24 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 864 AND  l_shipdate between "2017-03-25 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 889 AND  l_shipdate between "2017-03-26 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 885 AND  l_shipdate between "2017-03-27 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 848 AND  l_shipdate between "2017-03-28 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 888 AND  l_shipdate between "2017-03-29 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 863 AND  l_shipdate between "2017-03-30 12:00:00" AND  "2017-04-17 02:22:02") OR
+       (l_orderkey = 930 AND  l_shipdate between "2017-03-31 12:00:00" AND  "2017-04-17 02:22:03") OR
+       (l_orderkey = 891 AND  l_shipdate between "2017-05-01 12:00:00" AND  "2017-06-17 02:22:02") OR
+       (l_orderkey = 890 AND  l_shipdate between "2017-05-02 12:00:00" AND  "2017-06-17 02:22:03") OR
+       (l_orderkey = 926 AND  l_shipdate between "2017-05-03 12:00:00" AND  "2017-06-17 02:22:02") OR
+       (l_orderkey = 927 AND  l_shipdate between "2017-05-04 12:00:00" AND  "2017-06-17 02:22:03") OR
+       (l_orderkey = 892 AND  l_shipdate between "2017-05-05 12:00:00" AND  "2017-06-17 02:22:02") OR
+       (l_orderkey = 925 AND  l_shipdate between "2017-05-06 12:00:00" AND  "2017-06-17 02:22:03") OR
+       (l_orderkey = 887 AND  l_shipdate between "2017-05-07 12:00:00" AND  "2017-06-17 02:22:02") OR
+       (l_orderkey = 898 AND  l_shipdate between "2017-05-08 12:00:00" AND  "2017-06-17 02:22:03") OR
+       (l_orderkey = 902 AND  l_shipdate between "2017-05-09 12:00:00" AND  "2017-06-17 02:22:02") OR
+       (l_orderkey = 903 AND  l_shipdate between "2017-05-10 12:00:00" AND  "2017-06-17 02:22:03") OR
+       (l_orderkey = 904 AND  l_shipdate between "2017-05-11 12:00:00" AND  "2017-06-17 02:22:02") OR
+       (l_orderkey = 897 AND  l_shipdate between "2017-05-12 12:00:00" AND  "2017-06-17 02:22:03") OR
+       (l_orderkey = 900 AND  l_shipdate between "2017-05-13 12:00:00" AND  "2017-06-17 02:22:02") OR
+       (l_orderkey = 598 AND  l_shipdate between "2017-05-14 12:00:00" AND  "2017-06-17 02:22:03") OR
+       (l_orderkey = 597 AND  l_shipdate between "2017-05-15 12:00:00" AND  "2017-06-17 02:22:02") OR
+       (l_orderkey = 607 AND  l_shipdate between "2017-05-16 12:00:00" AND  "2017-06-17 02:22:03") OR
+       (l_orderkey = 288 AND  l_shipdate between "2017-01-01 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 289 AND  l_shipdate between "2017-01-02 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 281 AND  l_shipdate between "2017-01-03 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 287 AND  l_shipdate between "2017-01-04 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 290 AND  l_shipdate between "2017-01-05 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 264 AND  l_shipdate between "2017-01-06 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 292 AND  l_shipdate between "2017-01-07 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 241 AND  l_shipdate between "2017-01-08 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 291 AND  l_shipdate between "2017-01-09 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 286 AND  l_shipdate between "2017-01-10 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 148 AND  l_shipdate between "2017-01-11 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 298 AND  l_shipdate between "2017-01-12 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 278 AND  l_shipdate between "2017-01-13 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 284 AND  l_shipdate between "2017-01-14 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 285 AND  l_shipdate between "2017-01-15 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 283 AND  l_shipdate between "2017-01-16 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 295 AND  l_shipdate between "2017-01-17 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 296 AND  l_shipdate between "2017-01-18 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 293 AND  l_shipdate between "2017-01-19 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 294 AND  l_shipdate between "2017-01-20 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 282 AND  l_shipdate between "2017-01-21 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 263 AND  l_shipdate between "2017-01-22 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 297 AND  l_shipdate between "2017-01-23 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 299 AND  l_shipdate between "2017-01-24 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 300 AND  l_shipdate between "2017-01-25 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 265 AND  l_shipdate between "2017-01-26 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 267 AND  l_shipdate between "2017-01-27 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 520 AND  l_shipdate between "2017-01-28 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 534 AND  l_shipdate between "2017-01-29 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 518 AND  l_shipdate between "2017-01-30 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 535 AND  l_shipdate between "2017-01-31 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 496 AND  l_shipdate between "2017-02-01 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 497 AND  l_shipdate between "2017-02-02 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 516 AND  l_shipdate between "2017-02-03 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 531 AND  l_shipdate between "2017-02-04 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 533 AND  l_shipdate between "2017-02-05 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 532 AND  l_shipdate between "2017-02-06 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 546 AND  l_shipdate between "2017-02-07 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 549 AND  l_shipdate between "2017-02-08 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 548 AND  l_shipdate between "2017-02-09 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 547 AND  l_shipdate between "2017-02-10 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 513 AND  l_shipdate between "2017-02-11 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 515 AND  l_shipdate between "2017-02-12 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 507 AND  l_shipdate between "2017-02-13 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 509 AND  l_shipdate between "2017-02-14 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 519 AND  l_shipdate between "2017-02-15 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 527 AND  l_shipdate between "2017-02-16 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 503 AND  l_shipdate between "2017-02-17 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 498 AND  l_shipdate between "2017-02-18 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 501 AND  l_shipdate between "2017-02-19 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 544 AND  l_shipdate between "2017-02-20 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 545 AND  l_shipdate between "2017-02-21 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 540 AND  l_shipdate between "2017-02-22 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 541 AND  l_shipdate between "2017-02-23 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 550 AND  l_shipdate between "2017-02-24 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 529 AND  l_shipdate between "2017-02-25 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 528 AND  l_shipdate between "2017-03-01 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 537 AND  l_shipdate between "2017-03-02 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 538 AND  l_shipdate between "2017-03-03 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 506 AND  l_shipdate between "2017-03-04 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 500 AND  l_shipdate between "2017-03-05 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 539 AND  l_shipdate between "2017-03-06 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 512 AND  l_shipdate between "2017-03-07 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 502 AND  l_shipdate between "2017-03-08 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 504 AND  l_shipdate between "2017-03-09 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 543 AND  l_shipdate between "2017-03-10 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 390 AND  l_shipdate between "2017-03-11 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 389 AND  l_shipdate between "2017-03-12 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 0   AND  l_shipdate between "2017-03-13 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 844 AND  l_shipdate between "2017-03-14 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 846 AND  l_shipdate between "2017-03-15 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 894 AND  l_shipdate between "2017-03-16 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 843 AND  l_shipdate between "2017-03-17 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 744 AND  l_shipdate between "2017-04-01 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 847 AND  l_shipdate between "2017-04-02 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 746 AND  l_shipdate between "2017-04-03 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 745 AND  l_shipdate between "2017-04-04 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 893 AND  l_shipdate between "2017-04-05 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 901 AND  l_shipdate between "2017-04-06 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 899 AND  l_shipdate between "2017-04-07 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 878 AND  l_shipdate between "2017-04-08 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 884 AND  l_shipdate between "2017-04-09 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 883 AND  l_shipdate between "2017-04-10 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 879 AND  l_shipdate between "2017-04-11 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 929 AND  l_shipdate between "2017-04-12 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 885 AND  l_shipdate between "2017-04-13 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 891 AND  l_shipdate between "2017-04-14 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 892 AND  l_shipdate between "2017-04-15 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 882 AND  l_shipdate between "2017-04-16 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 898 AND  l_shipdate between "2017-04-17 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 876 AND  l_shipdate between "2017-04-18 12:00:00" AND  "2017-04-18 02:22:02") OR
+       (l_orderkey = 743 AND  l_shipdate between "2017-04-19 12:00:00" AND  "2017-06-18 02:22:02") OR
+       (l_orderkey = 900 AND  l_shipdate between "2017-04-20 12:00:00" AND  "2017-06-18 02:22:02"));
+---- RESULTS
+---- TYPES
+====


[2/3] incubator-impala git commit: IMPALA-5400: Execute tests in subplans.test

Posted by he...@apache.org.
IMPALA-5400: Execute tests in subplans.test

This change executes the tests added to subplans.test and removes
a test which incorrectly references subplannull_data.test (a file
which does not exist)

Change-Id: I02b4f47553fb8f5fe3425cde2e0bcb3245c39b91
Reviewed-on: http://gerrit.cloudera.org:8080/7038
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/70657a86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/70657a86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/70657a86

Branch: refs/heads/master
Commit: 70657a860a2f08e43d6eb5d3845749d293ae08ce
Parents: 9123705
Author: anujphadke <ap...@cloudera.com>
Authored: Wed May 31 21:03:01 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Jun 8 02:02:00 2017 +0000

----------------------------------------------------------------------
 .../queries/QueryTest/nested-types-subplan.test |  37 ++-
 .../queries/QueryTest/subplans.test             | 230 -------------------
 tests/query_test/test_queries.py                |   6 -
 3 files changed, 34 insertions(+), 239 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/70657a86/testdata/workloads/functional-query/queries/QueryTest/nested-types-subplan.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-subplan.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-subplan.test
index 45116e6..e18f25e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-subplan.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-subplan.test
@@ -253,6 +253,23 @@ where c_custkey < 4
 bigint,string,bigint,decimal,decimal,string,string
 ====
 ---- QUERY
+# Test a left outer join inside a subplan.
+select count(okey), opriority
+from tpch_nested_parquet.customer c,
+  (select ca.o_orderkey okey, ca.o_orderpriority opriority
+   from c.c_orders ca left outer join c.c_orders cb
+   on ca.o_orderkey = cb.o_orderkey) v
+group by opriority
+---- TYPES
+bigint, string
+---- RESULTS: VERIFY_IS_EQUAL_SORTED
+300091,'2-HIGH'
+300254,'4-NOT SPECIFIED'
+300589,'5-LOW'
+300343,'1-URGENT'
+298723,'3-MEDIUM'
+====
+---- QUERY
 # Test left outer join of a relative table ref.
 select c_custkey, c_mktsegment, o_orderkey, o_orderdate
 from customer c left outer join c.c_orders
@@ -300,6 +317,20 @@ where c_custkey < 4
 bigint,string,bigint,string
 ====
 ---- QUERY
+# Test a right outer join inside a subplan.
+select count(okey), opriority
+from tpch_nested_parquet.customer c,
+  (select ca.o_orderkey okey, ca.o_orderpriority opriority
+   from c.c_orders ca right outer join c.c_orders cb
+   on ca.o_orderkey = cb.o_orderkey
+   where ca.o_totalprice + cb.o_totalprice < 2
+   and cb.o_orderpriority = '5-LOW') v
+group by opriority
+---- TYPES
+bigint, string
+---- RESULTS:
+====
+---- QUERY
 # Test left semi join of a relative table ref.
 select c_custkey, c_mktsegment
 from customer c left semi join c.c_orders
@@ -574,7 +605,7 @@ where c_custkey = 1
 1,4808192
 1,5133509
 ---- TYPES
-BIGINT,BIGINT
+bigint, bigint
 ====
 ---- QUERY
 # IMPALA-3678: Union in a subplan - passthrough should be disabled.
@@ -598,7 +629,7 @@ WHERE ca.o_orderkey = cb.o_orderkey limit 2) v limit 51
 ---- RESULTS
 1500000
 ---- TYPES
-BIGINT
+bigint
 ====
 ---- QUERY
 # IMPALA-5363: Reset probe_batch_ after reaching limit.
@@ -609,7 +640,7 @@ WHERE ca.o_orderkey < cb.o_orderkey limit 2) v limit 51
 ---- RESULTS
 199835
 ---- TYPES
-BIGINT
+bigint
 ====
 ---- QUERY
 # IMPALA-5438: Union with constant exprs in a subplan. The 'c_custkey % 100' was chosen

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/70657a86/testdata/workloads/functional-query/queries/QueryTest/subplans.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/subplans.test b/testdata/workloads/functional-query/queries/QueryTest/subplans.test
deleted file mode 100644
index 1cf9047..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/subplans.test
+++ /dev/null
@@ -1,230 +0,0 @@
-====
----- QUERY
-# Test a union inside a subplan with some constant operands.
-select c_custkey, o_orderkey from tpch_nested_parquet.customer c,
-  (select o_orderkey from c.c_orders
-   union all
-   values(100), (200), (300)) v
-where c_custkey in (1, 2, 3)
----- TYPES
-BIGINT, BIGINT
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-1,100
-1,200
-1,300
-1,454791
-1,579908
-1,3868359
-1,4273923
-1,4808192
-1,5133509
-2,100
-2,200
-2,300
-2,430243
-2,1071617
-2,1374019
-2,1763205
-2,1842406
-2,2992930
-2,3986496
-3,100
-3,200
-3,300
-====
----- QUERY
-# Test an order by + limit (topn node) inside a subplan.
-select c_custkey, o_orderkey from tpch_nested_parquet.customer c,
-  (select o_orderkey from c.c_orders
-   order by o_orderkey desc limit 2) v
-where c_custkey in (1, 2, 3)
----- TYPES
-BIGINT, BIGINT
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-1,5133509
-1,4808192
-2,3986496
-2,2992930
-====
----- QUERY
-# Test a select node inside a subplan.
-select c_custkey, o_orderkey from tpch_nested_parquet.customer c,
-  (select o_orderkey from c.c_orders
-   order by o_orderkey desc limit 2) v
-where c_custkey in (1, 2, 3) and o_orderkey % 2 = 0
----- TYPES
-BIGINT, BIGINT
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-1,4808192
-2,3986496
-2,2992930
-====
----- QUERY
-# Test an analytic function that requires a sort inside a subplan.
-select c_custkey, o_orderstatus, o_orderdate, o_orderkey, r from tpch_nested_parquet.customer c,
-  (select o_orderstatus, o_orderdate, o_orderkey,
-   row_number() over (partition by o_orderstatus order by o_orderdate) r
-   from c.c_orders) v
-where c_custkey in (1, 2, 3)
----- TYPES
-BIGINT, STRING, BIGINT, BIGINT
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-1,F,1992-04-19,454791,1
-1,F,1992-08-22,3868359,2
-1,O,1996-06-29,4808192,1
-1,O,1996-07-01,5133509,2
-1,O,1996-12-09,579908,3
-1,O,1997-03-23,4273923,4
-2,F,1992-04-05,1374019,1
-2,F,1994-05-21,2992930,2
-2,F,1994-08-28,1763205,3
-2,F,1994-12-24,430243,4
-2,O,1996-08-05,1842406,1
-2,O,1997-02-22,3986496,2
-2,P,1995-03-10,1071617,1
-====
----- QUERY
-# Test an analytic function that does not require a sort inside a subplan.
-select c_custkey, mp from tpch_nested_parquet.customer c,
-  (select max(o_totalprice) over () mp
-   from c.c_orders) v
-where c_custkey in (1, 2, 3)
----- TYPES
-BIGINT, DECIMAL
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-1,174645.94
-1,174645.94
-1,174645.94
-1,174645.94
-1,174645.94
-1,174645.94
-2,312692.22
-2,312692.22
-2,312692.22
-2,312692.22
-2,312692.22
-2,312692.22
-2,312692.22
-====
----- QUERY
-# Test a non-grouping aggregation inside a subplan.
-select c_custkey, cnt, avp
-from tpch_nested_parquet.customer c,
-  (select count(o_orderkey) cnt,
-          avg(o_totalprice) avp
-   from c.c_orders) v
-where c_custkey < 3
----- TYPES
-BIGINT, BIGINT, DECIMAL
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-1,6,97960.48
-2,7,146896.20
-====
----- QUERY
-SELECT c_custkey, avg(maxp)
-FROM tpch_nested_parquet.customer c,
-  (SELECT MAX(o_totalprice) maxp
-   FROM c.c_orders GROUP BY o_orderpriority) v
-WHERE c_custkey < 3
-GROUP BY c_custkey
----- TYPES
-BIGINT, DECIMAL
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-1,114777.96
-2,212462.41
-====
----- QUERY
-# Test a grouping aggregation inside a subplan.
-select c_custkey, o_orderpriority, cnt, avp
-from tpch_nested_parquet.customer c,
-  (select count(o_orderkey) cnt,
-          avg(o_totalprice) avp,
-          o_orderpriority
-   from c.c_orders
-   group by o_orderpriority) v
-where c_custkey < 3
----- TYPES
-BIGINT, STRING, BIGINT, DECIMAL
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-1,1-URGENT,2,124624.37
-1,2-HIGH,1,65478.05
-1,3-MEDIUM,1,95911.01
-1,5-LOW,2,88562.55
-2,1-URGENT,4,167623.89
-2,2-HIGH,1,221397.35
-2,4-NOT SPECIFIED,2,68190.25
-====
----- QUERY
-# Test a join inside a subplan.
-SELECT count(okey), opriority
-FROM tpch_nested_parquet.customer c,
-  (SELECT ca.o_orderkey okey, ca.o_orderpriority opriority
-   FROM c.c_orders ca, c.c_orders cb
-   WHERE ca.o_orderkey = cb.o_orderkey
-   AND ca.o_totalprice + cb.o_totalprice < 2) v
-GROUP BY opriority
----- TYPES
-BIGINT, STRING
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-====
----- QUERY
-# Test a self-join inside a subplan.
-SELECT count(okey), opriority
-FROM tpch_nested_parquet.customer c,
-  (SELECT ca.o_orderkey okey, ca.o_orderpriority opriority
-   FROM c.c_orders ca, c.c_orders cb
-   WHERE ca.o_orderkey = cb.o_orderkey) v
-GROUP BY opriority
----- TYPES
-BIGINT, STRING
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-300091,2-HIGH
-300254,4-NOT SPECIFIED
-300589,5-LOW
-300343,1-URGENT
-298723,3-MEDIUM
-====
----- QUERY
-# Test a left outer join inside a subplan.
-SELECT count(okey), opriority
-FROM tpch_nested_parquet.customer c,
-  (SELECT ca.o_orderkey okey, ca.o_orderpriority opriority
-   FROM c.c_orders ca LEFT OUTER JOIN c.c_orders cb
-   ON ca.o_orderkey = cb.o_orderkey) v
-GROUP BY opriority
----- TYPES
-BIGINT, STRING
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-300091,2-HIGH
-300254,4-NOT SPECIFIED
-300589,5-LOW
-300343,1-URGENT
-298723,3-MEDIUM
-====
----- QUERY
-# Test a right outer join inside a subplan.
-SELECT count(okey), opriority
-FROM tpch_nested_parquet.customer c,
-  (SELECT ca.o_orderkey okey, ca.o_orderpriority opriority
-   FROM c.c_orders ca RIGHT OUTER JOIN c.c_orders cb
-   ON ca.o_orderkey = cb.o_orderkey
-   WHERE ca.o_totalprice + cb.o_totalprice < 2
-   AND cb.o_orderpriority = '5-LOW') v
-GROUP BY opriority
----- TYPES
-BIGINT, STRING
----- RESULTS: VERIFY_IS_EQUAL_SORTED
-====
----- QUERY
-# IMPALA-2368: Test nested subplans with a non-trivial plan tree.
-SELECT count(*)
-FROM tpch_nested_parquet.customer c
-INNER JOIN c.c_orders o
-WHERE c_custkey < 10 AND o.pos IN
-  (SELECT LEAD(l.l_linenumber) OVER (ORDER BY l.pos)
-   FROM o.o_lineitems l)
----- TYPES
-BIGINT
----- RESULTS
-14
-====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/70657a86/tests/query_test/test_queries.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_queries.py b/tests/query_test/test_queries.py
index 1099330..40ca7f3 100644
--- a/tests/query_test/test_queries.py
+++ b/tests/query_test/test_queries.py
@@ -109,12 +109,6 @@ class TestQueries(ImpalaTestSuite):
   def test_subquery(self, vector):
     self.run_test_case('QueryTest/subquery', vector)
 
-  def test_subplans(self, vector):
-    pytest.xfail("Disabled due to missing nested types functionality.")
-    if vector.get_value('table_format').file_format != 'parquet':
-      pytest.xfail("Nested TPCH only available in parquet.")
-    self.run_test_case('QueryTest/subplannull_data', vector)
-
   def test_empty(self, vector):
     self.run_test_case('QueryTest/empty', vector)
 


[3/3] incubator-impala git commit: IMPALA-5056: Ensure analysis uses 'fresh' catalog after metadata loading

Posted by he...@apache.org.
IMPALA-5056: Ensure analysis uses 'fresh' catalog after metadata loading

If a query was waiting for metadata to be loaded at the same time as the
statestore restarted, it would not correctly see the metadata even after
the statestore had recovered and sent it.

This is because the AnalysisContext used for analysis is created with a
reference to a ImpaladCatalog once, before the analysis->metadata-load
loop started. If the catalog sent a *complete* update (which it would do
after a statestore restart), the ImpaladCatalog would be replaced, not
updated, and so the tables would exist in the new object, but not the
old one that the AnalysisContext had a reference to.

The fix is to update the AnalysisContext's catalog every time through
the loop, taking a recent reference to the ImpaladCatalog every
time. That way, if the ImpaladCatalog is updated, the next time through
the loop the analysis phase will see those changes.

Also fix a potential bug where access to ImpaladCatalog is not
synchronized between update and analysis threads, by using an
AtomicReference<ImpaladCatalog>.

Testing: manual - queries completed after statestore restarts with this
patch, where previously they consumed 100% CPU spinning forever.

Change-Id: I574d69fa75198499523dc291fbbd0d7e3d8d968f
Reviewed-on: http://gerrit.cloudera.org:8080/7045
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/54118010
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/54118010
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/54118010

Branch: refs/heads/master
Commit: 54118010590d8605303715bf570cac18e1d5e64e
Parents: 70657a8
Author: Henry Robinson <he...@cloudera.com>
Authored: Thu Jun 1 13:51:25 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Jun 8 03:40:56 2017 +0000

----------------------------------------------------------------------
 .../apache/impala/analysis/AnalysisContext.java | 12 +++--
 .../org/apache/impala/service/Frontend.java     | 54 ++++++++++----------
 2 files changed, 37 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/54118010/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java b/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
index c6ee814..34ac107 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
@@ -52,7 +52,7 @@ import com.google.common.collect.Maps;
  */
 public class AnalysisContext {
   private final static Logger LOG = LoggerFactory.getLogger(AnalysisContext.class);
-  private final ImpaladCatalog catalog_;
+  private ImpaladCatalog catalog_;
   private final TQueryCtx queryCtx_;
   private final AuthorizationConfig authzConfig_;
   private final ExprRewriter customRewriter_;
@@ -66,7 +66,7 @@ public class AnalysisContext {
 
   public AnalysisContext(ImpaladCatalog catalog, TQueryCtx queryCtx,
       AuthorizationConfig authzConfig) {
-    catalog_ = catalog;
+    setCatalog(catalog);
     queryCtx_ = queryCtx;
     authzConfig_ = authzConfig;
     customRewriter_ = null;
@@ -77,12 +77,18 @@ public class AnalysisContext {
    */
   protected AnalysisContext(ImpaladCatalog catalog, TQueryCtx queryCtx,
       AuthorizationConfig authzConfig, ExprRewriter rewriter) {
-    catalog_ = catalog;
+    setCatalog(catalog);
     queryCtx_ = queryCtx;
     authzConfig_ = authzConfig;
     customRewriter_ = rewriter;
   }
 
+  // Catalog may change between analysis attempts (e.g. when missing tables are loaded).
+  public void setCatalog(ImpaladCatalog catalog) {
+    Preconditions.checkNotNull(catalog);
+    catalog_ = catalog;
+  }
+
   static public class AnalysisResult {
     private StatementBase stmt_;
     private Analyzer analyzer_;

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/54118010/fe/src/main/java/org/apache/impala/service/Frontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 0345260..3f4cc8a 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -160,7 +160,8 @@ public class Frontend {
   //TODO: Make the reload interval configurable.
   private static final int AUTHORIZATION_POLICY_RELOAD_INTERVAL_SECS = 5 * 60;
 
-  private ImpaladCatalog impaladCatalog_;
+  private AtomicReference<ImpaladCatalog> impaladCatalog_ =
+      new AtomicReference<ImpaladCatalog>();
   private final AuthorizationConfig authzConfig_;
   private final AtomicReference<AuthorizationChecker> authzChecker_;
   private final ScheduledExecutorService policyReader_ =
@@ -177,10 +178,10 @@ public class Frontend {
    */
   public Frontend(AuthorizationConfig authorizationConfig, ImpaladCatalog catalog) {
     authzConfig_ = authorizationConfig;
-    impaladCatalog_ = catalog;
+    impaladCatalog_.set(catalog);
     defaultKuduMasterHosts_ = catalog.getDefaultKuduMasterHosts();
     authzChecker_ = new AtomicReference<AuthorizationChecker>(
-        new AuthorizationChecker(authzConfig_, impaladCatalog_.getAuthPolicy()));
+        new AuthorizationChecker(authzConfig_, impaladCatalog_.get().getAuthPolicy()));
     // If authorization is enabled, reload the policy on a regular basis.
     if (authzConfig_.isEnabled() && authzConfig_.isFileBasedPolicy()) {
       // Stagger the reads across nodes
@@ -215,27 +216,25 @@ public class Frontend {
     }
   }
 
-  public ImpaladCatalog getCatalog() { return impaladCatalog_; }
+  public ImpaladCatalog getCatalog() { return impaladCatalog_.get(); }
   public AuthorizationChecker getAuthzChecker() { return authzChecker_.get(); }
 
   public TUpdateCatalogCacheResponse updateCatalogCache(
       TUpdateCatalogCacheRequest req) throws CatalogException {
-    ImpaladCatalog catalog = impaladCatalog_;
-
-    if (req.is_delta) return catalog.updateCatalog(req);
+    if (req.is_delta) return impaladCatalog_.get().updateCatalog(req);
 
     // If this is not a delta, this update should replace the current
     // Catalog contents so create a new catalog and populate it.
-    catalog = new ImpaladCatalog(defaultKuduMasterHosts_);
+    ImpaladCatalog catalog = new ImpaladCatalog(defaultKuduMasterHosts_);
 
     TUpdateCatalogCacheResponse response = catalog.updateCatalog(req);
 
     // Now that the catalog has been updated, replace the references to
-    // impaladCatalog_/authzChecker_. This ensures that clients don't see
-    // the catalog disappear.
-    impaladCatalog_ = catalog;
-    authzChecker_.set(new AuthorizationChecker(authzConfig_,
-        impaladCatalog_.getAuthPolicy()));
+    // impaladCatalog_/authzChecker_. This ensures that clients don't see the catalog
+    // disappear. The catalog is guaranteed to be ready since updateCatalog() has a
+    // postcondition of isReady() == true.
+    impaladCatalog_.set(catalog);
+    authzChecker_.set(new AuthorizationChecker(authzConfig_, catalog.getAuthPolicy()));
     return response;
   }
 
@@ -547,11 +546,12 @@ public class Frontend {
     // Get the destination for the load. If the load is targeting a partition,
     // this the partition location. Otherwise this is the table location.
     String destPathString = null;
+    ImpaladCatalog catalog = impaladCatalog_.get();
     if (request.isSetPartition_spec()) {
-      destPathString = impaladCatalog_.getHdfsPartition(tableName.getDb(),
+      destPathString = catalog.getHdfsPartition(tableName.getDb(),
           tableName.getTbl(), request.getPartition_spec()).getLocation();
     } else {
-      destPathString = impaladCatalog_.getTable(tableName.getDb(), tableName.getTbl())
+      destPathString = catalog.getTable(tableName.getDb(), tableName.getTbl())
           .getMetaStoreTable().getSd().getLocation();
     }
 
@@ -607,7 +607,7 @@ public class Frontend {
    */
   public List<String> getTableNames(String dbName, PatternMatcher matcher,
       User user) throws ImpalaException {
-    List<String> tblNames = impaladCatalog_.getTableNames(dbName, matcher);
+    List<String> tblNames = impaladCatalog_.get().getTableNames(dbName, matcher);
     if (authzConfig_.isEnabled()) {
       Iterator<String> iter = tblNames.iterator();
       while (iter.hasNext()) {
@@ -651,7 +651,7 @@ public class Frontend {
    */
   public List<Db> getDbs(PatternMatcher matcher, User user)
       throws InternalException {
-    List<Db> dbs = impaladCatalog_.getDbs(matcher);
+    List<Db> dbs = impaladCatalog_.get().getDbs(matcher);
     // If authorization is enabled, filter out the databases the user does not
     // have permissions on.
     if (authzConfig_.isEnabled()) {
@@ -683,7 +683,7 @@ public class Frontend {
    * matches all data sources.
    */
   public List<DataSource> getDataSrcs(String pattern) {
-    return impaladCatalog_.getDataSources(
+    return impaladCatalog_.get().getDataSources(
         PatternMatcher.createHivePatternMatcher(pattern));
   }
 
@@ -692,7 +692,7 @@ public class Frontend {
    */
   public TResultSet getColumnStats(String dbName, String tableName)
       throws ImpalaException {
-    Table table = impaladCatalog_.getTable(dbName, tableName);
+    Table table = impaladCatalog_.get().getTable(dbName, tableName);
     TResultSet result = new TResultSet();
     TResultSetMetadata resultSchema = new TResultSetMetadata();
     result.setSchema(resultSchema);
@@ -720,7 +720,7 @@ public class Frontend {
    */
   public TResultSet getTableStats(String dbName, String tableName, TShowStatsOp op)
       throws ImpalaException {
-    Table table = impaladCatalog_.getTable(dbName, tableName);
+    Table table = impaladCatalog_.get().getTable(dbName, tableName);
     if (table instanceof HdfsTable) {
       return ((HdfsTable) table).getTableStats();
     } else if (table instanceof HBaseTable) {
@@ -746,7 +746,7 @@ public class Frontend {
   public List<Function> getFunctions(TFunctionCategory category,
       String dbName, String fnPattern, boolean exactMatch)
       throws DatabaseNotFoundException {
-    Db db = impaladCatalog_.getDb(dbName);
+    Db db = impaladCatalog_.get().getDb(dbName);
     if (db == null) {
       throw new DatabaseNotFoundException("Database '" + dbName + "' not found");
     }
@@ -774,7 +774,7 @@ public class Frontend {
    */
   public TDescribeResult describeDb(String dbName, TDescribeOutputStyle outputStyle)
       throws ImpalaException {
-    Db db = impaladCatalog_.getDb(dbName);
+    Db db = impaladCatalog_.get().getDb(dbName);
     return DescribeResultFactory.buildDescribeDbResult(db, outputStyle);
   }
 
@@ -785,7 +785,7 @@ public class Frontend {
    */
   public TDescribeResult describeTable(TTableName tableName,
       TDescribeOutputStyle outputStyle) throws ImpalaException {
-    Table table = impaladCatalog_.getTable(tableName.db_name, tableName.table_name);
+    Table table = impaladCatalog_.get().getTable(tableName.db_name, tableName.table_name);
     if (outputStyle == TDescribeOutputStyle.MINIMAL) {
       return DescribeResultFactory.buildDescribeMinimalResult(table);
     } else {
@@ -882,12 +882,12 @@ public class Frontend {
    */
   private AnalysisContext.AnalysisResult analyzeStmt(TQueryCtx queryCtx)
       throws AnalysisException, InternalException, AuthorizationException {
-    if (!impaladCatalog_.isReady()) {
+    if (!impaladCatalog_.get().isReady()) {
       throw new AnalysisException("This Impala daemon is not ready to accept user " +
           "requests. Status: Waiting for catalog update from the StateStore.");
     }
 
-    AnalysisContext analysisCtx = new AnalysisContext(impaladCatalog_, queryCtx,
+    AnalysisContext analysisCtx = new AnalysisContext(impaladCatalog_.get(), queryCtx,
         authzConfig_);
     LOG.info("Compiling query: " + queryCtx.client_request.stmt);
 
@@ -897,6 +897,8 @@ public class Frontend {
     // 3) Analysis fails with an AuthorizationException.
     try {
       while (true) {
+        // Ensure that catalog snapshot reflects any recent changes.
+        analysisCtx.setCatalog(impaladCatalog_.get());
         try {
           analysisCtx.analyze(queryCtx.client_request.stmt);
           Preconditions.checkState(analysisCtx.getAnalyzer().getMissingTbls().isEmpty());
@@ -1214,7 +1216,7 @@ public class Frontend {
    */
   public TResultSet getTableFiles(TShowFilesParams request)
       throws ImpalaException{
-    Table table = impaladCatalog_.getTable(request.getTable_name().getDb_name(),
+    Table table = impaladCatalog_.get().getTable(request.getTable_name().getDb_name(),
         request.getTable_name().getTable_name());
     if (table instanceof HdfsTable) {
       return ((HdfsTable) table).getFiles(request.getPartition_set());