You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/02/07 23:05:46 UTC

[impala] branch master updated (2fad3d0 -> 06d078e)

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.


    from 2fad3d0  IMPALA-8171: minicluster: use exec builtin to start Impala daemons
     new 7707eb0  IMPALA-7657: Codegen IsNotEmptyPredicate and ValidTupleIdExpr.
     new b1e4957  IMPALA-4555: Make QueryState's status reporting more robust
     new d7a0690  IMPALA-8044: [DOCS] Document REFRESH AUTHORIZATION statement
     new 06d078e  IMPALA-7929: Allow null qualifier in THBaseFilter

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 be/src/codegen/gen_ir_descriptions.py              |   4 +
 be/src/codegen/impala-ir.cc                        |   4 +-
 be/src/codegen/llvm-codegen.h                      |   3 +-
 be/src/common/global-flags.cc                      |   1 +
 be/src/exprs/CMakeLists.txt                        |   2 +
 be/src/exprs/is-not-empty-predicate.cc             | 156 ++++++++++++++++++++-
 be/src/exprs/is-not-empty-predicate.h              |   8 +-
 .../summary-util.h => exprs/null-literal-ir.cc}    |  15 +-
 be/src/exprs/null-literal.cc                       |   8 +-
 be/src/exprs/null-literal.h                        |   2 +
 be/src/exprs/scalar-expr.h                         |   1 -
 .../{runtime/tuple-row.cc => exprs/slot-ref-ir.cc} |  16 ++-
 be/src/exprs/slot-ref.cc                           |  16 +--
 be/src/exprs/slot-ref.h                            |   1 +
 be/src/exprs/valid-tuple-id.cc                     | 123 +++++++++++++++-
 be/src/exprs/valid-tuple-id.h                      |  11 +-
 be/src/runtime/coordinator-backend-state.cc        |  23 +--
 be/src/runtime/fragment-instance-state.cc          |  45 +++++-
 be/src/runtime/fragment-instance-state.h           |  20 ++-
 be/src/runtime/query-state.cc                      | 118 +++++++++-------
 be/src/runtime/query-state.h                       |  14 +-
 common/protobuf/control_service.proto              |  21 ++-
 common/thrift/PlanNodes.thrift                     |   3 +-
 docs/impala.ditamap                                |   1 +
 docs/shared/impala_common.xml                      |   5 +
 ..._files.xml => impala_refresh_authorization.xml} |  36 +++--
 .../org/apache/impala/planner/HBaseScanNode.java   |  11 +-
 .../queries/QueryTest/hbase-col-filter.test        |  25 ++++
 tests/custom_cluster/test_rpc_timeout.py           |  17 +++
 tests/query_test/test_hbase_queries.py             |  35 +++++
 30 files changed, 610 insertions(+), 135 deletions(-)
 copy be/src/{util/summary-util.h => exprs/null-literal-ir.cc} (77%)
 copy be/src/{runtime/tuple-row.cc => exprs/slot-ref-ir.cc} (60%)
 copy docs/topics/{impala_allow_erasure_coded_files.xml => impala_refresh_authorization.xml} (51%)
 create mode 100644 testdata/workloads/functional-query/queries/QueryTest/hbase-col-filter.test


[impala] 03/04: IMPALA-8044: [DOCS] Document REFRESH AUTHORIZATION statement

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit d7a0690142c5d965389a5a62b64a649d2619ffa2
Author: Alex Rodoni <ar...@cloudera.com>
AuthorDate: Tue Feb 5 16:50:29 2019 -0800

    IMPALA-8044: [DOCS] Document REFRESH AUTHORIZATION statement
    
    Change-Id: Ibbb1661affc01c72db8f76bd97d356c802ed9680
    Reviewed-on: http://gerrit.cloudera.org:8080/12378
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
    Reviewed-by: Fredy Wijaya <fw...@cloudera.com>
---
 docs/impala.ditamap                          |  1 +
 docs/shared/impala_common.xml                |  5 ++
 docs/topics/impala_refresh_authorization.xml | 75 ++++++++++++++++++++++++++++
 3 files changed, 81 insertions(+)

diff --git a/docs/impala.ditamap b/docs/impala.ditamap
index bf14702..ad6b89f 100644
--- a/docs/impala.ditamap
+++ b/docs/impala.ditamap
@@ -152,6 +152,7 @@ under the License.
       <topicref href="topics/impala_invalidate_metadata.xml"/>
       <topicref href="topics/impala_load_data.xml"/>
       <topicref href="topics/impala_refresh.xml"/>
+      <topicref href="topics/impala_refresh_authorization.xml"/>
       <topicref href="topics/impala_refresh_functions.xml"/>
       <topicref href="topics/impala_revoke.xml"/>
       <topicref href="topics/impala_select.xml">
diff --git a/docs/shared/impala_common.xml b/docs/shared/impala_common.xml
index bfe8a24..3c11f68 100644
--- a/docs/shared/impala_common.xml
+++ b/docs/shared/impala_common.xml
@@ -383,6 +383,11 @@ under the License.
                 <entry>TABLE</entry>
               </row>
               <row>
+                <entry>REFRESH AUTHORIZATION</entry>
+                <entry>REFRESH</entry>
+                <entry>SERVER</entry>
+              </row>
+              <row>
                 <entry>REFRESH FUNCTIONS</entry>
                 <entry>REFRESH</entry>
                 <entry>DATABASE</entry>
diff --git a/docs/topics/impala_refresh_authorization.xml b/docs/topics/impala_refresh_authorization.xml
new file mode 100644
index 0000000..cac4cd3
--- /dev/null
+++ b/docs/topics/impala_refresh_authorization.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
+<concept id="refresh_authorization">
+
+  <title>REFRESH AUTHORIZATION Statement</title>
+
+  <titlealts audience="PDF">
+
+    <navtitle>REFRESH AUTHORIZATION</navtitle>
+
+  </titlealts>
+
+  <prolog>
+    <metadata>
+      <data name="Category" value="Impala"/>
+      <data name="Category" value="SQL"/>
+      <data name="Category" value="DDL"/>
+      <data name="Category" value="Security"/>
+    </metadata>
+  </prolog>
+
+  <conbody>
+
+    <p>
+      The <codeph>REFRESH AUTHORIZATION</codeph> statement explicitly refreshes authorization
+      data, including privileges and principals. When there is an external update to
+      authorization metadata, use this statement to force Impala to refresh its authorization
+      data without having to wait for the Sentry polling or run <codeph>INVALIDATE
+      METADATA</codeph>.
+    </p>
+
+    <p>
+      Once issued, the <codeph>REFRESH AUTHORIZATION</codeph> statement cannot be cancelled.
+    </p>
+
+    <p>
+      See <xref href="impala_authorization.xml#authorization">Impala Authorization</xref> for
+      information on enabling and using authorization in Impala.
+    </p>
+
+    <p>
+      <b>Syntax:</b>
+    </p>
+
+<codeblock>REFRESH AUTHORIZATION</codeblock>
+
+    <p>
+      <b>Usage notes:</b> If authorization is not enabled, Impala returns an error.
+    </p>
+
+    <p>
+      <b>Added in:</b> Impala 3.2
+    </p>
+
+  </conbody>
+
+</concept>


[impala] 02/04: IMPALA-4555: Make QueryState's status reporting more robust

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit b1e4957ba78ef496d21728606889d1eb83ef6b27
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
AuthorDate: Tue Dec 4 23:53:38 2018 +0000

    IMPALA-4555: Make QueryState's status reporting more robust
    
    QueryState periodically collects runtime profiles from all of its
    fragment instances and sends them to the coordinator. Previously, each
    time this happens, if the rpc fails, QueryState will retry twice after
    a configurable timeout and then cancel the fragment instances under
    the assumption that the coordinator no longer exists.
    
    We've found in real clusters that this logic is too sensitive to
    failed rpcs and can result in fragment instances being cancelled even
    in cases where the coordinator is still running.
    
    This patch makes a few improvements to this logic:
    - When a report fails to send, instead of retrying the same report
      quickly (after waiting report_status_retry_interval_ms), we wait the
      regular reporting interval (status_report_interval_ms), regenerate
      any stale portions of the report, and then retry.
    - A new flag, --status_report_max_retries, is introduced, which
      controls the number of failed reports that are allowed before the
      query is cancelled. --report_status_retry_interval_ms is removed.
    - Backoff is used for repeated failed attempts, such that for a period
      between retries of 't', on try 'n' the actual timeout will be t * n.
    
    Testing:
    - Added a test which results in a large number of failed intermediate
      status reports but still succeeds.
    
    Change-Id: Ib6007013fc2c9e8eeba11b752ee58fb3038da971
    Reviewed-on: http://gerrit.cloudera.org:8080/12049
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/common/global-flags.cc               |   1 +
 be/src/runtime/coordinator-backend-state.cc |  23 ++++--
 be/src/runtime/fragment-instance-state.cc   |  45 ++++++++++-
 be/src/runtime/fragment-instance-state.h    |  20 ++++-
 be/src/runtime/query-state.cc               | 118 +++++++++++++++++-----------
 be/src/runtime/query-state.h                |  14 +++-
 common/protobuf/control_service.proto       |  21 ++++-
 tests/custom_cluster/test_rpc_timeout.py    |  17 ++++
 8 files changed, 193 insertions(+), 66 deletions(-)

diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 6620bdc..470a111 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -313,6 +313,7 @@ REMOVED_FLAG(llama_registration_timeout_secs);
 REMOVED_FLAG(llama_registration_wait_secs);
 REMOVED_FLAG(local_nodemanager_url);
 REMOVED_FLAG(max_free_io_buffers);
+REMOVED_FLAG(report_status_retry_interval_ms);
 REMOVED_FLAG(resource_broker_cnxn_attempts);
 REMOVED_FLAG(resource_broker_cnxn_retry_interval_ms);
 REMOVED_FLAG(resource_broker_recv_timeout);
diff --git a/be/src/runtime/coordinator-backend-state.cc b/be/src/runtime/coordinator-backend-state.cc
index ba4f355..2bc8547 100644
--- a/be/src/runtime/coordinator-backend-state.cc
+++ b/be/src/runtime/coordinator-backend-state.cc
@@ -319,10 +319,11 @@ bool Coordinator::BackendState::ApplyExecStatusReport(
     int instance_idx = GetInstanceIdx(instance_exec_status.fragment_instance_id());
     DCHECK_EQ(instance_stats_map_.count(instance_idx), 1);
     InstanceStats* instance_stats = instance_stats_map_[instance_idx];
+    int64_t last_report_seq_no = instance_stats->last_report_seq_no_;
     DCHECK(instance_stats->exec_params_.instance_id ==
         ProtoToQueryId(instance_exec_status.fragment_instance_id()));
     // Ignore duplicate or out-of-order messages.
-    if (report_seq_no <= instance_stats->last_report_seq_no_) {
+    if (report_seq_no <= last_report_seq_no) {
       VLOG_QUERY << Substitute("Ignoring stale update for query instance $0 with "
           "seq no $1", PrintId(instance_stats->exec_params_.instance_id), report_seq_no);
       continue;
@@ -340,13 +341,19 @@ bool Coordinator::BackendState::ApplyExecStatusReport(
       dml_exec_state->Update(instance_exec_status.dml_exec_status());
     }
 
-    // Log messages aggregated by type
-    if (instance_exec_status.error_log_size() > 0) {
-      // Append the log messages from each update with the global state of the query
-      // execution
-      MergeErrorMaps(instance_exec_status.error_log(), &error_log_);
-      VLOG_FILE << "host=" << TNetworkAddressToString(host_) << " error log: " <<
-          PrintErrorMapToString(error_log_);
+    // Handle the non-idempotent parts of the report for any sequence numbers that we
+    // haven't seen yet.
+    if (instance_exec_status.stateful_report_size() > 0) {
+      for (const auto& stateful_report : instance_exec_status.stateful_report()) {
+        DCHECK_LE(stateful_report.report_seq_no(), report_seq_no);
+        if (last_report_seq_no < stateful_report.report_seq_no()) {
+          // Append the log messages from each update with the global state of the query
+          // execution
+          MergeErrorMaps(stateful_report.error_log(), &error_log_);
+          VLOG_FILE << "host=" << TNetworkAddressToString(host_)
+                    << " error log: " << PrintErrorMapToString(error_log_);
+        }
+      }
     }
 
     DCHECK_GT(num_remaining_instances_, 0);
diff --git a/be/src/runtime/fragment-instance-state.cc b/be/src/runtime/fragment-instance-state.cc
index c5799ee..749e135 100644
--- a/be/src/runtime/fragment-instance-state.cc
+++ b/be/src/runtime/fragment-instance-state.cc
@@ -255,7 +255,13 @@ void FragmentInstanceState::GetStatusReport(FragmentInstanceExecStatusPB* instan
   if (per_host_mem_usage_ != nullptr) {
     per_host_mem_usage_->Set(runtime_state()->query_mem_tracker()->peak_consumption());
   }
-  instance_status->set_report_seq_no(AdvanceReportSeqNo());
+  if (final_report_generated_) {
+    // Since execution was already finished, the contents of this report will be identical
+    // to the last report, so don't advance the sequence number.
+    instance_status->set_report_seq_no(report_seq_no_);
+  } else {
+    instance_status->set_report_seq_no(AdvanceReportSeqNo());
+  }
   const TUniqueId& finstance_id = instance_id();
   TUniqueIdToUniqueIdPB(finstance_id, instance_status->mutable_fragment_instance_id());
   const bool done = IsDone();
@@ -267,10 +273,41 @@ void FragmentInstanceState::GetStatusReport(FragmentInstanceExecStatusPB* instan
   if (done) {
     runtime_state()->dml_exec_state()->ToProto(
         instance_status->mutable_dml_exec_status());
-    final_report_sent_ = true;
+    final_report_generated_ = true;
+  }
+  if (prev_stateful_reports_.size() > 0) {
+    // Send errors from previous reports that failed.
+    *instance_status->mutable_stateful_report() =
+        {prev_stateful_reports_.begin(), prev_stateful_reports_.end()};
+  }
+  if (runtime_state()->HasErrors()) {
+    // Add any new errors.
+    StatefulStatusPB* stateful_report = instance_status->add_stateful_report();
+    stateful_report->set_report_seq_no(report_seq_no_);
+    runtime_state()->GetUnreportedErrors(stateful_report->mutable_error_log());
+  }
+}
+
+void FragmentInstanceState::ReportSuccessful(
+    const FragmentInstanceExecStatusPB& instance_exec_status) {
+  prev_stateful_reports_.clear();
+  if (instance_exec_status.done()) final_report_sent_ = true;
+}
+
+void FragmentInstanceState::ReportFailed(
+    const FragmentInstanceExecStatusPB& instance_exec_status) {
+  int num_reports = instance_exec_status.stateful_report_size();
+  if (num_reports > 0 && prev_stateful_reports_.size() != num_reports) {
+    // If a stateful report was generated in GetStatusReport(), copy it to
+    // 'prev_stateful_reports_'. It will be the last one in the list and will have a seq
+    // no that matches the overall report's seq no. There can be at most 1 new stateful
+    // report that has been generated since the last call to ReportSuccessful()/Failed().
+    DCHECK_EQ(prev_stateful_reports_.size() + 1, num_reports);
+    const StatefulStatusPB& stateful_report =
+        instance_exec_status.stateful_report()[num_reports - 1];
+    DCHECK_EQ(stateful_report.report_seq_no(), instance_exec_status.report_seq_no());
+    prev_stateful_reports_.emplace_back(stateful_report);
   }
-  // Send new errors to coordinator.
-  runtime_state()->GetUnreportedErrors(instance_status->mutable_error_log());
 }
 
 Status FragmentInstanceState::Open() {
diff --git a/be/src/runtime/fragment-instance-state.h b/be/src/runtime/fragment-instance-state.h
index 3a8768c..240d44f 100644
--- a/be/src/runtime/fragment-instance-state.h
+++ b/be/src/runtime/fragment-instance-state.h
@@ -96,6 +96,13 @@ class FragmentInstanceState {
   void GetStatusReport(FragmentInstanceExecStatusPB* instance_status,
       TRuntimeProfileTree* thrift_profile);
 
+  /// After each call to GetStatusReport(), the query state thread should call one of the
+  /// following to indicate if the report rpc was successful. Note that in the case of
+  /// ReportFailed(), the report may have been received by the coordinator even though the
+  /// rpc appeared to fail.
+  void ReportSuccessful(const FragmentInstanceExecStatusPB& instance_status);
+  void ReportFailed(const FragmentInstanceExecStatusPB& instance_status);
+
   /// Returns fragment instance's sink if this is the root fragment instance. Valid after
   /// the Prepare phase. May be nullptr.
   PlanRootSink* root_sink() { return root_sink_; }
@@ -158,6 +165,14 @@ class FragmentInstanceState {
   /// state thread only. Written in GetStatusReport() by the query state thread.
   bool final_report_sent_ = false;
 
+  /// The non-idempotent parts of any reports that were generated but may not have been
+  /// received by the coordinator.
+  std::vector<StatefulStatusPB> prev_stateful_reports_;
+
+  /// True if a report has been generated where 'done' is true, after which the sequence
+  /// number should not be bumped for future reports.
+  bool final_report_generated_ = false;
+
   /// Profile for timings for each stage of the plan fragment instance's lifecycle.
   /// Lives in obj_pool().
   RuntimeProfile* timings_profile_ = nullptr;
@@ -212,7 +227,10 @@ class FragmentInstanceState {
 
   /// Returns the monotonically increasing sequence number.
   /// Called by query state thread only.
-  int64_t AdvanceReportSeqNo() { return ++report_seq_no_; }
+  int64_t AdvanceReportSeqNo() {
+    DCHECK(!final_report_generated_);
+    return ++report_seq_no_;
+  }
 
   /// A counter for the per query, per host peak mem usage. Note that this is not the
   /// max of the peak memory of all fragments running on a host since it needs to take
diff --git a/be/src/runtime/query-state.cc b/be/src/runtime/query-state.cc
index c0b5cc6..beef32c 100644
--- a/be/src/runtime/query-state.cc
+++ b/be/src/runtime/query-state.cc
@@ -54,11 +54,13 @@ using kudu::rpc::RpcSidecar;
 
 #include "common/names.h"
 
-DEFINE_int32(status_report_interval_ms, 5000,
-    "interval between profile reports; in milliseconds");
-DEFINE_int32(report_status_retry_interval_ms, 100,
-    "The interval in milliseconds to wait before retrying a failed status report RPC to "
-    "the coordinator.");
+static const int DEFAULT_REPORT_WAIT_TIME_MS = 5000;
+
+DEFINE_int32(status_report_interval_ms, DEFAULT_REPORT_WAIT_TIME_MS,
+    "Interval between profile reports in milliseconds. If set to <= 0, periodic "
+    "reporting is disabled.");
+DEFINE_int32(status_report_max_retries, 3,
+    "Max number of times to retry sending the status report before cancelling");
 DECLARE_int32(backend_client_rpc_timeout_ms);
 DECLARE_int64(rpc_max_message_size);
 
@@ -268,7 +270,7 @@ void QueryState::UpdateBackendExecState() {
   // Send one last report if the query has reached the terminal state.
   if (IsTerminalState()) {
     VLOG_QUERY << "UpdateBackendExecState(): last report for " << PrintId(query_id());
-    ReportExecStatus();
+    while (!ReportExecStatus()) SleepForMs(GetReportWaitTimeMs());
   }
 }
 
@@ -317,7 +319,7 @@ void QueryState::ConstructReport(bool instances_started,
   }
 }
 
-void QueryState::ReportExecStatus() {
+bool QueryState::ReportExecStatus() {
 #ifndef NDEBUG
   if (FLAGS_stress_status_report_delay_ms) {
     LOG(INFO) << "Sleeping " << FLAGS_stress_status_report_delay_ms << "ms before "
@@ -353,52 +355,64 @@ void QueryState::ReportExecStatus() {
         serialize_status.ok() ? "OK" : serialize_status.GetDetail(), profile_len);
   }
 
-  // Try to send the RPC 3 times before failing. Sleep for 100ms between retries.
-  // It's safe to retry the RPC as the coordinator handles duplicate RPC messages.
   Status rpc_status;
   Status result_status;
-  for (int i = 0; i < 3; ++i) {
-    RpcController rpc_controller;
-
-    // The profile is a thrift structure serialized to a string and sent as a sidecar.
-    // We keep the runtime profile as Thrift object as Impala client still communicates
-    // with Impala server with Thrift RPC.
-    //
-    // Note that the sidecar is created with faststring so the ownership of the Thrift
-    // profile buffer is transferred to RPC layer and it is freed after the RPC payload
-    // is sent. If serialization of the profile to RPC sidecar fails, we will proceed
-    // without the profile so that the coordinator can still get the status instead of
-    // hitting IMPALA-2990.
-    if (profile_buf != nullptr) {
-      unique_ptr<kudu::faststring> sidecar_buf = make_unique<kudu::faststring>();
-      sidecar_buf->assign_copy(profile_buf, profile_len);
-      unique_ptr<RpcSidecar> sidecar = RpcSidecar::FromFaststring(move(sidecar_buf));
-
-      int sidecar_idx;
-      kudu::Status sidecar_status =
-          rpc_controller.AddOutboundSidecar(move(sidecar), &sidecar_idx);
-      if (LIKELY(sidecar_status.ok())) {
-        report.set_thrift_profiles_sidecar_idx(sidecar_idx);
-      } else {
-        LOG(DFATAL) <<
-            FromKuduStatus(sidecar_status, "Failed to add sidecar").GetDetail();
-      }
+  RpcController rpc_controller;
+
+  // The profile is a thrift structure serialized to a string and sent as a sidecar.
+  // We keep the runtime profile as Thrift object as Impala client still communicates
+  // with Impala server with Thrift RPC.
+  //
+  // Note that the sidecar is created with faststring so the ownership of the Thrift
+  // profile buffer is transferred to RPC layer and it is freed after the RPC payload
+  // is sent. If serialization of the profile to RPC sidecar fails, we will proceed
+  // without the profile so that the coordinator can still get the status instead of
+  // hitting IMPALA-2990.
+  if (profile_buf != nullptr) {
+    unique_ptr<kudu::faststring> sidecar_buf = make_unique<kudu::faststring>();
+    sidecar_buf->assign_copy(profile_buf, profile_len);
+    unique_ptr<RpcSidecar> sidecar = RpcSidecar::FromFaststring(move(sidecar_buf));
+
+    int sidecar_idx;
+    kudu::Status sidecar_status =
+        rpc_controller.AddOutboundSidecar(move(sidecar), &sidecar_idx);
+    if (LIKELY(sidecar_status.ok())) {
+      report.set_thrift_profiles_sidecar_idx(sidecar_idx);
+    } else {
+      LOG(DFATAL) << FromKuduStatus(sidecar_status, "Failed to add sidecar").GetDetail();
     }
+  }
+
+  rpc_controller.set_timeout(
+      MonoDelta::FromMilliseconds(FLAGS_backend_client_rpc_timeout_ms));
+  ReportExecStatusResponsePB resp;
+  rpc_status = FromKuduStatus(proxy_->ReportExecStatus(report, &resp, &rpc_controller),
+      "ReportExecStatus() RPC failed");
+  result_status = Status(resp.status());
+  if (rpc_status.ok()) {
+    num_failed_reports_ = 0;
+  } else {
+    ++num_failed_reports_;
+    LOG(WARNING) << Substitute("Failed to send ReportExecStatus() RPC for query $0. "
+                               "Consecutive failed reports = $1",
+        PrintId(query_id()), num_failed_reports_);
+  }
 
-    rpc_controller.set_timeout(
-        MonoDelta::FromMilliseconds(FLAGS_backend_client_rpc_timeout_ms));
-    ReportExecStatusResponsePB resp;
-    rpc_status = FromKuduStatus(proxy_->ReportExecStatus(report, &resp, &rpc_controller),
-        "ReportExecStatus() RPC failed");
-    result_status = Status(resp.status());
-    if (rpc_status.ok()) break;
-    //TODO: Consider exponential backoff.
-    if (i < 2) SleepForMs(FLAGS_report_status_retry_interval_ms);
-    LOG(WARNING) <<
-        Substitute("Retrying ReportExecStatus() RPC for query $0", PrintId(query_id()));
+  // Notify the fragment instances of the report's status.
+  for (const FragmentInstanceExecStatusPB& instance_exec_status :
+      report.instance_exec_status()) {
+    const TUniqueId& id = ProtoToQueryId(instance_exec_status.fragment_instance_id());
+    FragmentInstanceState* fis = fis_map_[id];
+    if (rpc_status.ok()) {
+      fis->ReportSuccessful(instance_exec_status);
+    } else {
+      fis->ReportFailed(instance_exec_status);
+    }
   }
 
-  if ((!rpc_status.ok() || !result_status.ok()) && instances_started) {
+  if (((!rpc_status.ok() && num_failed_reports_ >= FLAGS_status_report_max_retries)
+          || !result_status.ok())
+      && instances_started) {
     // TODO: should we try to keep rpc_status for the final report? (but the final
     // report, following this Cancel(), may not succeed anyway.)
     // TODO: not keeping an error status here means that all instances might
@@ -418,7 +432,15 @@ void QueryState::ReportExecStatus() {
                 << "Returned status: " << result_status.GetDetail();
     }
     Cancel();
+    return true;
   }
+
+  return rpc_status.ok();
+}
+
+int64_t QueryState::GetReportWaitTimeMs() const {
+  return (FLAGS_status_report_interval_ms > 0 ? FLAGS_status_report_interval_ms :
+      DEFAULT_REPORT_WAIT_TIME_MS) * (num_failed_reports_ + 1);
 }
 
 void QueryState::ErrorDuringPrepare(const Status& status, const TUniqueId& finst_id) {
@@ -564,7 +586,7 @@ void QueryState::MonitorFInstances() {
   DCHECK(backend_exec_state_ == BackendExecState::EXECUTING)
       << BackendExecStateToString(backend_exec_state_);
   if (FLAGS_status_report_interval_ms > 0) {
-    while (!WaitForFinishOrTimeout(FLAGS_status_report_interval_ms)) {
+    while (!WaitForFinishOrTimeout(GetReportWaitTimeMs())) {
       ReportExecStatus();
     }
   } else {
diff --git a/be/src/runtime/query-state.h b/be/src/runtime/query-state.h
index 74afb8a..7aae063 100644
--- a/be/src/runtime/query-state.h
+++ b/be/src/runtime/query-state.h
@@ -387,6 +387,9 @@ class QueryState {
   /// Tracks host resource usage of this backend. Owned by 'obj_pool_', created in c'tor.
   RuntimeProfile* const host_profile_;
 
+  /// The number of failed intermediate reports since the last successfully sent report.
+  int64_t num_failed_reports_ = 0;
+
   /// Create QueryState w/ a refcnt of 0 and a memory limit of 'mem_limit' bytes applied
   /// to the query mem tracker. The query is associated with the resource pool set in
   /// 'query_ctx.request_pool' or from 'request_pool', if the former is not set (needed
@@ -413,8 +416,15 @@ class QueryState {
       TRuntimeProfileForest* profiles_forest);
 
   /// Gather statues and profiles of all fragment instances belonging to this query state
-  /// and send it to the coordinator via ReportExecStatus() RPC.
-  void ReportExecStatus();
+  /// and send it to the coordinator via ReportExecStatus() RPC. Returns true if the
+  /// report rpc was successful or if it was unsuccessful and we've reached the maximum
+  /// number of allowed failures and cancelled.
+  bool ReportExecStatus();
+
+  /// Returns the amount of time in ms to wait before sending the next status report,
+  /// calculated as a function of the status report interval with backoff based on the
+  /// number of consecutive failed reports.
+  int64_t GetReportWaitTimeMs() const;
 
   /// Returns true if the overall backend status is already set with an error.
   bool HasErrorStatus() const {
diff --git a/common/protobuf/control_service.proto b/common/protobuf/control_service.proto
index 273e0ae..8e6749d 100644
--- a/common/protobuf/control_service.proto
+++ b/common/protobuf/control_service.proto
@@ -109,6 +109,21 @@ enum FInstanceExecStatePB {
   FINISHED = 8;
 }
 
+// Represents any part of the status report that isn't idempotent. If the executor thinks
+// the report failed, we'll retransmit these parts, and this allows us to keep them
+// associated with their original sequence number so that if the coordinator actually did
+// receive the original report it won't reapply them.
+message StatefulStatusPB {
+  // Sequence number prevents out-of-order or duplicated updates from being applied.
+  // 'report_seq_no' will be <= the 'report_seq_no' in the FragmentInstanceExecStatusPB
+  // that contains this StatefulStatusPB.
+  optional int64 report_seq_no = 1;
+
+  // Map of TErrorCode to ErrorLogEntryPB; New errors that have not been reported to
+  // the coordinator by this fragment instance. Not idempotent.
+  map<int32, ErrorLogEntryPB> error_log = 2;
+}
+
 message FragmentInstanceExecStatusPB {
   // Sequence number prevents out-of-order or duplicated updates from being applied.
   optional int64 report_seq_no = 1;
@@ -126,9 +141,9 @@ message FragmentInstanceExecStatusPB {
   // instance. This is sent only when 'done' above is true. Not idempotent.
   optional DmlExecStatusPB dml_exec_status = 5;
 
-  // Map of TErrorCode to ErrorLogEntryPB; New errors that have not been reported to
-  // the coordinator by this fragment instance. Not idempotent.
-  map<int32, ErrorLogEntryPB> error_log = 6;
+  // The non-idempotent parts of the report, and any prior reports that are not known to
+  // have been received by the coordinator.
+  repeated StatefulStatusPB stateful_report = 6;
 }
 
 message ReportExecStatusRequestPB {
diff --git a/tests/custom_cluster/test_rpc_timeout.py b/tests/custom_cluster/test_rpc_timeout.py
index bbec46a..d007ef4 100644
--- a/tests/custom_cluster/test_rpc_timeout.py
+++ b/tests/custom_cluster/test_rpc_timeout.py
@@ -148,3 +148,20 @@ class TestRPCTimeout(CustomClusterTestSuite):
   def test_reportexecstatus_profile_fail(self):
     query_options = {'debug_action': 'REPORT_EXEC_STATUS_PROFILE:FAIL@0.8'}
     self.execute_query_verify_metrics(self.TEST_QUERY, query_options, 10)
+
+  @pytest.mark.execute_serially
+  @CustomClusterTestSuite.with_args("--backend_client_rpc_timeout_ms=100"
+      " --status_report_interval_ms=1000 --status_report_max_retries=100000")
+  def test_reportexecstatus_retries(self, unique_database):
+    tbl = "%s.kudu_test" % unique_database
+    self.execute_query("create table %s (a int primary key) stored as kudu" % tbl)
+    # Since the sleep time (1000ms) is much longer than the rpc timeout (100ms), all
+    # reports will appear to fail. The query is designed to result in many intermediate
+    # status reports but fewer than the max allowed failures, so the query should succeed.
+    query_options = {'debug_action': 'REPORT_EXEC_STATUS_DELAY:SLEEP@1000'}
+    result = self.execute_query(
+        "insert into %s select 0 from tpch.lineitem limit 100000" % tbl, query_options)
+    assert result.success, str(result)
+    # Ensure that the error log was tracked correctly - all but the first row inserted
+    # should result in a 'key already present' insert error.
+    assert "(1 of 99999 similar)" in result.log, str(result)


[impala] 04/04: IMPALA-7929: Allow null qualifier in THBaseFilter

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 06d078e76bad6944ad487fd432ff6d0aa9174960
Author: Yongjun Zhang <yz...@cloudera.com>
AuthorDate: Fri Jan 11 13:44:29 2019 -0800

    IMPALA-7929: Allow null qualifier in THBaseFilter
    
    Impala query failed with "IntenalException: Required field 'qualifier'
    was not present!" on table created via hive and mapped to HBase, because
    the qualifier of the HBase key column is null in the mapped table, and
    Impala required non-null qualifier. The fix here is to relax this
    requirement.
    
    Test:
    Added unit test.
    Tested in real cluster.
    
    Change-Id: I378c2249604481067b5b1c3a3bbb28c30ad4d751
    Reviewed-on: http://gerrit.cloudera.org:8080/12213
    Reviewed-by: Joe McDonnell <jo...@cloudera.com>
    Tested-by: Tim Armstrong <ta...@cloudera.com>
---
 common/thrift/PlanNodes.thrift                     |  3 +-
 .../org/apache/impala/planner/HBaseScanNode.java   | 11 +++++--
 .../queries/QueryTest/hbase-col-filter.test        | 25 ++++++++++++++++
 tests/query_test/test_hbase_queries.py             | 35 ++++++++++++++++++++++
 4 files changed, 70 insertions(+), 4 deletions(-)

diff --git a/common/thrift/PlanNodes.thrift b/common/thrift/PlanNodes.thrift
index 331e432..a4e0028 100644
--- a/common/thrift/PlanNodes.thrift
+++ b/common/thrift/PlanNodes.thrift
@@ -284,7 +284,8 @@ struct TDataSourceScanNode {
 
 struct THBaseFilter {
   1: required string family
-  2: required string qualifier
+  // The qualifier for HBase Key column can be null, thus the field is optional here.
+  2: optional string qualifier
   // Ordinal number into enum HBase CompareFilter.CompareOp.
   // We don't use TExprOperator because the op is interpreted by an HBase Filter, and
   // not the c++ expr eval.
diff --git a/fe/src/main/java/org/apache/impala/planner/HBaseScanNode.java b/fe/src/main/java/org/apache/impala/planner/HBaseScanNode.java
index 808ab41..23adfa3 100644
--- a/fe/src/main/java/org/apache/impala/planner/HBaseScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HBaseScanNode.java
@@ -380,9 +380,14 @@ public class HBaseScanNode extends ScanNode {
 
         StringLiteral literal = (StringLiteral) bindingExpr;
         HBaseColumn col = (HBaseColumn) slot.getColumn();
-        filters_.add(new THBaseFilter(
-            col.getColumnFamily(), col.getColumnQualifier(),
-            (byte) hbaseOp.ordinal(), literal.getUnescapedValue()));
+        // IMPALA-7929: Since the qualifier can be null (e.g. for the key column of an
+        // HBase table), the qualifier field must be optional in order to express the
+        // null value. Constructors in Thrift do not set optional fields, so the qualifier
+        // must be set separately.
+        THBaseFilter thbf = new THBaseFilter(col.getColumnFamily(),
+            (byte) hbaseOp.ordinal(), literal.getUnescapedValue());
+        thbf.setQualifier(col.getColumnQualifier());
+        filters_.add(thbf);
         analyzer.materializeSlots(Lists.newArrayList(e));
       }
     }
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hbase-col-filter.test b/testdata/workloads/functional-query/queries/QueryTest/hbase-col-filter.test
new file mode 100644
index 0000000..87769c1
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/hbase-col-filter.test
@@ -0,0 +1,25 @@
+====
+---- QUERY
+select count(*) from hbase_col_filter_testkeyx
+WHERE k != 'row1'
+---- RESULTS
+3
+====
+---- QUERY
+select count(*) from hbase_col_filter_testkeyx
+WHERE k = 'row1'
+---- RESULTS
+1
+====
+---- QUERY
+select count(*) from hbase_col_filter_testkeyx
+WHERE c != 'c2'
+---- RESULTS
+2
+====
+---- QUERY
+select count(*) from hbase_col_filter_testkeyx
+WHERE c = 'c4'
+---- RESULTS
+1
+====
diff --git a/tests/query_test/test_hbase_queries.py b/tests/query_test/test_hbase_queries.py
index 8fbdfb0..4fa286b 100644
--- a/tests/query_test/test_hbase_queries.py
+++ b/tests/query_test/test_hbase_queries.py
@@ -19,6 +19,13 @@
 
 import pytest
 
+from tests.common.skip import (
+    SkipIfIsilon,
+    SkipIfS3,
+    SkipIfABFS,
+    SkipIfADLS,
+    SkipIfLocal)
+
 from tests.common.impala_test_suite import ImpalaTestSuite
 
 class TestHBaseQueries(ImpalaTestSuite):
@@ -56,3 +63,31 @@ class TestHBaseQueries(ImpalaTestSuite):
   @pytest.mark.execute_serially
   def test_hbase_inserts(self, vector):
     self.run_test_case('QueryTest/hbase-inserts', vector)
+
+  @SkipIfIsilon.hive
+  @SkipIfS3.hive
+  @SkipIfABFS.hive
+  @SkipIfADLS.hive
+  @SkipIfLocal.hive
+  def test_hbase_col_filter(self, vector, unique_database):
+    """IMPALA-7929: test query with table created with hive and mapped to hbase. The key
+    column doesn't have qualifier and the query with predicate on key column name should
+    not fail"""
+    table_name = "{0}.hbase_col_filter_testkeyx".format(unique_database)
+    cr_table = """CREATE TABLE {0} (k STRING, c STRING) STORED BY
+               'org.apache.hadoop.hive.hbase.HBaseStorageHandler' WITH SERDEPROPERTIES
+               ('hbase.columns.mapping'=':key,cf:c', 'serialization.format'='1')
+               TBLPROPERTIES ('hbase.table.name'=\'{1}\',
+               'storage_handler'='org.apache.hadoop.hive.hbase.HBaseStorageHandler')
+               """.format(table_name, table_name)
+    add_data = """INSERT INTO TABLE {0} VALUES ('row1', 'c1'), ('row2', 'c2'),
+             ('row3', 'c2'), ('row4', 'c4')""".format(table_name)
+    del_table = "DROP TABLE IF EXISTS {0}".format(table_name)
+
+    try:
+      self.run_stmt_in_hive(cr_table)
+      self.run_stmt_in_hive(add_data)
+      self.client.execute("invalidate metadata {0}".format(table_name))
+      self.run_test_case('QueryTest/hbase-col-filter', vector, unique_database)
+    finally:
+      self.run_stmt_in_hive(del_table)


[impala] 01/04: IMPALA-7657: Codegen IsNotEmptyPredicate and ValidTupleIdExpr.

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 7707eb041728456ea5e05ae8acc5ab59c715b98a
Author: Andrew Sherman <as...@cloudera.com>
AuthorDate: Mon Jan 7 17:27:19 2019 -0800

    IMPALA-7657: Codegen IsNotEmptyPredicate and ValidTupleIdExpr.
    
    These two classes evaluate scalar expressions. Previously codegen was
    done by calling ScalarExpr::GetCodegendComputeFnWrapper which generates
    a static method that calls the scalar expression evaluation method. Make
    this more efficient by generating code which is customized using
    information available at codegen time.
    
    Add new cross-compiled files null-literal-ir.cc slot-ref-ir.cc
    
    IsNotEmptyPredicate works by getting a CollectionVal object from the
    single child Expr node, and counting its tuples. At codegen time we know
    the type and value of the child node. Generate a call to a node-specific
    non-virtual cross-compiled method to get the CollectionVal object from
    the child. Then generate a code that examines the CollectionVal and
    returns an IntVal.
    
    A ValidTupleIdExpr node contains a vector of tuple ids. It works by
    probing each row for the tuple ids in the vector to find a non-null
    tuple. At codegen time we know the vector of tuple ids. We unroll the
    loop through the tuple ids, generating code that evaluates if the tuple
    is non-null, and returns the tuple id if/when a non-null tuple is found.
    
    IMPALA-7657 also requests replacing GetCodegendComputeFnWrapper() in
    TupleIsNullPredicate. In the current Impala code this method is never
    called. This is because TupleIsNullPredicate is always wrapped in an
    IfExpr. This is always codegen'd by IfExpr's
    GetCodegendComputeFnWrapper() method. There is a separate Jira
    IMPALA-7655 to improve codegen of IfExpr.
    
    Minor corrections:
      Correct the link to llvm tutorial in LlvmCodegen.
    
    PERFORMANCE:
      I tested performance on a local mini-cluster. I wrote some
      pathological queries to test the new code. The new codegen'd code is
      very similar in performance. Both ValidTupleIdExpr and
      IsNotEmptyPredicate seem very slightly faster than the old code.
      Overall these changes are not purely for performance but to move away
      from GetCodegendComputeFnWrapper.
    
    TESTING:
      The changed scalar expressions are well exercised by current tests.
      Ran exhaustive end-to-end tests.
    
    Change-Id: Ifb87b9e3b879c278ce8638d97bcb320a7555a6b3
    Reviewed-on: http://gerrit.cloudera.org:8080/12068
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/codegen/gen_ir_descriptions.py              |   4 +
 be/src/codegen/impala-ir.cc                        |   4 +-
 be/src/codegen/llvm-codegen.h                      |   3 +-
 be/src/exprs/CMakeLists.txt                        |   2 +
 be/src/exprs/is-not-empty-predicate.cc             | 156 ++++++++++++++++++++-
 be/src/exprs/is-not-empty-predicate.h              |   8 +-
 ...is-not-empty-predicate.h => null-literal-ir.cc} |  32 +----
 be/src/exprs/null-literal.cc                       |   8 +-
 be/src/exprs/null-literal.h                        |   2 +
 be/src/exprs/scalar-expr.h                         |   1 -
 .../{is-not-empty-predicate.h => slot-ref-ir.cc}   |  37 ++---
 be/src/exprs/slot-ref.cc                           |  16 +--
 be/src/exprs/slot-ref.h                            |   1 +
 be/src/exprs/valid-tuple-id.cc                     | 123 +++++++++++++++-
 be/src/exprs/valid-tuple-id.h                      |  11 +-
 15 files changed, 317 insertions(+), 91 deletions(-)

diff --git a/be/src/codegen/gen_ir_descriptions.py b/be/src/codegen/gen_ir_descriptions.py
index 57986ed..4829539 100755
--- a/be/src/codegen/gen_ir_descriptions.py
+++ b/be/src/codegen/gen_ir_descriptions.py
@@ -93,6 +93,10 @@ ir_functions = [
    "_ZN6impala10ScalarExpr15GetTimestampValEPS0_PNS_19ScalarExprEvaluatorEPKNS_8TupleRowE"],
   ["SCALAR_EXPR_GET_DECIMAL_VAL",
    "_ZN6impala10ScalarExpr13GetDecimalValEPS0_PNS_19ScalarExprEvaluatorEPKNS_8TupleRowE"],
+  ["SCALAR_EXPR_SLOT_REF_GET_COLLECTION_VAL",
+   "_ZNK6impala7SlotRef16GetCollectionValEPNS_19ScalarExprEvaluatorEPKNS_8TupleRowE"],
+  ["SCALAR_EXPR_NULL_LITERAL_GET_COLLECTION_VAL",
+   "_ZNK6impala11NullLiteral16GetCollectionValEPNS_19ScalarExprEvaluatorEPKNS_8TupleRowE"],
   ["HASH_CRC", "IrCrcHash"],
   ["HASH_MURMUR", "IrMurmurHash"],
   ["PHJ_PROCESS_BUILD_BATCH",
diff --git a/be/src/codegen/impala-ir.cc b/be/src/codegen/impala-ir.cc
index 520c9f3..52586f9 100644
--- a/be/src/codegen/impala-ir.cc
+++ b/be/src/codegen/impala-ir.cc
@@ -37,8 +37,8 @@
 #include "exec/select-node-ir.cc"
 #include "exec/topn-node-ir.cc"
 #include "exec/union-node-ir.cc"
-#include "exprs/aggregate-functions-ir.cc"
 #include "exprs/agg-fn-evaluator-ir.cc"
+#include "exprs/aggregate-functions-ir.cc"
 #include "exprs/bit-byte-functions-ir.cc"
 #include "exprs/cast-functions-ir.cc"
 #include "exprs/compound-predicates-ir.cc"
@@ -49,8 +49,10 @@
 #include "exprs/is-null-predicate-ir.cc"
 #include "exprs/like-predicate-ir.cc"
 #include "exprs/math-functions-ir.cc"
+#include "exprs/null-literal-ir.cc"
 #include "exprs/operators-ir.cc"
 #include "exprs/scalar-expr-ir.cc"
+#include "exprs/slot-ref-ir.cc"
 #include "exprs/string-functions-ir.cc"
 #include "exprs/timestamp-functions-ir.cc"
 #include "exprs/udf-builtins-ir.cc"
diff --git a/be/src/codegen/llvm-codegen.h b/be/src/codegen/llvm-codegen.h
index 33d4b61..97300a0 100644
--- a/be/src/codegen/llvm-codegen.h
+++ b/be/src/codegen/llvm-codegen.h
@@ -94,7 +94,6 @@ class LlvmBuilder : public llvm::IRBuilder<> {
   using llvm::IRBuilder<>::IRBuilder;
 };
 
-
 /// LLVM code generator.  This is the top level object to generate jitted code.
 //
 /// LLVM provides a c++ IR builder interface so IR does not need to be written
@@ -103,7 +102,7 @@ class LlvmBuilder : public llvm::IRBuilder<> {
 /// The llvm documentation is not fantastic and a lot of this was figured out
 /// by experimenting.  Thankfully, their API is pretty well designed so it's
 /// possible to get by without great documentation.  The llvm tutorial is very
-/// helpful, http://llvm.org/docs/tutorial/LangImpl1.html.  In this tutorial, they
+/// helpful, https://llvm.org/docs/tutorial/LangImpl01.html.  In this tutorial, they
 /// go over how to JIT an AST for a toy language they create.
 /// It is also helpful to use their online app that lets you compile c/c++ to IR.
 /// http://llvm.org/demo/index.cgi.
diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt
index 734892d..6158d59 100644
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -48,11 +48,13 @@ add_library(Exprs
   literal.cc
   math-functions-ir.cc
   null-literal.cc
+  null-literal-ir.cc
   operators-ir.cc
   scalar-expr.cc
   scalar-expr-evaluator.cc
   scalar-expr-ir.cc
   slot-ref.cc
+  slot-ref-ir.cc
   string-functions.cc
   string-functions-ir.cc
   timestamp-functions.cc
diff --git a/be/src/exprs/is-not-empty-predicate.cc b/be/src/exprs/is-not-empty-predicate.cc
index e0ac41b..9d5e277 100644
--- a/be/src/exprs/is-not-empty-predicate.cc
+++ b/be/src/exprs/is-not-empty-predicate.cc
@@ -18,16 +18,21 @@
 #include "exprs/is-not-empty-predicate.h"
 
 #include <sstream>
+#include <codegen/codegen-anyval.h>
+#include <codegen/llvm-codegen.h>
+#include <llvm/IR/BasicBlock.h>
 
 #include "gen-cpp/Exprs_types.h"
 
 #include "common/names.h"
+#include "exprs/null-literal.h"
+#include "exprs/slot-ref.h"
 
 namespace impala {
 
-IsNotEmptyPredicate::IsNotEmptyPredicate(const TExprNode& node)
-  : Predicate(node) {
-}
+const char* IsNotEmptyPredicate::LLVM_CLASS_NAME = "class.impala::IsNotEmptyPredicate";
+
+IsNotEmptyPredicate::IsNotEmptyPredicate(const TExprNode& node) : Predicate(node) {}
 
 BooleanVal IsNotEmptyPredicate::GetBooleanVal(
     ScalarExprEvaluator* eval, const TupleRow* row) const {
@@ -42,13 +47,150 @@ Status IsNotEmptyPredicate::Init(const RowDescriptor& row_desc, RuntimeState* st
   return Status::OK();
 }
 
-Status IsNotEmptyPredicate::GetCodegendComputeFn(LlvmCodeGen* codegen,
-    llvm::Function** fn) {
-  return GetCodegendComputeFnWrapper(codegen, fn);
+// Sample IR output (when child is a SlotRef): // FIXME needs review
+//
+//   define i16 @IsNotEmptyPredicate(%"class.impala::ScalarExprEvaluator"* %eval,
+//     %"class.impala::TupleRow"* %row) #38 {
+//   entry:
+//   %0 = alloca i16
+//   %1 = alloca i16
+//   %collection_val = alloca %"struct.impala_udf::CollectionVal"
+//   call void @_ZNK6impala7SlotRef16GetCollectionValEPNS_
+//     19ScalarExprEvaluatorEPKNS_8TupleRowE(%"struct.impala_udf::CollectionVal"*
+//     %collection_val, %"class.impala::SlotRef"* inttoptr (i64 140731643050560
+//     to %"class.impala::SlotRef"*),
+//     %"class.impala::ScalarExprEvaluator"* %eval, %"class.impala::TupleRow"* %row)
+//   %anyval_ptr = getelementptr inbounds %"struct.impala_udf::CollectionVal",
+//     %"struct.impala_udf::CollectionVal"* %collection_val, i32 0, i32 0
+//   %is_null_ptr = getelementptr inbounds %"struct.impala_udf::AnyVal",
+//     %"struct.impala_udf::AnyVal"* %anyval_ptr, i32 0, i32 0
+//   %is_null = load i8, i8* %is_null_ptr
+//   %is_null_bool = icmp ne i8 %is_null, 0
+//   %num_tuples_ptr = getelementptr inbounds %"struct.impala_udf::CollectionVal",
+//     %"struct.impala_udf::CollectionVal"* %collection_val, i32 0, i32 3
+//   %num_tuples = load i32, i32* %num_tuples_ptr
+//   br i1 %is_null_bool, label %ret_null, label %check_count
+//
+// check_count:                                      ; preds = %entry
+//   %4 = icmp ne i32 %num_tuples, 0
+//   %ret2 = load i16, i16* %0
+//   %5 = zext i1 %4 to i16
+//   %6 = shl i16 %5, 8
+//   %7 = and i16 %ret2, 255
+//   %ret21 = or i16 %7, %6
+//   ret i16 %ret21
+//
+// ret_null:                                         ; preds = %entry
+//   %ret = load i16, i16* %1
+//   ret i16 1
+// }
+//
+Status IsNotEmptyPredicate::GetCodegendComputeFn(
+    LlvmCodeGen* codegen, llvm::Function** fn) {
+  if (ir_compute_fn_ != nullptr) {
+    *fn = ir_compute_fn_;
+    return Status::OK();
+  }
+
+  // Create a method with the expected signature.
+  llvm::LLVMContext& context = codegen->context();
+  llvm::Value* args[2];
+  llvm::Function* new_fn =
+      CreateIrFunctionPrototype("IsNotEmptyPredicate", codegen, &args);
+  llvm::BasicBlock* entry_block = llvm::BasicBlock::Create(context, "entry", new_fn);
+  LlvmBuilder builder(entry_block);
+
+  ScalarExpr* child = children_[0]; // The child node, on which to call GetCollectionVal.
+  llvm::Value* child_expr; //  a Value for 'child' with the correct subtype of ScalarExpr.
+  llvm::Function* get_collection_val_fn; // a type specific GetCollectionVal method
+
+  //  To save a virtual method call, find the type of child and lookup the non-virtual
+  //  GetCollection method to call.
+  if (dynamic_cast<SlotRef*>(child)) {
+    // Lookup SlotRef::GetCollectionVal().
+    get_collection_val_fn =
+        codegen->GetFunction(IRFunction::SCALAR_EXPR_SLOT_REF_GET_COLLECTION_VAL, false);
+    child_expr = codegen->CastPtrToLlvmPtr(
+        codegen->GetNamedPtrType(SlotRef::LLVM_CLASS_NAME), child);
+  } else if (dynamic_cast<NullLiteral*>(child)) {
+    // Lookup NullLiteral::GetCollectionVal().
+    get_collection_val_fn = codegen->GetFunction(
+        IRFunction::SCALAR_EXPR_NULL_LITERAL_GET_COLLECTION_VAL, false);
+    child_expr = codegen->CastPtrToLlvmPtr(
+        codegen->GetNamedPtrType(NullLiteral::LLVM_CLASS_NAME), child);
+  } else {
+    // This may mean someone implemented GetCollectionVal in a new subclass of ScalarExpr.
+    DCHECK(false) << "Unknown GetCollectionVal implementation: " << typeid(*child).name();
+    return Status("Codegen'd IsNotEmptyPredicate function found unknown GetCollectionVal,"
+                  " see log.");
+  }
+  DCHECK(get_collection_val_fn != nullptr);
+  DCHECK(child_expr != nullptr);
+
+  // Find type for the CollectionVal struct.
+  llvm::Type* collection_type = codegen->GetNamedType("struct.impala_udf::CollectionVal");
+  DCHECK(collection_type->isStructTy());
+
+  // Allocate space for the CollectionVal on the stack
+  llvm::Value* collection_val_ptr =
+      codegen->CreateEntryBlockAlloca(builder, collection_type, "collection_val");
+
+  // The get_collection_val_fn returns a CollectionVal by value. In llvm we have to pass a
+  // pointer as the first argument. The second argument is the object on which the call is
+  // made.
+  llvm::Value* get_coll_call_args[] = {collection_val_ptr, child_expr, args[0], args[1]};
+
+  // Construct the call to the evaluation method, and return the result.
+  llvm::Value* collection = builder.CreateCall(get_collection_val_fn, get_coll_call_args);
+  DCHECK(collection != nullptr);
+
+  // Find the 'is_null' field of the CollectionVal.
+  llvm::Value* anyval_ptr =
+      builder.CreateStructGEP(nullptr, collection_val_ptr, 0, "anyval_ptr");
+  llvm::Value* is_null_ptr =
+      builder.CreateStructGEP(nullptr, anyval_ptr, 0, "is_null_ptr");
+  llvm::Value* is_null = builder.CreateLoad(is_null_ptr, "is_null");
+
+  // Check if 'is_null' is true.
+  llvm::Value* is_null_bool =
+      builder.CreateICmpNE(is_null, codegen->GetI8Constant(0), "is_null_bool");
+
+  llvm::BasicBlock* check_count =
+      llvm::BasicBlock::Create(context, "check_count", new_fn);
+  llvm::BasicBlock* ret_null = llvm::BasicBlock::Create(context, "ret_null", new_fn);
+  builder.CreateCondBr(is_null_bool, ret_null, check_count);
+
+  // Add code to the block that is executed if is_null was true.
+  builder.SetInsertPoint(ret_null);
+  // allocate a BooleanVal, set null in it, and return it.
+  CodegenAnyVal null_result(codegen, &builder, TYPE_BOOLEAN, nullptr, "null_result");
+  builder.CreateRet(null_result.GetNullVal(codegen, TYPE_BOOLEAN));
+
+  // Back to the branch where 'is_null' is false.
+  builder.SetInsertPoint(check_count);
+  // Load the value of 'num_tuples'
+  llvm::Value* num_tuples_ptr =
+      builder.CreateStructGEP(nullptr, collection_val_ptr, 3, "num_tuples_ptr");
+  llvm::Value* num_tuples = builder.CreateLoad(num_tuples_ptr, "num_tuples");
+
+  llvm::Value* has_values = builder.CreateICmpNE(num_tuples, codegen->GetI32Constant(0));
+  CodegenAnyVal has_values_result(
+      codegen, &builder, TYPE_BOOLEAN, nullptr, "has_values_result");
+  has_values_result.SetVal(has_values);
+  builder.CreateRet(has_values_result.GetLoweredValue());
+
+  *fn = codegen->FinalizeFunction(new_fn);
+  if (UNLIKELY(*fn == nullptr)) {
+    return Status(TErrorCode::IR_VERIFY_FAILED, "IsNotEmptyPredicate");
+  }
+
+  ir_compute_fn_ = *fn;
+
+  return Status::OK();
 }
 
 string IsNotEmptyPredicate::DebugString() const {
   return ScalarExpr::DebugString("IsNotEmptyPredicate");
 }
 
-}
+} // namespace impala
diff --git a/be/src/exprs/is-not-empty-predicate.h b/be/src/exprs/is-not-empty-predicate.h
index 0747a16..bf20350 100644
--- a/be/src/exprs/is-not-empty-predicate.h
+++ b/be/src/exprs/is-not-empty-predicate.h
@@ -28,8 +28,9 @@ class TExprNode;
 /// Predicate that checks whether a collection is empty or not.
 /// TODO: Implement this predicate via the UDF interface once the
 /// interface supports CollectionVals.
-class IsNotEmptyPredicate: public Predicate {
+class IsNotEmptyPredicate : public Predicate {
  public:
+  static const char* LLVM_CLASS_NAME;
   virtual Status GetCodegendComputeFn(LlvmCodeGen* codegen, llvm::Function** fn) override;
   virtual BooleanVal GetBooleanVal(ScalarExprEvaluator*, const TupleRow*) const override;
   virtual std::string DebugString() const override;
@@ -38,10 +39,9 @@ class IsNotEmptyPredicate: public Predicate {
   friend class ScalarExpr;
 
   virtual Status Init(const RowDescriptor& row_desc, RuntimeState* state) override;
-  IsNotEmptyPredicate(const TExprNode& node);
-
+  explicit IsNotEmptyPredicate(const TExprNode& node);
 };
 
-}
+} // namespace impala
 
 #endif
diff --git a/be/src/exprs/is-not-empty-predicate.h b/be/src/exprs/null-literal-ir.cc
similarity index 50%
copy from be/src/exprs/is-not-empty-predicate.h
copy to be/src/exprs/null-literal-ir.cc
index 0747a16..7c5fd86 100644
--- a/be/src/exprs/is-not-empty-predicate.h
+++ b/be/src/exprs/null-literal-ir.cc
@@ -15,33 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef IMPALA_EXPRS_IS_NOT_EMPTY_PREDICATE_H_
-#define IMPALA_EXPRS_IS_NOT_EMPTY_PREDICATE_H_
-
-#include "exprs/predicate.h"
+#include "null-literal.h"
+#include "udf/udf.h"
 
 namespace impala {
 
-class ScalarExprEvaluator;
-class TExprNode;
-
-/// Predicate that checks whether a collection is empty or not.
-/// TODO: Implement this predicate via the UDF interface once the
-/// interface supports CollectionVals.
-class IsNotEmptyPredicate: public Predicate {
- public:
-  virtual Status GetCodegendComputeFn(LlvmCodeGen* codegen, llvm::Function** fn) override;
-  virtual BooleanVal GetBooleanVal(ScalarExprEvaluator*, const TupleRow*) const override;
-  virtual std::string DebugString() const override;
-
- protected:
-  friend class ScalarExpr;
-
-  virtual Status Init(const RowDescriptor& row_desc, RuntimeState* state) override;
-  IsNotEmptyPredicate(const TExprNode& node);
-
-};
-
+CollectionVal NullLiteral::GetCollectionVal(
+    ScalarExprEvaluator* eval, const TupleRow* row) const {
+  DCHECK(type_.IsCollectionType());
+  return CollectionVal::null();
 }
 
-#endif
+} // namespace impala
\ No newline at end of file
diff --git a/be/src/exprs/null-literal.cc b/be/src/exprs/null-literal.cc
index 8ce3a71..a03631a 100644
--- a/be/src/exprs/null-literal.cc
+++ b/be/src/exprs/null-literal.cc
@@ -29,6 +29,8 @@ using namespace impala_udf;
 
 namespace impala {
 
+const char* NullLiteral::LLVM_CLASS_NAME = "class.impala::NullLiteral";
+
 BooleanVal NullLiteral::GetBooleanVal(
     ScalarExprEvaluator* eval, const TupleRow* row) const {
   DCHECK_EQ(type_.type, TYPE_BOOLEAN) << type_;
@@ -89,12 +91,6 @@ DecimalVal NullLiteral::GetDecimalVal(
   return DecimalVal::null();
 }
 
-CollectionVal NullLiteral::GetCollectionVal(
-    ScalarExprEvaluator* eval, const TupleRow* row) const {
-  DCHECK(type_.IsCollectionType());
-  return CollectionVal::null();
-}
-
 // Generated IR for a bigint NULL literal:
 //
 // define { i8, i64 } @NullLiteral(
diff --git a/be/src/exprs/null-literal.h b/be/src/exprs/null-literal.h
index 91f4ca4..c7d9b10 100644
--- a/be/src/exprs/null-literal.h
+++ b/be/src/exprs/null-literal.h
@@ -47,6 +47,8 @@ class NullLiteral: public ScalarExpr {
   /// Constructor for test.
   NullLiteral(PrimitiveType type) : ScalarExpr(type, true) { }
 
+  static const char* LLVM_CLASS_NAME;
+
  protected:
   friend class ScalarExpr;
   friend class ScalarExprEvaluator;
diff --git a/be/src/exprs/scalar-expr.h b/be/src/exprs/scalar-expr.h
index 6eb221e..54b0d9a 100644
--- a/be/src/exprs/scalar-expr.h
+++ b/be/src/exprs/scalar-expr.h
@@ -208,7 +208,6 @@ class ScalarExpr : public Expr {
   friend class ExprTest;
   friend class ExprCodegenTest;
   friend class HashTableTest;
-  friend class OldHashTableTest;
 
   /// Cached LLVM IR for the compute function. Set this in GetCodegendComputeFn().
   llvm::Function* ir_compute_fn_ = nullptr;
diff --git a/be/src/exprs/is-not-empty-predicate.h b/be/src/exprs/slot-ref-ir.cc
similarity index 50%
copy from be/src/exprs/is-not-empty-predicate.h
copy to be/src/exprs/slot-ref-ir.cc
index 0747a16..c9bdd4f 100644
--- a/be/src/exprs/is-not-empty-predicate.h
+++ b/be/src/exprs/slot-ref-ir.cc
@@ -15,33 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef IMPALA_EXPRS_IS_NOT_EMPTY_PREDICATE_H_
-#define IMPALA_EXPRS_IS_NOT_EMPTY_PREDICATE_H_
-
-#include "exprs/predicate.h"
+#include "exprs/slot-ref.h"
+#include "runtime/collection-value.h"
+#include "runtime/tuple-row.h"
 
 namespace impala {
 
-class ScalarExprEvaluator;
-class TExprNode;
-
-/// Predicate that checks whether a collection is empty or not.
-/// TODO: Implement this predicate via the UDF interface once the
-/// interface supports CollectionVals.
-class IsNotEmptyPredicate: public Predicate {
- public:
-  virtual Status GetCodegendComputeFn(LlvmCodeGen* codegen, llvm::Function** fn) override;
-  virtual BooleanVal GetBooleanVal(ScalarExprEvaluator*, const TupleRow*) const override;
-  virtual std::string DebugString() const override;
-
- protected:
-  friend class ScalarExpr;
-
-  virtual Status Init(const RowDescriptor& row_desc, RuntimeState* state) override;
-  IsNotEmptyPredicate(const TExprNode& node);
-
-};
-
+CollectionVal SlotRef::GetCollectionVal(
+    ScalarExprEvaluator* eval, const TupleRow* row) const {
+  DCHECK(type_.IsCollectionType());
+  Tuple* t = row->GetTuple(tuple_idx_);
+  if (t == NULL || t->IsNull(null_indicator_offset_)) return CollectionVal::null();
+  CollectionValue* coll_value =
+      reinterpret_cast<CollectionValue*>(t->GetSlot(slot_offset_));
+  return CollectionVal(coll_value->ptr, coll_value->num_tuples);
 }
 
-#endif
+} // namespace impala
\ No newline at end of file
diff --git a/be/src/exprs/slot-ref.cc b/be/src/exprs/slot-ref.cc
index 00616b3..62a119a 100644
--- a/be/src/exprs/slot-ref.cc
+++ b/be/src/exprs/slot-ref.cc
@@ -38,6 +38,8 @@ using namespace impala_udf;
 
 namespace impala {
 
+const char* SlotRef::LLVM_CLASS_NAME = "class.impala::SlotRef";
+
 SlotRef::SlotRef(const TExprNode& node)
   : ScalarExpr(node),
     slot_offset_(-1),  // invalid
@@ -155,7 +157,7 @@ string SlotRef::DebugString() const {
 // }
 //
 // TODO: We could generate a typed struct (and not a char*) for Tuple for llvm.  We know
-// the types from the TupleDesc.  It will likey make this code simpler to reason about.
+// the types from the TupleDesc.  It will likely make this code simpler to reason about.
 Status SlotRef::GetCodegendComputeFn(LlvmCodeGen* codegen, llvm::Function** fn) {
   if (type_.type == TYPE_CHAR) {
     *fn = NULL;
@@ -475,14 +477,4 @@ DecimalVal SlotRef::GetDecimalVal(
   }
 }
 
-CollectionVal SlotRef::GetCollectionVal(
-    ScalarExprEvaluator* eval, const TupleRow* row) const {
-  DCHECK(type_.IsCollectionType());
-  Tuple* t = row->GetTuple(tuple_idx_);
-  if (t == NULL || t->IsNull(null_indicator_offset_)) return CollectionVal::null();
-  CollectionValue* coll_value =
-      reinterpret_cast<CollectionValue*>(t->GetSlot(slot_offset_));
-  return CollectionVal(coll_value->ptr, coll_value->num_tuples);
-}
-
-}
+} // namespace impala
diff --git a/be/src/exprs/slot-ref.h b/be/src/exprs/slot-ref.h
index 82d25ef..f26012e 100644
--- a/be/src/exprs/slot-ref.h
+++ b/be/src/exprs/slot-ref.h
@@ -56,6 +56,7 @@ class SlotRef : public ScalarExpr {
   virtual bool IsSlotRef() const override { return true; }
   virtual int GetSlotIds(std::vector<SlotId>* slot_ids) const override;
   const SlotId& slot_id() const { return slot_id_; }
+  static const char* LLVM_CLASS_NAME;
 
  protected:
   friend class ScalarExpr;
diff --git a/be/src/exprs/valid-tuple-id.cc b/be/src/exprs/valid-tuple-id.cc
index 84cdf2f..d450aad 100644
--- a/be/src/exprs/valid-tuple-id.cc
+++ b/be/src/exprs/valid-tuple-id.cc
@@ -19,14 +19,16 @@
 
 #include <sstream>
 
-#include "gen-cpp/Exprs_types.h"
-
+#include <codegen/codegen-anyval.h>
+#include "codegen/llvm-codegen.h"
 #include "common/names.h"
 #include "runtime/descriptors.h"
 #include "runtime/tuple-row.h"
 
 namespace impala {
 
+const char* ValidTupleIdExpr::LLVM_CLASS_NAME = "class.impala::ValidTupleIdExpr";
+
 ValidTupleIdExpr::ValidTupleIdExpr(const TExprNode& node) : ScalarExpr(node) {}
 
 Status ValidTupleIdExpr::Init(const RowDescriptor& row_desc, RuntimeState* state) {
@@ -39,8 +41,7 @@ Status ValidTupleIdExpr::Init(const RowDescriptor& row_desc, RuntimeState* state
   return Status::OK();
 }
 
-int ValidTupleIdExpr::ComputeNonNullCount(const TupleRow* row) const {
-  int num_tuples = tuple_ids_.size();
+int ValidTupleIdExpr::ComputeNonNullCount(const TupleRow* row, int num_tuples) {
   int non_null_count = 0;
   for (int i = 0; i < num_tuples; ++i) non_null_count += (row->GetTuple(i) != nullptr);
   return non_null_count;
@@ -48,16 +49,126 @@ int ValidTupleIdExpr::ComputeNonNullCount(const TupleRow* row) const {
 
 IntVal ValidTupleIdExpr::GetIntVal(ScalarExprEvaluator* eval, const TupleRow* row) const {
   // Validate that exactly one tuple is non-NULL.
-  DCHECK_EQ(1, ComputeNonNullCount(row));
   int num_tuples = tuple_ids_.size();
+  DCHECK_EQ(1, ComputeNonNullCount(row, num_tuples));
   for (int i = 0; i < num_tuples; ++i) {
     if (row->GetTuple(i) != nullptr) return IntVal(tuple_ids_[i]);
   }
   return IntVal::null();
 }
 
+// Sample IR output (with 3 tuple ids in the vector):
+//
+// define i64 @ValidTupleId(%"class.impala::ScalarExprEvaluator"* %eval,
+//   %"class.impala::TupleRow"* %row) #48 {
+// entry:
+//   %0 = alloca i64
+//   %1 = alloca i64
+//   %2 = alloca i64
+//   %tuple_row_ptr = getelementptr inbounds %"class.impala::TupleRow",
+//     %"class.impala::TupleRow"* %row, i32 0
+//   %tuple_ptr = bitcast %"class.impala::TupleRow"* %tuple_row_ptr to
+//     %"class.impala::Tuple"**
+//   %tuple_val = load %"class.impala::Tuple"*, %"class.impala::Tuple"** %tuple_ptr
+//   %is_not_null = icmp ne %"class.impala::Tuple"* %tuple_val, null
+//   br i1 %is_not_null, label %return, label %next
+//
+// next:                                             ; preds = %entry
+//   %tuple_row_ptr2 = getelementptr inbounds %"class.impala::TupleRow",
+//     %"class.impala::TupleRow"* %row, i32 1
+//   %tuple_ptr3 = bitcast %"class.impala::TupleRow"* %tuple_row_ptr2 to
+//     %"class.impala::Tuple"**
+//   %tuple_val4 = load %"class.impala::Tuple"*, %"class.impala::Tuple"** %tuple_ptr3
+//   %is_not_null5 = icmp ne %"class.impala::Tuple"* %tuple_val4, null
+//   br i1 %is_not_null5, label %return7, label %next6
+//
+// return:                                           ; preds = %entry
+//   %ret = load i64, i64* %2
+//   %3 = and i64 %ret, 4294967295
+//   %ret1 = or i64 %3, 4294967296    -- 1
+//   ret i64 %ret1
+//
+// next6:                                            ; preds = %next
+//   %tuple_row_ptr10 = getelementptr inbounds %"class.impala::TupleRow",
+//     %"class.impala::TupleRow"* %row, i32 2
+//   %tuple_ptr11 = bitcast %"class.impala::TupleRow"* %tuple_row_ptr10 to
+//     %"class.impala::Tuple"**
+//   %tuple_val12 = load %"class.impala::Tuple"*, %"class.impala::Tuple"** %tuple_ptr11
+//   %is_not_null13 = icmp ne %"class.impala::Tuple"* %tuple_val12, null
+//   br i1 %is_not_null13, label %return15, label %next14
+//
+// return7:                                          ; preds = %next
+//   %ret8 = load i64, i64* %1
+//   %4 = and i64 %ret8, 4294967295
+//   %ret9 = or i64 %4, 12884901888   -- 3
+//   ret i64 %ret9
+//
+// next14:                                           ; preds = %next6
+//   ret i64 1
+//
+// return15:                                         ; preds = %next6
+//   %ret16 = load i64, i64* %0
+//   %5 = and i64 %ret16, 4294967295
+//   %ret17 = or i64 %5, 21474836480   -- 5
+//   ret i64 %ret17
+// }
+//
 Status ValidTupleIdExpr::GetCodegendComputeFn(LlvmCodeGen* codegen, llvm::Function** fn) {
-  return GetCodegendComputeFnWrapper(codegen, fn);
+  if (ir_compute_fn_ != nullptr) {
+    *fn = ir_compute_fn_;
+    return Status::OK();
+  }
+
+  // Create a method with the expected signature.
+  llvm::Value* args[2];
+  llvm::Function* new_fn = CreateIrFunctionPrototype("ValidTupleId", codegen, &args);
+  llvm::LLVMContext& context = codegen->context();
+  llvm::BasicBlock* entry_block = llvm::BasicBlock::Create(context, "entry", new_fn);
+  LlvmBuilder builder(entry_block);
+  llvm::Value* row_ptr = args[1];
+
+  // Unroll the loop.
+  for (uint32_t i = 0; i < tuple_ids_.size(); i++) {
+    // Get the i'th Tuple* from the row.
+    llvm::Value* tuple_row_ptr =
+        builder.CreateInBoundsGEP(row_ptr, codegen->GetI32Constant(i), "tuple_row_ptr");
+    // Cast to Tuple**
+    llvm::Type* tuple_ptr_ptr_type =
+        codegen->GetPtrType(codegen->GetNamedPtrType(Tuple::LLVM_CLASS_NAME));
+    llvm::Value* tuple_ptr_ptr =
+        builder.CreateBitCast(tuple_row_ptr, tuple_ptr_ptr_type, "tuple_ptr_ptr");
+    // Get the Tuple* and compare to nullptr.
+    llvm::Value* tuple_ptr = builder.CreateLoad(tuple_ptr_ptr, "tuple_ptr");
+    llvm::Value* is_not_null = builder.CreateIsNotNull(tuple_ptr, "is_not_null");
+
+    // Create a conditional on the result.
+    llvm::BasicBlock* next = llvm::BasicBlock::Create(context, "next", new_fn);
+    llvm::BasicBlock* ret_block = llvm::BasicBlock::Create(context, "return", new_fn);
+    builder.CreateCondBr(is_not_null, ret_block, next);
+
+    // Add code to the block that is executed if the Tuple* was not a nullptr.
+    builder.SetInsertPoint(ret_block);
+    // Generate code returning an IntVal containing the tuple id.
+    CodegenAnyVal result(codegen, &builder, TYPE_INT, nullptr, "ret");
+    llvm::Constant* tuple_id =
+        codegen->GetI32Constant(static_cast<uint64_t>(tuple_ids_[i]));
+    result.SetVal(tuple_id);
+    builder.CreateRet(result.GetLoweredValue());
+
+    // Add code to the  block that is executed if the Tuple* was null.
+    builder.SetInsertPoint(next);
+  }
+  // We fell out of the loop, return null.
+  builder.CreateRet(CodegenAnyVal::GetNullVal(codegen, type()));
+
+  *fn = codegen->FinalizeFunction(new_fn);
+  if (UNLIKELY(*fn == nullptr)) {
+    return Status(TErrorCode::IR_VERIFY_FAILED, "ValidTupleId");
+  }
+
+  ir_compute_fn_ = *fn;
+
+  return Status::OK();
 }
 
 string ValidTupleIdExpr::DebugString() const {
diff --git a/be/src/exprs/valid-tuple-id.h b/be/src/exprs/valid-tuple-id.h
index 7eb03d9..7de6af0 100644
--- a/be/src/exprs/valid-tuple-id.h
+++ b/be/src/exprs/valid-tuple-id.h
@@ -27,10 +27,13 @@ class TExprNode;
 /// Returns the tuple id of the single non-NULL tuple in the input row.
 /// Valid input rows must have exactly one non-NULL tuple.
 class ValidTupleIdExpr : public ScalarExpr {
+ public:
+  static const char* LLVM_CLASS_NAME;
+
  protected:
   friend class ScalarExpr;
 
-  ValidTupleIdExpr(const TExprNode& node);
+  explicit ValidTupleIdExpr(const TExprNode& node);
 
   virtual Status Init(const RowDescriptor& row_desc, RuntimeState* state) override;
   virtual Status GetCodegendComputeFn(
@@ -44,7 +47,11 @@ class ValidTupleIdExpr : public ScalarExpr {
   std::vector<TupleId> tuple_ids_;
 
   /// Returns the number of tuples in 'row' that are non-null. Used for debugging.
-  int ComputeNonNullCount(const TupleRow* row) const;
+  static int ComputeNonNullCount(const TupleRow* row, int num_tuples);
+
+  /// Returns true if the tuple of the row at the specified index is non-null.
+  /// Called by Codegen.
+  static bool IsTupleFromRowNonNull(const TupleRow* row, int index);
 };
 
 } // namespace impala