You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ab...@apache.org on 2016/07/30 01:49:32 UTC

[1/8] incubator-impala git commit: Use impala-python when building shell tarball

Repository: incubator-impala
Updated Branches:
  refs/heads/master 68e9eed81 -> 36b4ea6f6


Use impala-python when building shell tarball

This is a minor build fix that allows buildall.sh to succeed on systems
where setuptools isn't installed for system python.

Change-Id: I33a24ea4f77e655acaa0de22211c9ef008f5e650
Reviewed-on: http://gerrit.cloudera.org:8080/3797
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/34bc6a72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/34bc6a72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/34bc6a72

Branch: refs/heads/master
Commit: 34bc6a72dbbb4fd00939c3177da4b8e9f525898b
Parents: 68e9eed
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Wed Jul 27 14:34:36 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Jul 28 21:33:41 2016 +0000

----------------------------------------------------------------------
 shell/make_shell_tarball.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/34bc6a72/shell/make_shell_tarball.sh
----------------------------------------------------------------------
diff --git a/shell/make_shell_tarball.sh b/shell/make_shell_tarball.sh
index 2fc2de9..c2bcb63 100755
--- a/shell/make_shell_tarball.sh
+++ b/shell/make_shell_tarball.sh
@@ -85,7 +85,7 @@ for MODULE in ${SHELL_HOME}/ext-py/*; do
   rm -rf dist 2>&1 > /dev/null
   rm -rf build 2>&1 > /dev/null
   echo "Creating an egg for ${MODULE}"
-  python setup.py -q bdist_egg clean
+  impala-python setup.py -q bdist_egg clean
   cp dist/*.egg ${TARBALL_ROOT}/ext-py
   popd 2>&1 > /dev/null
 done


[7/8] incubator-impala git commit: IMPALA-3866: consistent user-facing terminology for scratch dirs

Posted by ab...@apache.org.
IMPALA-3866: consistent user-facing terminology for scratch dirs

Error messages inconsistently referred to "temporary" versus "scratch".
Make the error messages consistent to make it easier for users to debug.
Also make sure the --scratch_dirs configuration option is always
mentioned.

Change-Id: I011e633721e206a2b6ecf1c34d5328f34fbc01ab
Reviewed-on: http://gerrit.cloudera.org:8080/3805
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Reviewed-by: Marcell Szabo <sz...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/dd33e8fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/dd33e8fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/dd33e8fb

Branch: refs/heads/master
Commit: dd33e8fbae5de13c0c91a49a87d576a83cff36e4
Parents: 904265c
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Thu Jul 28 08:54:45 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Fri Jul 29 21:48:07 2016 +0000

----------------------------------------------------------------------
 be/src/runtime/buffered-block-mgr.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/dd33e8fb/be/src/runtime/buffered-block-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/buffered-block-mgr.cc b/be/src/runtime/buffered-block-mgr.cc
index a9e95cd..b560cc6 100644
--- a/be/src/runtime/buffered-block-mgr.cc
+++ b/be/src/runtime/buffered-block-mgr.cc
@@ -820,12 +820,13 @@ Status BufferedBlockMgr::AllocateScratchSpace(int64_t block_size,
     if (status.ok()) return Status::OK();
     // Log error and try other files if there was a problem. Problematic files will be
     // blacklisted so we will not repeatedly log the same error.
-    LOG(WARNING) << "Error while allocating temporary file range: "
-                 << status.msg().msg() << ". Will try another temporary file.";
+    LOG(WARNING) << "Error while allocating range in scratch file '"
+                 << (*tmp_file)->path() << "': " << status.msg().msg()
+                 << ". Will try another scratch file.";
     errs.push_back(status);
   }
-  Status err_status("No usable temporary files: space could not be allocated on any "
-      "temporary device.");
+  Status err_status("No usable scratch files: space could not be allocated in any "
+      "of the configured scratch directories (--scratch_dirs).");
   for (int i = 0; i < errs.size(); ++i) {
     err_status.MergeStatus(errs[i]);
   }
@@ -1100,7 +1101,7 @@ Status BufferedBlockMgr::FindBuffer(unique_lock<mutex>& lock,
     if (disable_spill_) {
       if (block_write_threshold_ == 0) {
         return Status("Spilling has been disabled due to no usable scratch space. "
-            "Please specify a usable scratch space location via the scratch_dirs "
+            "Please specify a usable scratch space location via the --scratch_dirs "
             "impalad flag.");
       } else {
         return Status("Spilling has been disabled for plans that do not have stats and "


[8/8] incubator-impala git commit: IMPALA-1683: Allow REFRESH on a single partition

Posted by ab...@apache.org.
IMPALA-1683: Allow REFRESH on a single partition

Currently the only way to refresh metadata for a partition was to refresh
the whole table. This is a relatively time consuming process especially if
there are many partitions and only one is to be refreshed.
This patch allows the client to REFRESH on a single partition by using the
following syntax:
REFRESH [database_name.]table_name PARTITION (partition_spec)

Testing:
Added parsing and authorization tests in ParserTest.java and
AuthorizationTest.java respectively. A new test file
"test_refresh_partition.py" was added for testing functionality.

Performance:
For a table with 10000 partitions and 1 file per partition

                     execResetMetadata()       Total Execution Time

Refresh Table              3795 ms                   4630 ms

Refersh Partition            42 ms                    680 ms

We see that the time to refresh improves by a factor of 90x but due to
significant overhead of about 640ms in this case the effective improvement
is about 7x. As the size of the table and number of partitions increase,
this improvement would be more significant.

Change-Id: Ia9aa25d190ada367fbebaca47ae8b2cafbea16fb
Reviewed-on: http://gerrit.cloudera.org:8080/3813
Reviewed-by: Dimitris Tsirogiannis <dt...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/36b4ea6f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/36b4ea6f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/36b4ea6f

Branch: refs/heads/master
Commit: 36b4ea6f651e904a62f4f4f8cab8c05a8f1b20dd
Parents: dd33e8f
Author: Bikramjeet Vig <bi...@cloudera.com>
Authored: Tue Jun 14 16:25:17 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Fri Jul 29 23:57:50 2016 +0000

----------------------------------------------------------------------
 common/thrift/CatalogService.thrift             |   4 +
 fe/src/main/cup/sql-parser.cup                  |   8 +-
 .../impala/analysis/AnalysisContext.java        |   6 +-
 .../impala/analysis/ResetMetadataStmt.java      |  18 +-
 .../impala/catalog/CatalogServiceCatalog.java   |  49 ++++
 .../com/cloudera/impala/catalog/HdfsTable.java  |  32 +++
 .../impala/service/CatalogOpExecutor.java       |   7 +-
 .../cloudera/impala/analysis/AnalyzerTest.java  |  15 ++
 .../impala/analysis/AuthorizationTest.java      |  10 +
 .../cloudera/impala/analysis/ParserTest.java    |   7 +
 testdata/workloads/tpch/tpch_core.csv           |   1 -
 tests/metadata/test_refresh_partition.py        | 257 +++++++++++++++++++
 12 files changed, 406 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/common/thrift/CatalogService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/CatalogService.thrift b/common/thrift/CatalogService.thrift
index c2f2cec..87de568 100644
--- a/common/thrift/CatalogService.thrift
+++ b/common/thrift/CatalogService.thrift
@@ -186,6 +186,10 @@ struct TResetMetadataRequest {
   // Fully qualified name of the table to refresh or invalidate; not set if invalidating
   // the entire catalog
   3: optional CatalogObjects.TTableName table_name
+
+  // If set, refreshes the specified partition, otherwise
+  // refreshes the whole table
+  5: optional list<CatalogObjects.TPartitionKeyValue> partition_spec
 }
 
 // Response from TResetMetadataRequest

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/fe/src/main/cup/sql-parser.cup
----------------------------------------------------------------------
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index e0f86c9..00ee242 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -621,11 +621,13 @@ overwrite_val ::=
 
 reset_metadata_stmt ::=
   KW_INVALIDATE KW_METADATA
-  {: RESULT = new ResetMetadataStmt(null, false); :}
+  {: RESULT = new ResetMetadataStmt(null, false, null); :}
   | KW_INVALIDATE KW_METADATA table_name:table
-  {: RESULT = new ResetMetadataStmt(table, false); :}
+  {: RESULT = new ResetMetadataStmt(table, false, null); :}
   | KW_REFRESH table_name:table
-  {: RESULT = new ResetMetadataStmt(table, true); :}
+  {: RESULT = new ResetMetadataStmt(table, true, null); :}
+  | KW_REFRESH table_name:table partition_spec:partition
+  {: RESULT = new ResetMetadataStmt(table, true, partition); :}
   ;
 
 explain_stmt ::=

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java b/fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java
index 098501c..043747d 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java
@@ -389,7 +389,8 @@ public class AnalysisContext {
       throws AuthorizationException, InternalException {
     Preconditions.checkNotNull(analysisResult_);
     Analyzer analyzer = getAnalyzer();
-    // Process statements for which column-level privilege requests may be registered.
+    // Process statements for which column-level privilege requests may be registered
+    // except for DESCRIBE TABLE or REFRESH/INVALIDATE statements
     if (analysisResult_.isQueryStmt() || analysisResult_.isInsertStmt() ||
         analysisResult_.isUpdateStmt() || analysisResult_.isDeleteStmt() ||
         analysisResult_.isCreateTableAsSelectStmt() ||
@@ -434,7 +435,8 @@ public class AnalysisContext {
       for (PrivilegeRequest privReq: analyzer.getPrivilegeReqs()) {
         Preconditions.checkState(
             !(privReq.getAuthorizeable() instanceof AuthorizeableColumn) ||
-            analysisResult_.isDescribeTableStmt());
+            analysisResult_.isDescribeTableStmt() ||
+            analysisResult_.isResetMetadataStmt());
         authorizePrivilegeRequest(authzChecker, privReq);
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/fe/src/main/java/com/cloudera/impala/analysis/ResetMetadataStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/ResetMetadataStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/ResetMetadataStmt.java
index 24656f6..b2834ee 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/ResetMetadataStmt.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/ResetMetadataStmt.java
@@ -21,6 +21,7 @@ import com.cloudera.impala.common.AnalysisException;
 import com.cloudera.impala.thrift.TResetMetadataRequest;
 import com.cloudera.impala.thrift.TTableName;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 
 /**
  * Representation of a REFRESH/INVALIDATE METADATA statement.
@@ -32,10 +33,17 @@ public class ResetMetadataStmt extends StatementBase {
   // true if it is a REFRESH statement.
   private final boolean isRefresh_;
 
-  public ResetMetadataStmt(TableName name, boolean isRefresh) {
+  // not null when refreshing a single partition
+  private final PartitionSpec partitionSpec_;
+
+  public ResetMetadataStmt(TableName name, boolean isRefresh,
+      PartitionSpec partitionSpec) {
     Preconditions.checkArgument(!isRefresh || name != null);
+    Preconditions.checkArgument(isRefresh || partitionSpec == null);
     this.tableName_ = name;
     this.isRefresh_ = isRefresh;
+    this.partitionSpec_ = partitionSpec;
+    if (partitionSpec_ != null) partitionSpec_.setTableName(tableName_);
   }
 
   public TableName getTableName() { return tableName_; }
@@ -57,6 +65,10 @@ public class ResetMetadataStmt extends StatementBase {
           // to Impala.
           throw new AnalysisException(Analyzer.TBL_DOES_NOT_EXIST_ERROR_MSG + tableName_);
         }
+        if (partitionSpec_ != null) {
+          partitionSpec_.setPrivilegeRequirement(Privilege.ANY);
+          partitionSpec_.analyze(analyzer);
+        }
       } else {
         // Verify the user has privileges to access this table.
         analyzer.registerPrivReq(new PrivilegeRequestBuilder()
@@ -77,6 +89,7 @@ public class ResetMetadataStmt extends StatementBase {
     }
 
     if (tableName_ != null) result.append(" ").append(tableName_);
+    if (partitionSpec_ != null) result.append(" " + partitionSpec_.toSql());
     return result.toString();
   }
 
@@ -86,6 +99,9 @@ public class ResetMetadataStmt extends StatementBase {
     if (tableName_ != null) {
       params.setTable_name(new TTableName(tableName_.getDb(), tableName_.getTbl()));
     }
+    if (partitionSpec_ != null) {
+      params.setPartition_spec(partitionSpec_.toThrift());
+    }
     return params;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
index dddc545..2e0e411 100644
--- a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FunctionType;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.ResourceType;
 import org.apache.hadoop.hive.metastore.api.ResourceUri;
 import org.apache.hadoop.hive.metastore.api.UnknownDBException;
@@ -1167,4 +1168,52 @@ public class CatalogServiceCatalog extends Catalog {
   public TableId getNextTableId() { return new TableId(nextTableId_.getAndIncrement()); }
   public SentryProxy getSentryProxy() { return sentryProxy_; }
   public AuthorizationPolicy getAuthPolicy() { return authPolicy_; }
+
+  /**
+   * Reloads metadata for the partition defined by the partition spec
+   * 'partitionSpec' in table 'tbl'. Returns the table object with partition
+   * metadata reloaded
+   */
+  public Table reloadPartition(Table tbl, List<TPartitionKeyValue> partitionSpec)
+      throws CatalogException {
+    catalogLock_.writeLock().lock();
+    synchronized (tbl) {
+      long newCatalogVersion = incrementAndGetCatalogVersion();
+      catalogLock_.writeLock().unlock();
+      HdfsTable hdfsTable = (HdfsTable) tbl;
+      HdfsPartition hdfsPartition = hdfsTable
+          .getPartitionFromThriftPartitionSpec(partitionSpec);
+      // Retrieve partition name from existing partition or construct it from
+      // the partition spec
+      String partitionName = hdfsPartition == null
+          ? HdfsTable.constructPartitionName(partitionSpec)
+          : hdfsPartition.getPartitionName();
+      LOG.debug(String.format("Refreshing Partition metadata: %s %s",
+          hdfsTable.getFullName(), partitionName));
+      MetaStoreClient msClient = getMetaStoreClient();
+      try {
+        org.apache.hadoop.hive.metastore.api.Partition hmsPartition = null;
+        try {
+          hmsPartition = msClient.getHiveClient().getPartition(
+              hdfsTable.getDb().getName(), hdfsTable.getName(), partitionName);
+        } catch (NoSuchObjectException e) {
+          // If partition does not exist in Hive Metastore, remove it from the
+          // catalog
+          if (hdfsPartition != null) {
+            hdfsTable.dropPartition(partitionSpec);
+            hdfsTable.setCatalogVersion(newCatalogVersion);
+          }
+          return hdfsTable;
+        } catch (Exception e) {
+          throw new CatalogException("Error loading metadata for partition: "
+              + hdfsTable.getFullName() + " " + partitionName, e);
+        }
+        hdfsTable.reloadPartition(hdfsPartition, hmsPartition);
+      } finally {
+        msClient.release();
+      }
+      hdfsTable.setCatalogVersion(newCatalogVersion);
+      return hdfsTable;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java b/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java
index 1457bb6..dafcafa 100644
--- a/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.util.StringUtils;
@@ -85,6 +86,7 @@ import com.cloudera.impala.util.MetaStoreUtil;
 import com.cloudera.impala.util.TAccessLevelUtil;
 import com.cloudera.impala.util.TResultRowBuilder;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
@@ -1920,4 +1922,34 @@ public class HdfsTable extends Table {
     }
     return result;
   }
+
+  /**
+   * Constructs a partition name from a list of TPartitionKeyValue objects.
+   */
+  public static String constructPartitionName(List<TPartitionKeyValue> partitionSpec) {
+    List<String> partitionCols = Lists.newArrayList();
+    List<String> partitionVals = Lists.newArrayList();
+    for (TPartitionKeyValue kv: partitionSpec) {
+      partitionCols.add(kv.getName());
+      partitionVals.add(kv.getValue());
+    }
+    return org.apache.hadoop.hive.common.FileUtils.makePartName(partitionCols,
+        partitionVals);
+  }
+
+  /**
+   * Reloads the metadata of partition 'oldPartition' by removing
+   * it from the table and reconstructing it from the HMS partition object
+   * 'hmsPartition'. If old partition is null then nothing is removed and
+   * and partition constructed from 'hmsPartition' is simply added.
+   */
+  public void reloadPartition(HdfsPartition oldPartition, Partition hmsPartition)
+      throws CatalogException {
+    HdfsPartition refreshedPartition = createPartition(
+        hmsPartition.getSd(), hmsPartition);
+    Preconditions.checkArgument(oldPartition == null
+        || oldPartition.compareTo(refreshedPartition) == 0);
+    dropPartition(oldPartition);
+    addPartition(refreshedPartition);
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java b/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java
index a63427a..8cc433c 100644
--- a/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java
@@ -2749,7 +2749,12 @@ public class CatalogOpExecutor {
         if (tbl == null) {
           modifiedObjects.second = null;
         } else {
-          modifiedObjects.second = catalog_.reloadTable(tbl);
+          if (req.isSetPartition_spec()) {
+            modifiedObjects.second = catalog_.reloadPartition(tbl,
+                req.getPartition_spec());
+          } else {
+            modifiedObjects.second = catalog_.reloadTable(tbl);
+          }
         }
       } else {
         wasRemoved = catalog_.invalidateTable(req.getTable_name(), modifiedObjects);

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java
index 6c03352..f7ee934 100644
--- a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java
+++ b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java
@@ -645,6 +645,8 @@ public class AnalyzerTest {
     AnalyzesOk("refresh functional.alltypessmall");
     AnalyzesOk("refresh functional.alltypes_view");
     AnalyzesOk("refresh functional.bad_serde");
+    AnalyzesOk("refresh functional.alltypessmall partition (year=2009, month=1)");
+    AnalyzesOk("refresh functional.alltypessmall partition (year=2009, month=NULL)");
 
     // invalidate metadata <table name> checks the Hive Metastore for table existence
     // and should not throw an AnalysisError if the table or db does not exist.
@@ -655,6 +657,19 @@ public class AnalyzerTest {
         "Table does not exist: functional.unknown_table");
     AnalysisError("refresh unknown_db.unknown_table",
         "Database does not exist: unknown_db");
+    AnalysisError("refresh functional.alltypessmall partition (year=2009, int_col=10)",
+        "Column 'int_col' is not a partition column in table: functional.alltypessmall");
+    AnalysisError("refresh functional.alltypessmall partition (year=2009)",
+        "Items in partition spec must exactly match the partition columns in "
+            + "the table definition: functional.alltypessmall (1 vs 2)");
+    AnalysisError("refresh functional.alltypessmall partition (year=2009, year=2009)",
+        "Duplicate partition key name: year");
+    AnalysisError(
+        "refresh functional.alltypessmall partition (year=2009, month='foo')",
+        "Value of partition spec (column=month) has incompatible type: 'STRING'. "
+            + "Expected type: 'INT'");
+    AnalysisError("refresh functional.zipcode_incomes partition (year=2009, month=1)",
+        "Table is not partitioned: functional.zipcode_incomes");
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java b/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java
index 9f371c7..7bb675b 100644
--- a/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java
+++ b/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java
@@ -791,6 +791,10 @@ public class AuthorizationTest {
     AuthzOk("refresh functional.alltypesagg");
     AuthzOk("invalidate metadata functional.view_view");
     AuthzOk("refresh functional.view_view");
+    // Positive cases for checking refresh partition
+    AuthzOk("refresh functional.alltypesagg partition (year=2010, month=1, day=1)");
+    AuthzOk("refresh functional.alltypes partition (year=2009, month=1)");
+    AuthzOk("refresh functional_seq_snap.alltypes partition (year=2009, month=1)");
 
     AuthzError("invalidate metadata unknown_db.alltypessmall",
         "User '%s' does not have privileges to access: unknown_db.alltypessmall");
@@ -815,6 +819,12 @@ public class AuthorizationTest {
 
     AuthzError("invalidate metadata",
         "User '%s' does not have privileges to access: server");
+    AuthzError(
+        "refresh functional_rc.alltypesagg partition (year=2010, month=1, day=1)",
+        "User '%s' does not have privileges to access: functional_rc.alltypesagg");
+    AuthzError(
+        "refresh functional_rc.alltypesagg partition (year=2010, month=1, day=9999)",
+        "User '%s' does not have privileges to access: functional_rc.alltypesagg");
 
     // TODO: Add test support for dynamically changing privileges for
     // file-based policy.

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java
index aa58056..f506c3a 100644
--- a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java
@@ -2751,11 +2751,18 @@ public class ParserTest {
     ParsesOk("invalidate metadata Foo.S");
     ParsesOk("refresh Foo");
     ParsesOk("refresh Foo.S");
+    ParsesOk("refresh Foo partition (col=2)");
+    ParsesOk("refresh Foo.S partition (col=2)");
+    ParsesOk("refresh Foo.S partition (col1 = 2, col2 = 3)");
 
     ParserError("invalidate");
     ParserError("invalidate metadata Foo.S.S");
+    ParserError("invalidate metadata partition (col=2)");
+    ParserError("invalidate metadata Foo.S partition (col=2)");
     ParserError("REFRESH Foo.S.S");
     ParserError("refresh");
+    ParserError("refresh Foo.S partition (col1 = 2, col2)");
+    ParserError("refresh Foo.S partition ()");
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/testdata/workloads/tpch/tpch_core.csv
----------------------------------------------------------------------
diff --git a/testdata/workloads/tpch/tpch_core.csv b/testdata/workloads/tpch/tpch_core.csv
index 86804ac..04d7896 100644
--- a/testdata/workloads/tpch/tpch_core.csv
+++ b/testdata/workloads/tpch/tpch_core.csv
@@ -7,4 +7,3 @@ file_format:rc, dataset:tpch, compression_codec:none, compression_type:none
 file_format:avro, dataset:tpch, compression_codec: none, compression_type: none
 file_format:avro, dataset:tpch, compression_codec: snap, compression_type: block
 file_format:parquet, dataset:tpch, compression_codec: none, compression_type: none
-file_format:kudu, dataset:tpch, compression_codec: none, compression_type: none

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36b4ea6f/tests/metadata/test_refresh_partition.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_refresh_partition.py b/tests/metadata/test_refresh_partition.py
new file mode 100644
index 0000000..92f9a19
--- /dev/null
+++ b/tests/metadata/test_refresh_partition.py
@@ -0,0 +1,257 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from subprocess import check_call
+
+from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
+from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.test_dimensions import create_single_exec_option_dimension
+from tests.common.test_dimensions import create_uncompressed_text_dimension
+from tests.common.skip import SkipIfS3, SkipIfIsilon, SkipIfLocal
+from tests.util.filesystem_utils import get_fs_path
+
+
+@SkipIfS3.hive
+@SkipIfIsilon.hive
+@SkipIfLocal.hive
+class TestRefreshPartition(ImpalaTestSuite):
+  """
+  This class tests the functionality to refresh a partition individually
+  for a table in HDFS
+  """
+
+  @classmethod
+  def get_workload(self):
+    return 'functional-query'
+
+  @classmethod
+  def add_test_dimensions(cls):
+    super(TestRefreshPartition, cls).add_test_dimensions()
+
+    # There is no reason to run these tests using all dimensions.
+    cls.TestMatrix.add_dimension(create_single_exec_option_dimension())
+    cls.TestMatrix.add_dimension(
+        create_uncompressed_text_dimension(cls.get_workload()))
+
+  def impala_partition_names(self, table_name):
+    """
+    Find the names of the partitions of a table, as Impala sees them.
+    The return format is a list of lists of strings. Each string represents
+    a partition value of a given column.
+    """
+    rows = self.client.execute('show partitions %s' %
+                               table_name).get_data().split('\n')
+    """
+    According to the output of 'show partitions' query, the first (n-8)
+    columns are the columns on which the table is partitioned
+    """
+    return [row.split('\t')[0:-8] for row in rows[:-1]]
+
+  def hive_partition_names(self, table_name):
+    """
+    Find the names of the partitions of a table, as Hive sees them.
+    The return format is a list of strings. Each string represents a partition
+    value of a given column in a format like 'column1=7/column2=8'.
+    """
+    return self.run_stmt_in_hive(
+        'show partitions %s' % table_name).split('\n')[1:-1]
+
+  def test_add_hive_partition_and_refresh(self, vector, unique_database):
+    """
+    Partition added in Hive can be viewed in Impala after refreshing
+    partition.
+    """
+    table_name = unique_database + '.' + "partition_test_table"
+    self.client.execute(
+        'create table %s (x int) partitioned by (y int, z int)' %
+        table_name)
+    assert [] == self.impala_partition_names(table_name)
+    self.run_stmt_in_hive(
+        'alter table %s add partition (y=333, z=5309)' % table_name)
+    # Make sure Impala can't see the partition yet
+    assert [] == self.impala_partition_names(table_name)
+    self.client.execute('refresh %s partition (y=333, z=5309)' % table_name)
+    # Impala can see the partition
+    assert [['333', '5309']] == self.impala_partition_names(table_name)
+    # Impala's refresh didn't alter Hive's knowledge of the partition
+    assert ['y=333/z=5309'] == self.hive_partition_names(table_name)
+
+  def test_drop_hive_partition_and_refresh(self, vector, unique_database):
+    """
+    Partition dropped in Hive is removed in Impala as well after refreshing
+    partition.
+    """
+    table_name = unique_database + '.' + "partition_test_table"
+    self.client.execute(
+        'create table %s (x int) partitioned by (y int, z int)' %
+        table_name)
+    self.client.execute(
+        'alter table %s add partition (y=333, z=5309)' % table_name)
+    assert [['333', '5309']] == self.impala_partition_names(table_name)
+    self.run_stmt_in_hive(
+        'alter table %s drop partition (y=333, z=5309)' % table_name)
+    # Make sure Impala can still see the partition
+    assert [['333', '5309']] == self.impala_partition_names(table_name)
+    self.client.execute('refresh %s partition (y=333, z=5309)' % table_name)
+    # Impala can see the partition is not there anymore
+    assert [] == self.impala_partition_names(table_name)
+    # Impala's refresh didn't alter Hive's knowledge of the partition
+    assert [] == self.hive_partition_names(table_name)
+
+  def test_add_data_and_refresh(self, vector, unique_database):
+    """
+    Data added through hive is visible in impala after refresh of partition.
+    """
+    table_name = unique_database + '.' + "partition_test_table"
+    self.client.execute(
+        'create table %s (x int) partitioned by (y int, z int)' %
+        table_name)
+    self.client.execute(
+        'alter table %s add partition (y=333, z=5309)' % table_name)
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str('0')]
+    self.run_stmt_in_hive(
+        'insert into table %s partition (y=333, z=5309) values (2)'
+        % table_name)
+    # Make sure its still shows the same result before refreshing
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str('0')]
+
+    self.client.execute('refresh %s partition (y=333, z=5309)' % table_name)
+    assert '2\t333\t5309' == self.client.execute(
+        'select * from %s' % table_name).get_data()
+
+  def test_refresh_invalid_partition(self, vector, unique_database):
+    """
+    Trying to refresh a partition that does not exist does not modify anything
+    either in impala or hive.
+    """
+    table_name = unique_database + '.' + "partition_test_table"
+    self.client.execute(
+        'create table %s (x int) partitioned by (y int, z int)' %
+        table_name)
+    self.client.execute(
+        'alter table %s add partition (y=333, z=5309)' % table_name)
+    assert [['333', '5309']] == self.impala_partition_names(table_name)
+    assert ['y=333/z=5309'] == self.hive_partition_names(table_name)
+    self.client.execute('refresh %s partition (y=71, z=8857)' % table_name)
+    assert [['333', '5309']] == self.impala_partition_names(table_name)
+    assert ['y=333/z=5309'] == self.hive_partition_names(table_name)
+
+  def test_remove_data_and_refresh(self, vector, unique_database):
+    """
+    Data removed through hive is visible in impala after refresh of partition.
+    """
+    expected_error = 'Error(2): No such file or directory'
+    table_name = unique_database + '.' + "partition_test_table"
+    self.client.execute(
+        'create table %s (x int) partitioned by (y int, z int)' %
+        table_name)
+    self.client.execute(
+        'alter table %s add partition (y=333, z=5309)' % table_name)
+    self.client.execute(
+        'insert into table %s partition (y=333, z=5309) values (2)' % table_name)
+    assert '2\t333\t5309' == self.client.execute(
+        'select * from %s' % table_name).get_data()
+
+    self.run_stmt_in_hive(
+        'alter table %s drop partition (y=333, z=5309)' % table_name)
+        # Query the table and check for expected error.
+    try:
+      self.client.execute("select * from %s" % table_name)
+      assert False, "Query was expected to fail"
+    except ImpalaBeeswaxException as e:
+      assert expected_error in str(e)
+
+    self.client.execute('refresh %s partition (y=333, z=5309)' % table_name)
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str('0')]
+
+  def test_add_delete_data_to_hdfs_and_refresh(self, vector, unique_database):
+    """
+    Data added/deleted directly in HDFS is visible in impala after refresh of
+    partition.
+    """
+    table_name = unique_database + '.' + "partition_test_table"
+    table_location = get_fs_path("/test-warehouse/%s" % unique_database)
+    file_name = "alltypes.parq"
+    src_file = get_fs_path("/test-warehouse/alltypesagg_parquet/year=2010/month=1/"
+      "day=9/*.parq")
+    file_num_rows = 1000
+    self.client.execute("""
+      create table %s like functional.alltypes stored as parquet
+      location '%s'
+    """ % (table_name, table_location))
+    self.client.execute("alter table %s add partition (year=2010, month=1)" %
+        table_name)
+    self.client.execute("refresh %s" % table_name)
+    # Check that there is no data in table
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str(0)]
+    dst_path = "%s/year=2010/month=1/%s" % (table_location, file_name)
+    check_call(["hadoop", "fs", "-cp", "-f", src_file, dst_path], shell=False)
+    # Check that data added is not visible before refresh
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str(0)]
+    # Chech that data is visible after refresh
+    self.client.execute("refresh %s partition (year=2010, month=1)" % table_name)
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str(file_num_rows)]
+    # Check that after deleting the file and refreshing, it returns zero rows
+    check_call(["hadoop", "fs", "-rm", dst_path], shell=False)
+    self.client.execute("refresh %s partition (year=2010, month=1)" % table_name)
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str(0)]
+
+  def test_confirm_individual_refresh(self, vector, unique_database):
+    """
+    Data added directly to HDFS is only visible for the partition refreshed
+    """
+    table_name = unique_database + '.' + "partition_test_table"
+    table_location = get_fs_path("/test-warehouse/%s" % unique_database)
+    file_name = "alltypes.parq"
+    src_file = get_fs_path("/test-warehouse/alltypesagg_parquet/year=2010/month=1/"
+      "day=9/*.parq")
+    file_num_rows = 1000
+    self.client.execute("""
+      create table %s like functional.alltypes stored as parquet
+      location '%s'
+    """ % (table_name, table_location))
+    for month in [1, 2]:
+        self.client.execute("alter table %s add partition (year=2010, month=%s)" %
+        (table_name, month))
+    self.client.execute("refresh %s" % table_name)
+    # Check that there is no data in table
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str(0)]
+    dst_path = table_location + "/year=2010/month=%s/" + file_name
+    for month in [1, 2]:
+        check_call(["hadoop", "fs", "-cp", "-f", src_file, dst_path % month],
+                   shell=False)
+    # Check that data added is not visible before refresh
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str(0)]
+    # Check that data is visible after refresh on the first partition only
+    self.client.execute("refresh %s partition (year=2010, month=1)" %
+        table_name)
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str(file_num_rows)]
+    # Check that the data is not yet visible for the second partition
+    # that was not refreshed
+    result = self.client.execute(
+        "select count(*) from %s where year=2010 and month=2" % table_name)
+    assert result.data == [str(0)]
+    # Check that data is visible for the second partition after refresh
+    self.client.execute("refresh %s partition (year=2010, month=2)" % table_name)
+    result = self.client.execute("select count(*) from %s" % table_name)
+    assert result.data == [str(file_num_rows*2)]


[2/8] incubator-impala git commit: Remove caveat about ASF git repo being unbuildable.

Posted by ab...@apache.org.
Remove caveat about ASF git repo being unbuildable.

While there, update some links to the ASF versions.

Change-Id: I009e287a6130325f4541fc8530818dc382b63456
Reviewed-on: http://gerrit.cloudera.org:8080/3806
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a9971300
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a9971300
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a9971300

Branch: refs/heads/master
Commit: a9971300e40e1004ef7800c15857988c486a34c4
Parents: 34bc6a7
Author: Jim Apple <jb...@cloudera.com>
Authored: Thu Jul 28 10:34:37 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Jul 28 23:24:55 2016 +0000

----------------------------------------------------------------------
 README.md | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a9971300/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index d875a79..b7778e9 100644
--- a/README.md
+++ b/README.md
@@ -19,15 +19,8 @@ you analyze, transform and combine data from a variety of data sources:
 ## More about Impala
 
 To learn more about Impala as a business user, or to try Impala live or in a VM, please
-visit the [Impala homepage](http://impala.io/index.html).
+visit the [Impala homepage](https://impala.apache.org).
 
 If you are interested in contributing to Impala as a developer, or learning more about
 Impala's internals and architecture, visit the
-[Impala wiki](https://github.com/cloudera/Impala/wiki).
-
-## Building Impala
-
-This Apache Incubator repository is currently not buildable but has the complete source
-code for Impala minus some third-party dependences. See https://github.com/cloudera/Impala
-for the buildable Impala source and https://issues.cloudera.org/browse/IMPALA-3223 to
-track progress on making this repository buildable.
+[Impala wiki](https://cwiki.apache.org/confluence/display/IMPALA/Impala+Home).
\ No newline at end of file


[3/8] incubator-impala git commit: Add TPC-H based planner tests for Kudu tables

Posted by ab...@apache.org.
Add TPC-H based planner tests for Kudu tables

This commit adds a set of planner tests for Kudu tables based on the 22
TPC-H queries.

Change-Id: I6c40534b72b9aa1ee582b9679c2a63cad52df703
Reviewed-on: http://gerrit.cloudera.org:8080/3790
Reviewed-by: Dimitris Tsirogiannis <dt...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/0e88f0d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/0e88f0d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/0e88f0d7

Branch: refs/heads/master
Commit: 0e88f0d7aa3450c6bf036a6559827c5152bfcb03
Parents: a997130
Author: Dimitris Tsirogiannis <dt...@cloudera.com>
Authored: Mon Jul 18 22:27:28 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Fri Jul 29 02:49:50 2016 +0000

----------------------------------------------------------------------
 .../impala/planner/KuduPlannerTest.java         |    3 +
 .../queries/PlannerTest/tpch-kudu.test          | 1265 ++++++++++++++++++
 2 files changed, 1268 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0e88f0d7/fe/src/test/java/com/cloudera/impala/planner/KuduPlannerTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/planner/KuduPlannerTest.java b/fe/src/test/java/com/cloudera/impala/planner/KuduPlannerTest.java
index fa58f79..000359b 100644
--- a/fe/src/test/java/com/cloudera/impala/planner/KuduPlannerTest.java
+++ b/fe/src/test/java/com/cloudera/impala/planner/KuduPlannerTest.java
@@ -43,4 +43,7 @@ public class KuduPlannerTest extends PlannerTestBase {
     options.setExplain_level(TExplainLevel.VERBOSE);
     runPlannerTestFile("kudu-selectivity", options);
   }
+
+  @Test
+  public void testTpch() { runPlannerTestFile("tpch-kudu"); }
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0e88f0d7/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
new file mode 100644
index 0000000..ddd37d4
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
@@ -0,0 +1,1265 @@
+# Q1 - Pricing Summary Report Query
+select
+  l_returnflag,
+  l_linestatus,
+  round(sum(l_quantity), 1) as sum_qty,
+  round(sum(l_extendedprice), 1) as sum_base_price,
+  round(sum(l_extendedprice * (1 - l_discount)), 1) as sum_disc_price,
+  round(sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)), 1) as sum_charge,
+  round(avg(l_quantity), 1) as avg_qty,
+  round(avg(l_extendedprice), 1) as avg_price,
+  round(avg(l_discount), 1) as avg_disc,
+  count(*) as count_order
+from
+  tpch_kudu.lineitem
+where
+  l_shipdate <= '1998-09-02'
+group by
+  l_returnflag,
+  l_linestatus
+order by
+  l_returnflag,
+  l_linestatus
+---- PLAN
+02:SORT
+|  order by: l_returnflag ASC, l_linestatus ASC
+|
+01:AGGREGATE [FINALIZE]
+|  output: sum(l_quantity), sum(l_extendedprice), sum(l_extendedprice * (1 - l_discount)), sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)), avg(l_quantity), avg(l_extendedprice), avg(l_discount), count(*)
+|  group by: l_returnflag, l_linestatus
+|
+00:SCAN KUDU [tpch_kudu.lineitem]
+   kudu predicates: l_shipdate <= '1998-09-02'
+====
+# Q2 - Minimum Cost Supplier Query
+select
+  round(s_acctbal, 2),
+  s_name,
+  n_name,
+  p_partkey,
+  p_mfgr,
+  s_address,
+  s_phone,
+  s_comment
+from
+  tpch_kudu.part,
+  tpch_kudu.supplier,
+  tpch_kudu.partsupp,
+  tpch_kudu.nation,
+  tpch_kudu.region
+where
+  p_partkey = ps_partkey
+  and s_suppkey = ps_suppkey
+  and p_size = 15
+  and p_type like '%BRASS'
+  and s_nationkey = n_nationkey
+  and n_regionkey = r_regionkey
+  and r_name = 'EUROPE'
+  and ps_supplycost = (
+    select
+      min(ps_supplycost)
+    from
+      tpch_kudu.partsupp,
+      tpch_kudu.supplier,
+      tpch_kudu.nation,
+      tpch_kudu.region
+    where
+      p_partkey = ps_partkey
+      and s_suppkey = ps_suppkey
+      and s_nationkey = n_nationkey
+      and n_regionkey = r_regionkey
+      and r_name = 'EUROPE'
+    )
+order by
+  s_acctbal desc,
+  n_name,
+  s_name,
+  p_partkey
+limit 100
+---- PLAN
+18:TOP-N [LIMIT=100]
+|  order by: s_acctbal DESC, n_name ASC, s_name ASC, p_partkey ASC
+|
+17:HASH JOIN [RIGHT SEMI JOIN]
+|  hash predicates: min(ps_supplycost) = ps_supplycost, ps_partkey = p_partkey
+|
+|--16:HASH JOIN [INNER JOIN]
+|  |  hash predicates: n_regionkey = r_regionkey
+|  |
+|  |--04:SCAN KUDU [tpch_kudu.region]
+|  |     kudu predicates: r_name = 'EUROPE'
+|  |
+|  15:HASH JOIN [INNER JOIN]
+|  |  hash predicates: s_nationkey = n_nationkey
+|  |
+|  |--03:SCAN KUDU [tpch_kudu.nation]
+|  |
+|  14:HASH JOIN [INNER JOIN]
+|  |  hash predicates: ps_suppkey = s_suppkey
+|  |
+|  |--01:SCAN KUDU [tpch_kudu.supplier]
+|  |
+|  13:HASH JOIN [INNER JOIN]
+|  |  hash predicates: ps_partkey = p_partkey
+|  |
+|  |--00:SCAN KUDU [tpch_kudu.part]
+|  |     predicates: p_type LIKE '%BRASS'
+|  |     kudu predicates: p_size = 15
+|  |
+|  02:SCAN KUDU [tpch_kudu.partsupp]
+|
+12:AGGREGATE [FINALIZE]
+|  output: min(ps_supplycost)
+|  group by: ps_partkey
+|
+11:HASH JOIN [INNER JOIN]
+|  hash predicates: n_regionkey = r_regionkey
+|
+|--08:SCAN KUDU [tpch_kudu.region]
+|     kudu predicates: r_name = 'EUROPE'
+|
+10:HASH JOIN [INNER JOIN]
+|  hash predicates: s_nationkey = n_nationkey
+|
+|--07:SCAN KUDU [tpch_kudu.nation]
+|
+09:HASH JOIN [INNER JOIN]
+|  hash predicates: ps_suppkey = s_suppkey
+|
+|--06:SCAN KUDU [tpch_kudu.supplier]
+|
+05:SCAN KUDU [tpch_kudu.partsupp]
+====
+# Q3 - Shipping Priority Query
+select
+  l_orderkey,
+  round(sum(l_extendedprice * (1 - l_discount)), 2) as revenue,
+  o_orderdate,
+  o_shippriority
+from
+  tpch_kudu.customer,
+  tpch_kudu.orders,
+  tpch_kudu.lineitem
+where
+  c_mktsegment = 'BUILDING'
+  and c_custkey = o_custkey
+  and l_orderkey = o_orderkey
+  and o_orderdate < '1995-03-15'
+  and l_shipdate > '1995-03-15'
+group by
+  l_orderkey,
+  o_orderdate,
+  o_shippriority
+order by
+  revenue desc,
+  o_orderdate
+limit 10
+---- PLAN
+06:TOP-N [LIMIT=10]
+|  order by: round(sum(l_extendedprice * (1 - l_discount)), 2) DESC, o_orderdate ASC
+|
+05:AGGREGATE [FINALIZE]
+|  output: sum(l_extendedprice * (1 - l_discount))
+|  group by: l_orderkey, o_orderdate, o_shippriority
+|
+04:HASH JOIN [INNER JOIN]
+|  hash predicates: o_custkey = c_custkey
+|
+|--00:SCAN KUDU [tpch_kudu.customer]
+|     kudu predicates: c_mktsegment = 'BUILDING'
+|
+03:HASH JOIN [INNER JOIN]
+|  hash predicates: l_orderkey = o_orderkey
+|
+|--01:SCAN KUDU [tpch_kudu.orders]
+|     predicates: o_orderdate < '1995-03-15'
+|
+02:SCAN KUDU [tpch_kudu.lineitem]
+   predicates: l_shipdate > '1995-03-15'
+====
+# Q4 - Order Priority Checking Query
+select
+  o_orderpriority,
+  count(*) as order_count
+from
+  tpch_kudu.orders
+where
+  o_orderdate >= '1993-07-01'
+  and o_orderdate < '1993-10-01'
+  and exists (
+    select
+      *
+    from
+      tpch_kudu.lineitem
+    where
+      l_orderkey = o_orderkey
+      and l_commitdate < l_receiptdate
+    )
+group by
+  o_orderpriority
+order by
+  o_orderpriority
+---- PLAN
+04:SORT
+|  order by: o_orderpriority ASC
+|
+03:AGGREGATE [FINALIZE]
+|  output: count(*)
+|  group by: o_orderpriority
+|
+02:HASH JOIN [RIGHT SEMI JOIN]
+|  hash predicates: l_orderkey = o_orderkey
+|
+|--00:SCAN KUDU [tpch_kudu.orders]
+|     predicates: o_orderdate < '1993-10-01'
+|     kudu predicates: o_orderdate >= '1993-07-01'
+|
+01:SCAN KUDU [tpch_kudu.lineitem]
+   predicates: l_commitdate < l_receiptdate
+====
+# Q5 - Local Supplier Volume Query
+select
+  n_name,
+  round(sum(l_extendedprice * (1 - l_discount)), 2) as revenue
+from
+  tpch_kudu.customer,
+  tpch_kudu.orders,
+  tpch_kudu.lineitem,
+  tpch_kudu.supplier,
+  tpch_kudu.nation,
+  tpch_kudu.region
+where
+  c_custkey = o_custkey
+  and l_orderkey = o_orderkey
+  and l_suppkey = s_suppkey
+  and c_nationkey = s_nationkey
+  and s_nationkey = n_nationkey
+  and n_regionkey = r_regionkey
+  and r_name = 'ASIA'
+  and o_orderdate >= '1994-01-01'
+  and o_orderdate < '1995-01-01'
+group by
+  n_name
+order by
+  revenue desc
+---- PLAN
+12:SORT
+|  order by: round(sum(l_extendedprice * (1 - l_discount)), 2) DESC
+|
+11:AGGREGATE [FINALIZE]
+|  output: sum(l_extendedprice * (1 - l_discount))
+|  group by: n_name
+|
+10:HASH JOIN [INNER JOIN]
+|  hash predicates: n_regionkey = r_regionkey
+|
+|--05:SCAN KUDU [tpch_kudu.region]
+|     kudu predicates: r_name = 'ASIA'
+|
+09:HASH JOIN [INNER JOIN]
+|  hash predicates: s_nationkey = n_nationkey
+|
+|--04:SCAN KUDU [tpch_kudu.nation]
+|
+08:HASH JOIN [INNER JOIN]
+|  hash predicates: l_suppkey = s_suppkey, c_nationkey = s_nationkey
+|
+|--03:SCAN KUDU [tpch_kudu.supplier]
+|
+07:HASH JOIN [INNER JOIN]
+|  hash predicates: o_custkey = c_custkey
+|
+|--00:SCAN KUDU [tpch_kudu.customer]
+|
+06:HASH JOIN [INNER JOIN]
+|  hash predicates: l_orderkey = o_orderkey
+|
+|--01:SCAN KUDU [tpch_kudu.orders]
+|     predicates: o_orderdate < '1995-01-01'
+|     kudu predicates: o_orderdate >= '1994-01-01'
+|
+02:SCAN KUDU [tpch_kudu.lineitem]
+====
+# Q6 - Forecasting Revenue Change Query
+select
+ round(sum(l_extendedprice * l_discount), 2) as revenue
+from
+  tpch_kudu.lineitem
+where
+  l_shipdate >= '1994-01-01'
+  and l_shipdate < '1995-01-01'
+  and l_discount between 0.05 and 0.07
+  and l_quantity < 24
+---- PLAN
+01:AGGREGATE [FINALIZE]
+|  output: sum(l_extendedprice * l_discount)
+|
+00:SCAN KUDU [tpch_kudu.lineitem]
+   predicates: l_quantity < 24, l_shipdate < '1995-01-01'
+   kudu predicates: l_discount >= 0.05, l_discount <= 0.07, l_shipdate >= '1994-01-01'
+====
+# Q7 - Volume Shipping Query
+select
+  supp_nation,
+  cust_nation,
+  l_year,
+  round(sum(volume), 2) as revenue
+from (
+  select
+    n1.n_name as supp_nation,
+    n2.n_name as cust_nation,
+    year(l_shipdate) as l_year,
+    round(l_extendedprice * (1 - l_discount), 2) as volume
+  from
+    tpch_kudu.supplier,
+    tpch_kudu.lineitem,
+    tpch_kudu.orders,
+    tpch_kudu.customer,
+    tpch_kudu.nation n1,
+    tpch_kudu.nation n2
+  where
+    s_suppkey = l_suppkey
+    and o_orderkey = l_orderkey
+    and c_custkey = o_custkey
+    and s_nationkey = n1.n_nationkey
+    and c_nationkey = n2.n_nationkey
+    and (
+      (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY')
+      or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE')
+    )
+    and l_shipdate between '1995-01-01' and '1996-12-31'
+  ) as shipping
+group by
+  supp_nation,
+  cust_nation,
+  l_year
+order by
+  supp_nation,
+  cust_nation,
+  l_year
+---- PLAN
+12:SORT
+|  order by: supp_nation ASC, cust_nation ASC, l_year ASC
+|
+11:AGGREGATE [FINALIZE]
+|  output: sum(round(l_extendedprice * (1 - l_discount), 2))
+|  group by: n1.n_name, n2.n_name, year(l_shipdate)
+|
+10:HASH JOIN [INNER JOIN]
+|  hash predicates: c_nationkey = n2.n_nationkey
+|  other predicates: ((n1.n_name = 'FRANCE' AND n2.n_name = 'GERMANY') OR (n1.n_name = 'GERMANY' AND n2.n_name = 'FRANCE'))
+|
+|--05:SCAN KUDU [tpch_kudu.nation n2]
+|
+09:HASH JOIN [INNER JOIN]
+|  hash predicates: s_nationkey = n1.n_nationkey
+|
+|--04:SCAN KUDU [tpch_kudu.nation n1]
+|
+08:HASH JOIN [INNER JOIN]
+|  hash predicates: o_custkey = c_custkey
+|
+|--03:SCAN KUDU [tpch_kudu.customer]
+|
+07:HASH JOIN [INNER JOIN]
+|  hash predicates: l_suppkey = s_suppkey
+|
+|--00:SCAN KUDU [tpch_kudu.supplier]
+|
+06:HASH JOIN [INNER JOIN]
+|  hash predicates: l_orderkey = o_orderkey
+|
+|--02:SCAN KUDU [tpch_kudu.orders]
+|
+01:SCAN KUDU [tpch_kudu.lineitem]
+   kudu predicates: l_shipdate >= '1995-01-01', l_shipdate <= '1996-12-31'
+====
+# Q8 - National Market Share Query
+select
+  o_year,
+  round(sum(case
+    when nation = 'BRAZIL'
+    then volume
+    else 0
+  end) / sum(volume), 4) as mkt_share
+from (
+  select
+    year(o_orderdate) as o_year,
+    round(l_extendedprice * (1 - l_discount), 2) as volume,
+    n2.n_name as nation
+  from
+    tpch_kudu.part,
+    tpch_kudu.supplier,
+    tpch_kudu.lineitem,
+    tpch_kudu.orders,
+    tpch_kudu.customer,
+    tpch_kudu.nation n1,
+    tpch_kudu.nation n2,
+    tpch_kudu.region
+  where
+    p_partkey = l_partkey
+    and s_suppkey = l_suppkey
+    and l_orderkey = o_orderkey
+    and o_custkey = c_custkey
+    and c_nationkey = n1.n_nationkey
+    and n1.n_regionkey = r_regionkey
+    and r_name = 'AMERICA'
+    and s_nationkey = n2.n_nationkey
+    and o_orderdate between '1995-01-01' and '1996-12-31'
+    and p_type = 'ECONOMY ANODIZED STEEL'
+  ) as all_nations
+group by
+  o_year
+order by
+  o_year
+---- PLAN
+16:SORT
+|  order by: o_year ASC
+|
+15:AGGREGATE [FINALIZE]
+|  output: sum(CASE WHEN n2.n_name = 'BRAZIL' THEN round(l_extendedprice * (1 - l_discount), 2) ELSE 0 END), sum(round(l_extendedprice * (1 - l_discount), 2))
+|  group by: year(o_orderdate)
+|
+14:HASH JOIN [INNER JOIN]
+|  hash predicates: s_nationkey = n2.n_nationkey
+|
+|--06:SCAN KUDU [tpch_kudu.nation n2]
+|
+13:HASH JOIN [INNER JOIN]
+|  hash predicates: n1.n_regionkey = r_regionkey
+|
+|--07:SCAN KUDU [tpch_kudu.region]
+|     kudu predicates: r_name = 'AMERICA'
+|
+12:HASH JOIN [INNER JOIN]
+|  hash predicates: c_nationkey = n1.n_nationkey
+|
+|--05:SCAN KUDU [tpch_kudu.nation n1]
+|
+11:HASH JOIN [INNER JOIN]
+|  hash predicates: o_custkey = c_custkey
+|
+|--04:SCAN KUDU [tpch_kudu.customer]
+|
+10:HASH JOIN [INNER JOIN]
+|  hash predicates: l_suppkey = s_suppkey
+|
+|--01:SCAN KUDU [tpch_kudu.supplier]
+|
+09:HASH JOIN [INNER JOIN]
+|  hash predicates: l_orderkey = o_orderkey
+|
+|--03:SCAN KUDU [tpch_kudu.orders]
+|     kudu predicates: o_orderdate >= '1995-01-01', o_orderdate <= '1996-12-31'
+|
+08:HASH JOIN [INNER JOIN]
+|  hash predicates: l_partkey = p_partkey
+|
+|--00:SCAN KUDU [tpch_kudu.part]
+|     kudu predicates: p_type = 'ECONOMY ANODIZED STEEL'
+|
+02:SCAN KUDU [tpch_kudu.lineitem]
+====
+# Q9 - Product Type Measure Query
+select
+  nation,
+  o_year,
+  round(sum(amount), 2) as sum_profit
+from(
+  select
+    n_name as nation,
+    year(o_orderdate) as o_year,
+    round(l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity, 2) as amount
+  from
+    tpch_kudu.part,
+    tpch_kudu.supplier,
+    tpch_kudu.lineitem,
+    tpch_kudu.partsupp,
+    tpch_kudu.orders,
+    tpch_kudu.nation
+  where
+    s_suppkey = l_suppkey
+    and ps_suppkey = l_suppkey
+    and ps_partkey = l_partkey
+    and p_partkey = l_partkey
+    and o_orderkey = l_orderkey
+    and s_nationkey = n_nationkey
+    and p_name like '%green%'
+  ) as profit
+group by
+  nation,
+  o_year
+order by
+  nation,
+  o_year desc
+---- PLAN
+12:SORT
+|  order by: nation ASC, o_year DESC
+|
+11:AGGREGATE [FINALIZE]
+|  output: sum(round(l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity, 2))
+|  group by: n_name, year(o_orderdate)
+|
+10:HASH JOIN [INNER JOIN]
+|  hash predicates: s_nationkey = n_nationkey
+|
+|--05:SCAN KUDU [tpch_kudu.nation]
+|
+09:HASH JOIN [INNER JOIN]
+|  hash predicates: l_suppkey = ps_suppkey, l_partkey = ps_partkey
+|
+|--03:SCAN KUDU [tpch_kudu.partsupp]
+|
+08:HASH JOIN [INNER JOIN]
+|  hash predicates: l_suppkey = s_suppkey
+|
+|--01:SCAN KUDU [tpch_kudu.supplier]
+|
+07:HASH JOIN [INNER JOIN]
+|  hash predicates: l_orderkey = o_orderkey
+|
+|--04:SCAN KUDU [tpch_kudu.orders]
+|
+06:HASH JOIN [INNER JOIN]
+|  hash predicates: l_partkey = p_partkey
+|
+|--00:SCAN KUDU [tpch_kudu.part]
+|     predicates: p_name LIKE '%green%'
+|
+02:SCAN KUDU [tpch_kudu.lineitem]
+====
+# Q10 - Returned Item Reporting Query
+# Converted select from multiple tables to joins
+select
+  c_custkey,
+  c_name,
+  round(sum(l_extendedprice * (1 - l_discount)), 1) as revenue,
+  cast (c_acctbal as bigint),
+  n_name,
+  c_address,
+  c_phone,
+  c_comment
+from
+  tpch_kudu.customer,
+  tpch_kudu.orders,
+  tpch_kudu.lineitem,
+  tpch_kudu.nation
+where
+  c_custkey = o_custkey
+  and l_orderkey = o_orderkey
+  and o_orderdate >= '1993-10-01'
+  and o_orderdate < '1994-01-01'
+  and l_returnflag = 'R'
+  and c_nationkey = n_nationkey
+group by
+  c_custkey,
+  c_name,
+  cast(c_acctbal as bigint),
+  c_phone,
+  n_name,
+  c_address,
+  c_comment
+order by
+  revenue desc
+limit 20
+---- PLAN
+08:TOP-N [LIMIT=20]
+|  order by: round(sum(l_extendedprice * (1 - l_discount)), 1) DESC
+|
+07:AGGREGATE [FINALIZE]
+|  output: sum(l_extendedprice * (1 - l_discount))
+|  group by: c_custkey, c_name, CAST(c_acctbal AS BIGINT), c_phone, n_name, c_address, c_comment
+|
+06:HASH JOIN [INNER JOIN]
+|  hash predicates: c_nationkey = n_nationkey
+|
+|--03:SCAN KUDU [tpch_kudu.nation]
+|
+05:HASH JOIN [INNER JOIN]
+|  hash predicates: o_custkey = c_custkey
+|
+|--00:SCAN KUDU [tpch_kudu.customer]
+|
+04:HASH JOIN [INNER JOIN]
+|  hash predicates: l_orderkey = o_orderkey
+|
+|--01:SCAN KUDU [tpch_kudu.orders]
+|     predicates: o_orderdate < '1994-01-01'
+|     kudu predicates: o_orderdate >= '1993-10-01'
+|
+02:SCAN KUDU [tpch_kudu.lineitem]
+   kudu predicates: l_returnflag = 'R'
+====
+# Q11 - Important Stock Identification
+# Modifications: query was rewritten to not have a subquery in the having clause
+select
+  *
+from (
+  select
+    ps_partkey,
+    round(sum(ps_supplycost * ps_availqty), 2) as value
+  from
+    tpch_kudu.partsupp,
+    tpch_kudu.supplier,
+    tpch_kudu.nation
+  where
+    ps_suppkey = s_suppkey
+    and s_nationkey = n_nationkey
+    and n_name = 'GERMANY'
+  group by
+    ps_partkey
+) as inner_query
+where
+  value > (
+    select
+      round(sum(ps_supplycost * ps_availqty) * 0.0001, 2)
+    from
+      tpch_kudu.partsupp,
+      tpch_kudu.supplier,
+      tpch_kudu.nation
+    where
+      ps_suppkey = s_suppkey
+      and s_nationkey = n_nationkey
+      and n_name = 'GERMANY'
+  )
+order by
+  value desc
+---- PLAN
+13:SORT
+|  order by: value DESC
+|
+12:NESTED LOOP JOIN [INNER JOIN]
+|  predicates: round(sum(ps_supplycost * ps_availqty), 2) > round(sum(ps_supplycost * ps_availqty) * 0.0001, 2)
+|
+|--11:AGGREGATE [FINALIZE]
+|  |  output: sum(ps_supplycost * ps_availqty)
+|  |
+|  10:HASH JOIN [INNER JOIN]
+|  |  hash predicates: s_nationkey = n_nationkey
+|  |
+|  |--08:SCAN KUDU [tpch_kudu.nation]
+|  |     kudu predicates: n_name = 'GERMANY'
+|  |
+|  09:HASH JOIN [INNER JOIN]
+|  |  hash predicates: ps_suppkey = s_suppkey
+|  |
+|  |--07:SCAN KUDU [tpch_kudu.supplier]
+|  |
+|  06:SCAN KUDU [tpch_kudu.partsupp]
+|
+05:AGGREGATE [FINALIZE]
+|  output: sum(ps_supplycost * ps_availqty)
+|  group by: ps_partkey
+|
+04:HASH JOIN [INNER JOIN]
+|  hash predicates: s_nationkey = n_nationkey
+|
+|--02:SCAN KUDU [tpch_kudu.nation]
+|     kudu predicates: n_name = 'GERMANY'
+|
+03:HASH JOIN [INNER JOIN]
+|  hash predicates: ps_suppkey = s_suppkey
+|
+|--01:SCAN KUDU [tpch_kudu.supplier]
+|
+00:SCAN KUDU [tpch_kudu.partsupp]
+====
+# Q12 - Shipping Mode and Order Priority Query
+select
+  l_shipmode,
+  sum(case
+    when o_orderpriority = '1-URGENT'
+      or o_orderpriority = '2-HIGH'
+    then 1
+    else 0
+  end) as high_line_count,
+  sum(case
+    when o_orderpriority <> '1-URGENT'
+      and o_orderpriority <> '2-HIGH'
+    then 1
+    else 0
+  end) as low_line_count
+from
+  tpch_kudu.orders,
+  tpch_kudu.lineitem
+where
+  o_orderkey = l_orderkey
+  and l_shipmode in ('MAIL', 'SHIP')
+  and l_commitdate < l_receiptdate
+  and l_shipdate < l_commitdate
+  and l_receiptdate >= '1994-01-01'
+  and l_receiptdate < '1995-01-01'
+group by
+  l_shipmode
+order by
+  l_shipmode
+---- PLAN
+04:SORT
+|  order by: l_shipmode ASC
+|
+03:AGGREGATE [FINALIZE]
+|  output: sum(CASE WHEN o_orderpriority = '1-URGENT' OR o_orderpriority = '2-HIGH' THEN 1 ELSE 0 END), sum(CASE WHEN o_orderpriority != '1-URGENT' AND o_orderpriority != '2-HIGH' THEN 1 ELSE 0 END)
+|  group by: l_shipmode
+|
+02:HASH JOIN [INNER JOIN]
+|  hash predicates: o_orderkey = l_orderkey
+|
+|--01:SCAN KUDU [tpch_kudu.lineitem]
+|     predicates: l_shipmode IN ('MAIL', 'SHIP'), l_commitdate < l_receiptdate, l_shipdate < l_commitdate, l_receiptdate < '1995-01-01'
+|     kudu predicates: l_receiptdate >= '1994-01-01'
+|
+00:SCAN KUDU [tpch_kudu.orders]
+====
+# Q13 - Customer Distribution Query
+select
+  c_count,
+  count(*) as custdist
+from (
+  select
+    c_custkey,
+    count(o_orderkey) as c_count
+  from
+    tpch_kudu.customer left outer join tpch_kudu.orders on (
+      c_custkey = o_custkey
+      and o_comment not like '%special%requests%'
+    )
+  group by
+    c_custkey
+  ) as c_orders
+group by
+  c_count
+order by
+  custdist desc,
+  c_count desc
+---- PLAN
+05:SORT
+|  order by: count(*) DESC, c_count DESC
+|
+04:AGGREGATE [FINALIZE]
+|  output: count(*)
+|  group by: count(o_orderkey)
+|
+03:AGGREGATE [FINALIZE]
+|  output: count(o_orderkey)
+|  group by: c_custkey
+|
+02:HASH JOIN [RIGHT OUTER JOIN]
+|  hash predicates: o_custkey = c_custkey
+|
+|--00:SCAN KUDU [tpch_kudu.customer]
+|
+01:SCAN KUDU [tpch_kudu.orders]
+   predicates: NOT o_comment LIKE '%special%requests%'
+====
+# Q14 - Promotion Effect
+select
+  round(100.00 * sum(case
+    when p_type like 'PROMO%'
+    then l_extendedprice * (1 - l_discount)
+    else 0.0
+    end) / sum(l_extendedprice * (1 - l_discount)), 4) as promo_revenue
+from
+  tpch_kudu.lineitem,
+  tpch_kudu.part
+where
+  l_partkey = p_partkey
+  and l_shipdate >= '1995-09-01'
+  and l_shipdate < '1995-10-01'
+---- PLAN
+03:AGGREGATE [FINALIZE]
+|  output: sum(CASE WHEN p_type LIKE 'PROMO%' THEN l_extendedprice * (1 - l_discount) ELSE 0.0 END), sum(l_extendedprice * (1 - l_discount))
+|
+02:HASH JOIN [INNER JOIN]
+|  hash predicates: l_partkey = p_partkey
+|
+|--01:SCAN KUDU [tpch_kudu.part]
+|
+00:SCAN KUDU [tpch_kudu.lineitem]
+   predicates: l_shipdate < '1995-10-01'
+   kudu predicates: l_shipdate >= '1995-09-01'
+====
+# Q15 - Top Supplier Query
+with revenue_view as (
+  select
+    l_suppkey as supplier_no,
+    round(sum(l_extendedprice * (1 - l_discount)), 1) as total_revenue
+  from
+    tpch_kudu.lineitem
+  where
+    l_shipdate >= '1996-01-01'
+    and l_shipdate < '1996-04-01'
+  group by
+    l_suppkey)
+select
+  s_suppkey,
+  s_name,
+  s_address,
+  s_phone,
+  total_revenue
+from
+  tpch_kudu.supplier,
+  revenue_view
+where
+  s_suppkey = supplier_no
+  and total_revenue = (
+    select
+      max(total_revenue)
+    from
+      revenue_view
+    )
+order by
+  s_suppkey
+---- PLAN
+08:SORT
+|  order by: s_suppkey ASC
+|
+07:HASH JOIN [LEFT SEMI JOIN]
+|  hash predicates: round(sum(l_extendedprice * (1 - l_discount)), 1) = max(total_revenue)
+|
+|--05:AGGREGATE [FINALIZE]
+|  |  output: max(round(sum(l_extendedprice * (1 - l_discount)), 1))
+|  |
+|  04:AGGREGATE [FINALIZE]
+|  |  output: sum(l_extendedprice * (1 - l_discount))
+|  |  group by: l_suppkey
+|  |
+|  03:SCAN KUDU [tpch_kudu.lineitem]
+|     predicates: l_shipdate < '1996-04-01'
+|     kudu predicates: l_shipdate >= '1996-01-01'
+|
+06:HASH JOIN [INNER JOIN]
+|  hash predicates: s_suppkey = l_suppkey
+|
+|--02:AGGREGATE [FINALIZE]
+|  |  output: sum(l_extendedprice * (1 - l_discount))
+|  |  group by: l_suppkey
+|  |
+|  01:SCAN KUDU [tpch_kudu.lineitem]
+|     predicates: l_shipdate < '1996-04-01'
+|     kudu predicates: l_shipdate >= '1996-01-01'
+|
+00:SCAN KUDU [tpch_kudu.supplier]
+====
+# Q16 - Parts/Supplier Relation Query
+select
+  p_brand,
+  p_type,
+  p_size,
+  count(distinct ps_suppkey) as supplier_cnt
+from
+  tpch_kudu.partsupp,
+  tpch_kudu.part
+where
+  p_partkey = ps_partkey
+  and p_brand <> 'Brand#45'
+  and p_type not like 'MEDIUM POLISHED%'
+  and p_size in (49, 14, 23, 45, 19, 3, 36, 9)
+  and ps_suppkey not in (
+    select
+      s_suppkey
+    from
+      tpch_kudu.supplier
+    where
+      s_comment like '%Customer%Complaints%'
+  )
+group by
+  p_brand,
+  p_type,
+  p_size
+order by
+  supplier_cnt desc,
+  p_brand,
+  p_type,
+  p_size
+---- PLAN
+07:SORT
+|  order by: count(ps_suppkey) DESC, p_brand ASC, p_type ASC, p_size ASC
+|
+06:AGGREGATE [FINALIZE]
+|  output: count(ps_suppkey)
+|  group by: p_brand, p_type, p_size
+|
+05:AGGREGATE
+|  group by: p_brand, p_type, p_size, ps_suppkey
+|
+04:HASH JOIN [NULL AWARE LEFT ANTI JOIN]
+|  hash predicates: ps_suppkey = s_suppkey
+|
+|--02:SCAN KUDU [tpch_kudu.supplier]
+|     predicates: s_comment LIKE '%Customer%Complaints%'
+|
+03:HASH JOIN [INNER JOIN]
+|  hash predicates: ps_partkey = p_partkey
+|
+|--01:SCAN KUDU [tpch_kudu.part]
+|     predicates: p_size IN (49, 14, 23, 45, 19, 3, 36, 9), p_brand != 'Brand#45', NOT p_type LIKE 'MEDIUM POLISHED%'
+|
+00:SCAN KUDU [tpch_kudu.partsupp]
+====
+# Q17 - Small-Quantity-Order Revenue Query
+select
+  round(sum(l_extendedprice) / 7.0, 2) as avg_yearly
+from
+  tpch_kudu.lineitem,
+  tpch_kudu.part
+where
+  p_partkey = l_partkey
+  and p_brand = 'Brand#23'
+  and p_container = 'MED BOX'
+  and l_quantity < (
+    select
+      round(0.2 * avg(l_quantity), 2)
+    from
+      tpch_kudu.lineitem
+    where
+      l_partkey = p_partkey
+  )
+---- PLAN
+06:AGGREGATE [FINALIZE]
+|  output: sum(l_extendedprice)
+|
+05:HASH JOIN [RIGHT SEMI JOIN]
+|  hash predicates: l_partkey = p_partkey
+|  other join predicates: l_quantity < round(0.2 * avg(l_quantity), 2)
+|
+|--04:HASH JOIN [INNER JOIN]
+|  |  hash predicates: l_partkey = p_partkey
+|  |
+|  |--01:SCAN KUDU [tpch_kudu.part]
+|  |     kudu predicates: p_container = 'MED BOX', p_brand = 'Brand#23'
+|  |
+|  00:SCAN KUDU [tpch_kudu.lineitem]
+|
+03:AGGREGATE [FINALIZE]
+|  output: avg(l_quantity)
+|  group by: l_partkey
+|
+02:SCAN KUDU [tpch_kudu.lineitem]
+====
+# Q18 - Large Value tpch_kudu.customer Query
+select
+  c_name,
+  c_custkey,
+  o_orderkey,
+  o_orderdate,
+  o_totalprice,
+  round(sum(l_quantity), 2)
+from
+  tpch_kudu.customer,
+  tpch_kudu.orders,
+  tpch_kudu.lineitem
+where
+  o_orderkey in (
+    select
+      l_orderkey
+    from
+      tpch_kudu.lineitem
+    group by
+      l_orderkey
+    having
+      sum(l_quantity) > 300
+    )
+  and c_custkey = o_custkey
+  and o_orderkey = l_orderkey
+group by
+  c_name,
+  c_custkey,
+  o_orderkey,
+  o_orderdate,
+  o_totalprice
+order by
+  o_totalprice desc,
+  o_orderdate
+limit 100
+---- PLAN
+09:TOP-N [LIMIT=100]
+|  order by: o_totalprice DESC, o_orderdate ASC
+|
+08:AGGREGATE [FINALIZE]
+|  output: sum(l_quantity)
+|  group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
+|
+07:HASH JOIN [LEFT SEMI JOIN]
+|  hash predicates: o_orderkey = l_orderkey
+|
+|--04:AGGREGATE [FINALIZE]
+|  |  output: sum(l_quantity)
+|  |  group by: l_orderkey
+|  |  having: sum(l_quantity) > 300
+|  |
+|  03:SCAN KUDU [tpch_kudu.lineitem]
+|
+06:HASH JOIN [INNER JOIN]
+|  hash predicates: o_custkey = c_custkey
+|
+|--00:SCAN KUDU [tpch_kudu.customer]
+|
+05:HASH JOIN [INNER JOIN]
+|  hash predicates: l_orderkey = o_orderkey
+|
+|--01:SCAN KUDU [tpch_kudu.orders]
+|
+02:SCAN KUDU [tpch_kudu.lineitem]
+====
+# Q19 - Discounted Revenue Query
+select
+  round(sum(l_extendedprice * (1 - l_discount)), 2) as revenue
+from
+  tpch_kudu.lineitem,
+  tpch_kudu.part
+where
+  p_partkey = l_partkey
+  and (
+    (
+      p_brand = 'Brand#12'
+      and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG')
+      and l_quantity >= 1 and l_quantity <= 11
+      and p_size between 1 and 5
+      and l_shipmode in ('AIR', 'AIR REG')
+      and l_shipinstruct = 'DELIVER IN PERSON'
+    )
+    or
+    (
+      p_brand = 'Brand#23'
+      and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK')
+      and l_quantity >= 10 and l_quantity <= 20
+      and p_size between 1 and 10
+      and l_shipmode in ('AIR', 'AIR REG')
+      and l_shipinstruct = 'DELIVER IN PERSON'
+    )
+    or
+    (
+      p_brand = 'Brand#34'
+      and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG')
+      and l_quantity >= 20 and l_quantity <= 30
+      and p_size between 1 and 15
+      and l_shipmode in ('AIR', 'AIR REG')
+      and l_shipinstruct = 'DELIVER IN PERSON'
+    )
+  )
+---- PLAN
+03:AGGREGATE [FINALIZE]
+|  output: sum(l_extendedprice * (1 - l_discount))
+|
+02:HASH JOIN [INNER JOIN]
+|  hash predicates: l_partkey = p_partkey
+|  other predicates: ((p_brand = 'Brand#12' AND p_container IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') AND l_quantity >= 1 AND l_quantity <= 11 AND p_size BETWEEN 1 AND 5 AND l_shipmode IN ('AIR', 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#23' AND p_container IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') AND l_quantity >= 10 AND l_quantity <= 20 AND p_size BETWEEN 1 AND 10 AND l_shipmode IN ('AIR', 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#34' AND p_container IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') AND l_quantity >= 20 AND l_quantity <= 30 AND p_size BETWEEN 1 AND 15 AND l_shipmode IN ('AIR', 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON'))
+|
+|--01:SCAN KUDU [tpch_kudu.part]
+|
+00:SCAN KUDU [tpch_kudu.lineitem]
+====
+# Q20 - Potential Part Promotion Query
+select
+  s_name,
+  s_address
+from
+  tpch_kudu.supplier, tpch_kudu.nation
+where
+  s_suppkey in (
+    select
+      ps_suppkey
+    from
+      tpch_kudu.partsupp
+    where
+      ps_partkey in (
+        select
+          p_partkey
+        from
+          tpch_kudu.part
+        where
+          p_name like 'forest%'
+        )
+      and ps_availqty > (
+        select
+          0.5 * sum(l_quantity)
+        from
+          tpch_kudu.lineitem
+        where
+          l_partkey = ps_partkey
+          and l_suppkey = ps_suppkey
+          and l_shipdate >= '1994-01-01'
+          and l_shipdate < '1995-01-01'
+        )
+    )
+  and s_nationkey = n_nationkey
+  and n_name = 'CANADA'
+order by
+  s_name
+---- PLAN
+10:SORT
+|  order by: s_name ASC
+|
+09:HASH JOIN [RIGHT SEMI JOIN]
+|  hash predicates: ps_suppkey = s_suppkey
+|
+|--08:HASH JOIN [INNER JOIN]
+|  |  hash predicates: s_nationkey = n_nationkey
+|  |
+|  |--01:SCAN KUDU [tpch_kudu.nation]
+|  |     kudu predicates: n_name = 'CANADA'
+|  |
+|  00:SCAN KUDU [tpch_kudu.supplier]
+|
+07:HASH JOIN [RIGHT SEMI JOIN]
+|  hash predicates: l_suppkey = ps_suppkey, l_partkey = ps_partkey
+|  other join predicates: ps_availqty > 0.5 * sum(l_quantity)
+|
+|--06:HASH JOIN [LEFT SEMI JOIN]
+|  |  hash predicates: ps_partkey = p_partkey
+|  |
+|  |--03:SCAN KUDU [tpch_kudu.part]
+|  |     predicates: p_name LIKE 'forest%'
+|  |
+|  02:SCAN KUDU [tpch_kudu.partsupp]
+|
+05:AGGREGATE [FINALIZE]
+|  output: sum(l_quantity)
+|  group by: l_partkey, l_suppkey
+|
+04:SCAN KUDU [tpch_kudu.lineitem]
+   predicates: l_shipdate < '1995-01-01'
+   kudu predicates: l_shipdate >= '1994-01-01'
+====
+# Q21 - Suppliers Who Kept Orders Waiting Query
+select
+  s_name,
+  count(*) as numwait
+from
+  tpch_kudu.supplier,
+  tpch_kudu.lineitem l1,
+  tpch_kudu.orders,
+  tpch_kudu.nation
+where
+  s_suppkey = l1.l_suppkey
+  and o_orderkey = l1.l_orderkey
+  and o_orderstatus = 'F'
+  and l1.l_receiptdate > l1.l_commitdate
+  and exists (
+    select
+      *
+    from
+      tpch_kudu.lineitem l2
+    where
+      l2.l_orderkey = l1.l_orderkey
+      and l2.l_suppkey <> l1.l_suppkey
+  )
+  and not exists (
+    select
+      *
+    from
+      tpch_kudu.lineitem l3
+    where
+      l3.l_orderkey = l1.l_orderkey
+      and l3.l_suppkey <> l1.l_suppkey
+      and l3.l_receiptdate > l3.l_commitdate
+  )
+  and s_nationkey = n_nationkey
+  and n_name = 'SAUDI ARABIA'
+group by
+  s_name
+order by
+  numwait desc,
+  s_name
+limit 100
+---- PLAN
+12:TOP-N [LIMIT=100]
+|  order by: count(*) DESC, s_name ASC
+|
+11:AGGREGATE [FINALIZE]
+|  output: count(*)
+|  group by: s_name
+|
+10:HASH JOIN [RIGHT ANTI JOIN]
+|  hash predicates: l3.l_orderkey = l1.l_orderkey
+|  other join predicates: l3.l_suppkey != l1.l_suppkey
+|
+|--09:HASH JOIN [RIGHT SEMI JOIN]
+|  |  hash predicates: l2.l_orderkey = l1.l_orderkey
+|  |  other join predicates: l2.l_suppkey != l1.l_suppkey
+|  |
+|  |--08:HASH JOIN [INNER JOIN]
+|  |  |  hash predicates: s_nationkey = n_nationkey
+|  |  |
+|  |  |--03:SCAN KUDU [tpch_kudu.nation]
+|  |  |     kudu predicates: n_name = 'SAUDI ARABIA'
+|  |  |
+|  |  07:HASH JOIN [INNER JOIN]
+|  |  |  hash predicates: l1.l_suppkey = s_suppkey
+|  |  |
+|  |  |--00:SCAN KUDU [tpch_kudu.supplier]
+|  |  |
+|  |  06:HASH JOIN [INNER JOIN]
+|  |  |  hash predicates: l1.l_orderkey = o_orderkey
+|  |  |
+|  |  |--02:SCAN KUDU [tpch_kudu.orders]
+|  |  |     kudu predicates: o_orderstatus = 'F'
+|  |  |
+|  |  01:SCAN KUDU [tpch_kudu.lineitem l1]
+|  |     predicates: l1.l_receiptdate > l1.l_commitdate
+|  |
+|  04:SCAN KUDU [tpch_kudu.lineitem l2]
+|
+05:SCAN KUDU [tpch_kudu.lineitem l3]
+   predicates: l3.l_receiptdate > l3.l_commitdate
+====
+# Q22 - Global Sales Opportunity Query
+select
+  cntrycode,
+  count(*) as numcust,
+  round(sum(c_acctbal), 2) as totacctbal
+from (
+  select
+    substr(c_phone, 1, 2) as cntrycode,
+    c_acctbal
+  from
+    tpch_kudu.customer
+  where
+    substr(c_phone, 1, 2) in ('13', '31', '23', '29', '30', '18', '17')
+    and c_acctbal > (
+      select
+        round(avg(c_acctbal), 1)
+      from
+        tpch_kudu.customer
+      where
+        c_acctbal > 0.00
+        and substr(c_phone, 1, 2) in ('13', '31', '23', '29', '30', '18', '17')
+      )
+    and not exists (
+      select
+        *
+      from
+        tpch_kudu.orders
+      where
+        o_custkey = c_custkey
+    )
+  ) as custsale
+group by
+  cntrycode
+order by
+  cntrycode
+---- PLAN
+07:SORT
+|  order by: cntrycode ASC
+|
+06:AGGREGATE [FINALIZE]
+|  output: count(*), sum(c_acctbal)
+|  group by: substr(c_phone, 1, 2)
+|
+05:HASH JOIN [RIGHT ANTI JOIN]
+|  hash predicates: o_custkey = c_custkey
+|
+|--04:NESTED LOOP JOIN [INNER JOIN]
+|  |  predicates: c_acctbal > round(avg(c_acctbal), 1)
+|  |
+|  |--02:AGGREGATE [FINALIZE]
+|  |  |  output: avg(c_acctbal)
+|  |  |
+|  |  01:SCAN KUDU [tpch_kudu.customer]
+|  |     predicates: c_acctbal > 0.00, substr(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')
+|  |
+|  00:SCAN KUDU [tpch_kudu.customer]
+|     predicates: substr(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')
+|
+03:SCAN KUDU [tpch_kudu.orders]
+====


[4/8] incubator-impala git commit: IMPALA-3915: Register privilege and audit requests when analyzing resolved table refs.

Posted by ab...@apache.org.
IMPALA-3915: Register privilege and audit requests when analyzing resolved table refs.

The bug: We used to register privilege requests for table refs in TableRef.analyze()
which only got called for unresolved TableRefs. As a result, a reference to a view that
contains a subquery did not get properly authorized, explained as follows.
1. In the first analysis pass the view is replaced by a an InlineViewRef and we
   correctly register an authorizarion request.
2. We rewrite the subquery via the StmtRewriter and wipe the analysis state, but
   preserve the InlineViewRef that replaces the view reference.
3. The rewritten statement is analyzed again, but since an InlineViewRef is
   considered to be resolved, we never call TableRef.analyze(), and hence
   never register an authorization event for the view.

The fix: We now register authorization and auditing events when calling analyze() on a
resolved TableRef (BaseTableRef, InlineViewRef, CollectionTableRef).

Change-Id: I18fa8af9a94ce190c5a3c29c3221c659a2ace659
Reviewed-on: http://gerrit.cloudera.org:8080/3783
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c77fb628
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c77fb628
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c77fb628

Branch: refs/heads/master
Commit: c77fb628f7f1c968e6fca8db6f3ec7d0efd372e6
Parents: 0e88f0d
Author: Alex Behm <al...@cloudera.com>
Authored: Tue Jul 26 16:02:32 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Fri Jul 29 05:16:23 2016 +0000

----------------------------------------------------------------------
 .../com/cloudera/impala/analysis/Analyzer.java  | 108 ++++++++++++++-----
 .../cloudera/impala/analysis/BaseTableRef.java  |  17 +--
 .../impala/analysis/CollectionTableRef.java     |  15 +--
 .../cloudera/impala/analysis/FromClause.java    |   2 +-
 .../cloudera/impala/analysis/InlineViewRef.java |   1 +
 .../com/cloudera/impala/analysis/TableRef.java  |  64 +----------
 .../cloudera/impala/analysis/AuditingTest.java  |  12 ++-
 .../impala/analysis/AuthorizationTest.java      |   9 ++
 .../functional/functional_schema_template.sql   |  10 ++
 .../datasets/functional/schema_constraints.csv  |   2 +
 10 files changed, 136 insertions(+), 104 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/fe/src/main/java/com/cloudera/impala/analysis/Analyzer.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/Analyzer.java b/fe/src/main/java/com/cloudera/impala/analysis/Analyzer.java
index 30e1f26..be3bf0d 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/Analyzer.java
@@ -502,8 +502,11 @@ public class Analyzer {
   /**
    * Resolves the given TableRef into a concrete BaseTableRef, ViewRef or
    * CollectionTableRef. Returns the new resolved table ref or the given table
-   * ref if it is already resolved. Adds audit events and privilege requests for
-   * the database and/or table.
+   * ref if it is already resolved.
+   * Registers privilege requests and throws an AnalysisException if the tableRef's
+   * path could not be resolved. The privilege requests are added to ensure that
+   * an AuthorizationException is preferred over an AnalysisException so as not to
+   * accidentally reveal the non-existence of tables/databases.
    */
   public TableRef resolveTableRef(TableRef tableRef) throws AnalysisException {
     // Return the table if it is already resolved.
@@ -523,8 +526,31 @@ public class Analyzer {
 
     // Resolve the table ref's path and determine what resolved table ref
     // to replace it with.
-    tableRef.analyze(this);
-    Path resolvedPath = tableRef.getResolvedPath();
+    List<String> rawPath = tableRef.getPath();
+    Path resolvedPath = null;
+    try {
+      resolvedPath = resolvePath(tableRef.getPath(), PathType.TABLE_REF);
+    } catch (AnalysisException e) {
+      if (!hasMissingTbls()) {
+        // Register privilege requests to prefer reporting an authorization error over
+        // an analysis error. We should not accidentally reveal the non-existence of a
+        // table/database if the user is not authorized.
+        if (rawPath.size() > 1) {
+          registerPrivReq(new PrivilegeRequestBuilder()
+              .onTable(rawPath.get(0), rawPath.get(1))
+              .allOf(Privilege.SELECT).toRequest());
+        }
+        registerPrivReq(new PrivilegeRequestBuilder()
+            .onTable(getDefaultDb(), rawPath.get(0))
+            .allOf(Privilege.SELECT).toRequest());
+      }
+      throw e;
+    } catch (TableLoadingException e) {
+      throw new AnalysisException(String.format(
+          "Failed to load metadata for table: '%s'", Joiner.on(".").join(rawPath)), e);
+    }
+
+    Preconditions.checkNotNull(resolvedPath);
     if (resolvedPath.destTable() != null) {
       Table table = resolvedPath.destTable();
       Preconditions.checkNotNull(table);
@@ -534,9 +560,9 @@ public class Analyzer {
           table instanceof KuduTable ||
           table instanceof HBaseTable ||
           table instanceof DataSourceTable);
-      return new BaseTableRef(tableRef);
+      return new BaseTableRef(tableRef, resolvedPath);
     } else {
-      return new CollectionTableRef(tableRef);
+      return new CollectionTableRef(tableRef, resolvedPath);
     }
   }
 
@@ -2333,10 +2359,9 @@ public class Analyzer {
    * Privilege level. The privilege request is tracked in the analyzer
    * and authorized post-analysis.
    *
-   * If the database does not exist in the catalog an AnalysisError is thrown.
+   * Registers a new access event if the catalog lookup was successful.
    *
-   * If addAccessEvent is true, this call will add a new entry to accessEvents if the
-   * catalog access was successful. If false, no accessEvent will be added.
+   * If the database does not exist in the catalog an AnalysisError is thrown.
    */
   public Db getDb(String dbName, Privilege privilege) throws AnalysisException {
     return getDb(dbName, privilege, true);
@@ -2487,6 +2512,51 @@ public class Analyzer {
   }
 
   /**
+   * Registers a new PrivilegeRequest in the analyzer. If authErrorMsg_ is set,
+   * the privilege request will be added to the list of "masked" privilege requests,
+   * using authErrorMsg_ as the auth failure error message. Otherwise it will get
+   * added as a normal privilege request that will use the standard error message
+   * on authorization failure.
+   * If enablePrivChecks_ is false, the registration request will be ignored. This
+   * is used when analyzing catalog views since users should be able to query a view
+   * even if they do not have privileges on the underlying tables.
+   */
+  public void registerPrivReq(PrivilegeRequest privReq) {
+    if (!enablePrivChecks_) return;
+
+    if (Strings.isNullOrEmpty(authErrorMsg_)) {
+      globalState_.privilegeReqs.add(privReq);
+    } else {
+      globalState_.maskedPrivilegeReqs.add(Pair.create(privReq, authErrorMsg_));
+    }
+  }
+
+  /**
+   * Registers a table-level SELECT privilege request and an access event for auditing
+   * for the given table. The given table must be a base table or a catalog view (not
+   * a local view).
+   */
+  public void registerAuthAndAuditEvent(Table table, Analyzer analyzer) {
+    // Add access event for auditing.
+    if (table instanceof View) {
+      View view = (View) table;
+      Preconditions.checkState(!view.isLocalView());
+      analyzer.addAccessEvent(new TAccessEvent(
+          table.getFullName(), TCatalogObjectType.VIEW,
+          Privilege.SELECT.toString()));
+    } else {
+      analyzer.addAccessEvent(new TAccessEvent(
+          table.getFullName(), TCatalogObjectType.TABLE,
+          Privilege.SELECT.toString()));
+    }
+    // Add privilege request.
+    TableName tableName = table.getTableName();
+    analyzer.registerPrivReq(new PrivilegeRequestBuilder()
+        .onTable(tableName.getDb(), tableName.getTbl())
+        .allOf(Privilege.SELECT).toRequest());
+  }
+
+  /**
    * Efficiently computes and stores the transitive closure of the value transfer graph
    * of slots. After calling computeValueTransfers(), value transfers between slots can
    * be queried via hasValueTransfer(). Value transfers can be uni-directional due to
@@ -2851,24 +2921,4 @@ public class Analyzer {
       return expectedStr.equals(actualStr);
     }
   }
-
-  /**
-   * Registers a new PrivilegeRequest in the analyzer. If authErrorMsg_ is set,
-   * the privilege request will be added to the list of "masked" privilege requests,
-   * using authErrorMsg_ as the auth failure error message. Otherwise it will get
-   * added as a normal privilege request that will use the standard error message
-   * on authorization failure.
-   * If enablePrivChecks_ is false, the registration request will be ignored. This
-   * is used when analyzing catalog views since users should be able to query a view
-   * even if they do not have privileges on the underlying tables.
-   */
-  public void registerPrivReq(PrivilegeRequest privReq) {
-    if (!enablePrivChecks_) return;
-
-    if (Strings.isNullOrEmpty(authErrorMsg_)) {
-      globalState_.privilegeReqs.add(privReq);
-    } else {
-      globalState_.maskedPrivilegeReqs.add(Pair.create(privReq, authErrorMsg_));
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/fe/src/main/java/com/cloudera/impala/analysis/BaseTableRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/BaseTableRef.java b/fe/src/main/java/com/cloudera/impala/analysis/BaseTableRef.java
index 3b5ecaf..7e9a921 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/BaseTableRef.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/BaseTableRef.java
@@ -14,10 +14,9 @@
 
 package com.cloudera.impala.analysis;
 
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.catalog.HdfsFileFormat;
 import com.cloudera.impala.catalog.HdfsTable;
 import com.cloudera.impala.catalog.Table;
+import com.cloudera.impala.common.AnalysisException;
 import com.google.common.base.Preconditions;
 
 /**
@@ -26,10 +25,16 @@ import com.google.common.base.Preconditions;
  * of a SelectStmt.
  */
 public class BaseTableRef extends TableRef {
-  public BaseTableRef(TableRef tableRef) {
+
+  /**
+   * Create a BaseTableRef from the original unresolved table ref as well as
+   * its resolved path. Sets table aliases and join-related attributes.
+   */
+  public BaseTableRef(TableRef tableRef, Path resolvedPath) {
     super(tableRef);
-    Preconditions.checkState(resolvedPath_.isResolved());
-    Preconditions.checkState(resolvedPath_.isRootedAtTable());
+    Preconditions.checkState(resolvedPath.isResolved());
+    Preconditions.checkState(resolvedPath.isRootedAtTable());
+    resolvedPath_ = resolvedPath;
     // Set implicit aliases if no explicit one was given.
     if (hasExplicitAlias()) return;
     aliases_ = new String[] {
@@ -50,7 +55,7 @@ public class BaseTableRef extends TableRef {
   @Override
   public void analyze(Analyzer analyzer) throws AnalysisException {
     if (isAnalyzed_) return;
-    Preconditions.checkNotNull(getPrivilegeRequirement());
+    analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), analyzer);
     desc_ = analyzer.registerTableRef(this);
     isAnalyzed_ = true;
     analyzeHints(analyzer);

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/fe/src/main/java/com/cloudera/impala/analysis/CollectionTableRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/CollectionTableRef.java b/fe/src/main/java/com/cloudera/impala/analysis/CollectionTableRef.java
index deeabb0..840ca99 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/CollectionTableRef.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/CollectionTableRef.java
@@ -42,12 +42,13 @@ public class CollectionTableRef extends TableRef {
   /////////////////////////////////////////
 
   /**
-   * Create a CollectionTableRef for the given collection type from the original
-   * unresolved table ref. Sets the explicit alias and the join-related attributes
-   * of the new collection table ref from the unresolved table ref.
+   * Create a CollectionTableRef from the original unresolved table ref as well as
+   * its resolved path. Sets table aliases and join-related attributes.
    */
-  public CollectionTableRef(TableRef tableRef) {
+  public CollectionTableRef(TableRef tableRef, Path resolvedPath) {
     super(tableRef);
+    Preconditions.checkState(resolvedPath.isResolved());
+    resolvedPath_ = resolvedPath;
     // Use the last path element as an implicit alias if no explicit alias was given.
     if (hasExplicitAlias()) return;
     String implicitAlias = rawPath_.get(rawPath_.size() - 1).toLowerCase();
@@ -76,7 +77,6 @@ public class CollectionTableRef extends TableRef {
   @Override
   public void analyze(Analyzer analyzer) throws AnalysisException {
     if (isAnalyzed_) return;
-    Preconditions.checkNotNull(getPrivilegeRequirement());
     desc_ = analyzer.registerTableRef(this);
     if (isRelative() && !analyzer.isWithClause()) {
       SlotDescriptor parentSlotDesc = analyzer.registerSlotRef(resolvedPath_);
@@ -97,7 +97,10 @@ public class CollectionTableRef extends TableRef {
       }
     }
     if (!isRelative()) {
-      // Register a column-level privilege request for the collection-typed column.
+      // Register a table-level privilege request as well as a column-level privilege request
+      // for the collection-typed column.
+      Preconditions.checkNotNull(resolvedPath_.getRootTable());
+      analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), analyzer);
       analyzer.registerPrivReq(new PrivilegeRequestBuilder().
           allOf(Privilege.SELECT).onColumn(desc_.getTableName().getDb(),
           desc_.getTableName().getTbl(), desc_.getPath().getRawPath().get(0))

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/fe/src/main/java/com/cloudera/impala/analysis/FromClause.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/FromClause.java b/fe/src/main/java/com/cloudera/impala/analysis/FromClause.java
index 724fa47..6062bc7 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/FromClause.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/FromClause.java
@@ -92,7 +92,7 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
         // Use the fully qualified raw path to preserve the original resolution.
         // Otherwise, non-fully qualified paths might incorrectly match a local view.
         // TODO for 2.3: This full qualification preserves analysis state which is
-        // contraty to the intended semantics of reset(). We could address this issue by
+        // contrary to the intended semantics of reset(). We could address this issue by
         // changing the WITH-clause analysis to register local views that have
         // fully-qualified table refs, and then remove the full qualification here.
         newTblRef.rawPath_ = origTblRef.getResolvedPath().getFullyQualifiedRawPath();

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java b/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java
index e80fe4b..50b712b 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java
@@ -138,6 +138,7 @@ public class InlineViewRef extends TableRef {
     // Catalog views refs require special analysis settings for authorization.
     boolean isCatalogView = (view_ != null && !view_.isLocalView());
     if (isCatalogView) {
+      analyzer.registerAuthAndAuditEvent(view_, analyzer);
       if (inlineViewAnalyzer_.isExplain()) {
         // If the user does not have privileges on the view's definition
         // then we report a masked authorization error so as not to reveal

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java b/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java
index b2ae8a0..c04c3f1 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java
@@ -19,19 +19,12 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
-import com.cloudera.impala.analysis.Path.PathType;
-import com.cloudera.impala.authorization.Privilege;
-import com.cloudera.impala.authorization.PrivilegeRequestBuilder;
 import com.cloudera.impala.catalog.HdfsTable;
 import com.cloudera.impala.catalog.Table;
-import com.cloudera.impala.catalog.TableLoadingException;
-import com.cloudera.impala.catalog.View;
 import com.cloudera.impala.common.AnalysisException;
 import com.cloudera.impala.common.Pair;
 import com.cloudera.impala.planner.JoinNode.DistributionMode;
 import com.cloudera.impala.planner.PlanNode;
-import com.cloudera.impala.thrift.TAccessEvent;
-import com.cloudera.impala.thrift.TCatalogObjectType;
 import com.cloudera.impala.thrift.TReplicaPreference;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
@@ -50,7 +43,7 @@ import com.google.common.collect.Sets;
  * 1. Resolution: A table ref's path is resolved and then the generic TableRef is
  * replaced by a concrete table ref (a BaseTableRef, CollectionTabeRef or ViewRef)
  * in the originating stmt and that is given the resolved path. This step is driven by
- * Analyzer.resolveTableRef() which calls into TableRef.analyze().
+ * Analyzer.resolveTableRef().
  *
  * 2. Analysis/registration: After resolution, the concrete table ref is analyzed
  * to register a tuple descriptor for its resolved path and register other table-ref
@@ -171,56 +164,10 @@ public class TableRef implements ParseNode {
     desc_ = other.desc_;
   }
 
-  /**
-   * Resolves this table ref's raw path and adds privilege requests and audit events
-   * for the referenced catalog entities.
-   */
   @Override
   public void analyze(Analyzer analyzer) throws AnalysisException {
-    try {
-      resolvedPath_ = analyzer.resolvePath(rawPath_, PathType.TABLE_REF);
-    } catch (AnalysisException e) {
-      if (!analyzer.hasMissingTbls()) {
-        // Register privilege requests to prefer reporting an authorization error over
-        // an analysis error. We should not accidentally reveal the non-existence of a
-        // table/database if the user is not authorized.
-        if (rawPath_.size() > 1) {
-          analyzer.registerPrivReq(new PrivilegeRequestBuilder()
-              .onTable(rawPath_.get(0), rawPath_.get(1))
-              .allOf(getPrivilegeRequirement()).toRequest());
-        }
-        analyzer.registerPrivReq(new PrivilegeRequestBuilder()
-            .onTable(analyzer.getDefaultDb(), rawPath_.get(0))
-            .allOf(getPrivilegeRequirement()).toRequest());
-      }
-      throw e;
-    } catch (TableLoadingException e) {
-      throw new AnalysisException(String.format(
-          "Failed to load metadata for table: '%s'", Joiner.on(".").join(rawPath_)), e);
-    }
-    Preconditions.checkState(resolvedPath_ != null);
-    if (resolvedPath_.isRootedAtTable()) {
-      // Add access event for auditing.
-      Table table = resolvedPath_.getRootTable();
-      if (table instanceof View) {
-        View view = (View) table;
-        if (!view.isLocalView()) {
-          analyzer.addAccessEvent(new TAccessEvent(
-              table.getFullName(), TCatalogObjectType.VIEW,
-              getPrivilegeRequirement().toString()));
-        }
-      } else {
-        analyzer.addAccessEvent(new TAccessEvent(
-            table.getFullName(), TCatalogObjectType.TABLE,
-            getPrivilegeRequirement().toString()));
-      }
-
-      // Add privilege requests for authorization.
-      TableName tableName = table.getTableName();
-      analyzer.registerPrivReq(new PrivilegeRequestBuilder()
-          .onTable(tableName.getDb(), tableName.getTbl())
-          .allOf(getPrivilegeRequirement()).toRequest());
-    }
+    throw new IllegalStateException(
+        "Should not call analyze() on an unresolved TableRef.");
   }
 
   /**
@@ -617,11 +564,6 @@ public class TableRef implements ParseNode {
   }
 
   /**
-   * Gets the privilege requirement. This is always SELECT for TableRefs.
-   */
-  public Privilege getPrivilegeRequirement() { return Privilege.SELECT; }
-
-  /**
    * Returns a deep clone of this table ref without also cloning the chain of table refs.
    * Sets leftTblRef_ in the returned clone to null.
    */

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/fe/src/test/java/com/cloudera/impala/analysis/AuditingTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/analysis/AuditingTest.java b/fe/src/test/java/com/cloudera/impala/analysis/AuditingTest.java
index 6b3c426..3dbe9b9 100644
--- a/fe/src/test/java/com/cloudera/impala/analysis/AuditingTest.java
+++ b/fe/src/test/java/com/cloudera/impala/analysis/AuditingTest.java
@@ -54,7 +54,17 @@ public class AuditingTest extends AnalyzerTest {
         new TAccessEvent("functional.alltypes", TCatalogObjectType.TABLE, "SELECT")
         ));
 
-    // Select from an inline-view.
+    // Tests audit events after a statement has been rewritten (IMPALA-3915).
+    // Select from a view that contains a subquery.
+    accessEvents = AnalyzeAccessEvents("select * from functional_rc.subquery_view");
+    Assert.assertEquals(accessEvents, Sets.newHashSet(
+        new TAccessEvent("functional_rc.alltypessmall", TCatalogObjectType.TABLE, "SELECT"),
+        new TAccessEvent("functional_rc.alltypes", TCatalogObjectType.TABLE, "SELECT"),
+        new TAccessEvent("functional_rc.subquery_view", TCatalogObjectType.VIEW, "SELECT"),
+        new TAccessEvent("_impala_builtins", TCatalogObjectType.DATABASE, "VIEW_METADATA")
+        ));
+
+    // Select from an inline view.
     accessEvents = AnalyzeAccessEvents(
         "select a.* from (select * from functional.alltypesagg) a");
     Assert.assertEquals(accessEvents, Sets.newHashSet(

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java b/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java
index 99a8d17..9f371c7 100644
--- a/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java
+++ b/fe/src/test/java/com/cloudera/impala/analysis/AuthorizationTest.java
@@ -470,6 +470,15 @@ public class AuthorizationTest {
         "User '%s' does not have privileges to execute 'SELECT' on: " +
         "functional.alltypes");
 
+    // Tests authorization after a statement has been rewritten (IMPALA-3915).
+    // User has SELECT privileges on the view which contains a subquery.
+    AuthzOk("select * from functional_seq_snap.subquery_view");
+
+    // User does not have SELECT privileges on the view which contains a subquery.
+    AuthzError("select * from functional_rc.subquery_view",
+        "User '%s' does not have privileges to execute 'SELECT' on: " +
+        "functional_rc.subquery_view");
+
     // Constant select.
     AuthzOk("select 1");
 

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/testdata/datasets/functional/functional_schema_template.sql
----------------------------------------------------------------------
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index e14492e..44604f1 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -859,6 +859,16 @@ AS SELECT * FROM {db_name}{db_suffix}.alltypes_view;
 ---- DATASET
 functional
 ---- BASE_TABLE_NAME
+subquery_view
+---- CREATE
+CREATE VIEW IF NOT EXISTS {db_name}{db_suffix}.{table_name}
+AS SELECT COUNT(*) FROM {db_name}{db_suffix}.alltypes
+WHERE id IN (SELECT id FROM {db_name}{db_suffix}.alltypessmall where int_col < 5);
+---- LOAD
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
 alltypes_parens
 ---- CREATE
 CREATE VIEW IF NOT EXISTS {db_name}{db_suffix}.{table_name}

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c77fb628/testdata/datasets/functional/schema_constraints.csv
----------------------------------------------------------------------
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index a49d445..29c70fe 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -95,6 +95,8 @@ table_name:complex_view, constraint:restrict_to, table_format:text/none/none
 table_name:complex_view, constraint:restrict_to, table_format:seq/snap/block
 table_name:view_view, constraint:restrict_to, table_format:text/none/none
 table_name:view_view, constraint:restrict_to, table_format:seq/snap/block
+table_name:subquery_view, constraint:restrict_to, table_format:seq/snap/block
+table_name:subquery_view, constraint:restrict_to, table_format:rc/none/none
 
 # liketbl and tblwithraggedcolumns all have
 # NULLs in primary key columns. hbase does not support


[6/8] incubator-impala git commit: Update .gitignore files for ninja, coredumps and pypi packages

Posted by ab...@apache.org.
Update .gitignore files for ninja, coredumps and pypi packages

Change-Id: Ie7d34fbd27150ba6c437207611f71bb95a0e4cba
Reviewed-on: http://gerrit.cloudera.org:8080/3814
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/904265cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/904265cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/904265cc

Branch: refs/heads/master
Commit: 904265ccb5c3f6b4b35a8f0638b1151dbe8fcd32
Parents: 857b94d
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Fri Jul 29 10:19:10 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Fri Jul 29 21:42:07 2016 +0000

----------------------------------------------------------------------
 .gitignore                   | 9 +++++++++
 infra/python/deps/.gitignore | 5 +++++
 2 files changed, 14 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/904265cc/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 0778512..ec52448 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,6 +28,10 @@ CMakeFiles
 cmake_install.cmake
 CTestTestfile.cmake
 !CMakeLists.txt
+.ninja_deps
+.ninja_log
+build.ninja
+rules.ninja
 
 Testing/
 llvm-ir/
@@ -43,3 +47,8 @@ logs/
 
 # Impala toolchain directory
 toolchain/
+
+# Core dumps and crash reports
+core
+core.*
+hs_err_pid*.log

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/904265cc/infra/python/deps/.gitignore
----------------------------------------------------------------------
diff --git a/infra/python/deps/.gitignore b/infra/python/deps/.gitignore
new file mode 100644
index 0000000..809485e
--- /dev/null
+++ b/infra/python/deps/.gitignore
@@ -0,0 +1,5 @@
+# Ignore downloaded python packages
+*.tar.bz2
+*.tar.gz
+*.whl
+*.zip


[5/8] incubator-impala git commit: IMPALA-3857: KuduScanNode race on returning "optional" threads

Posted by ab...@apache.org.
IMPALA-3857: KuduScanNode race on returning "optional" threads

The KuduScanNode could return all active scanner threads
when there were no more "optional" tokens available. In this
case, any remaining scan ranges wouldn't be picked up and
the query would produce incorrect results. This fixes the
issue by cleaning up the ScannerThread code and making sure
not to exit the last thread.

This was tested by running the tpch workload repeatedly
under load. That work to incorporate tpch data loading for
Kudu is actively being worked on, so this was tested manually.

Change-Id: I22adf2109b43b1b37d9a597de85e063431dff155
Reviewed-on: http://gerrit.cloudera.org:8080/3798
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/857b94d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/857b94d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/857b94d0

Branch: refs/heads/master
Commit: 857b94d03cf719da37d6bb95695c57944f004d36
Parents: c77fb62
Author: Matthew Jacobs <mj...@cloudera.com>
Authored: Mon Jul 25 16:59:56 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Fri Jul 29 21:36:50 2016 +0000

----------------------------------------------------------------------
 be/src/exec/kudu-scan-node.cc | 98 +++++++++++++++++++++++---------------
 be/src/exec/kudu-scan-node.h  | 10 ++--
 2 files changed, 66 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/857b94d0/be/src/exec/kudu-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scan-node.cc b/be/src/exec/kudu-scan-node.cc
index a08034f..c2ed16c 100644
--- a/be/src/exec/kudu-scan-node.cc
+++ b/be/src/exec/kudu-scan-node.cc
@@ -408,60 +408,80 @@ void KuduScanNode::ThreadTokenAvailableCb(
   }
 }
 
-void KuduScanNode::ScannerThread(const string& name, const TKuduKeyRange* key_range) {
+Status KuduScanNode::ProcessRange(KuduScanner* scanner, const TKuduKeyRange* key_range) {
+  RETURN_IF_ERROR(scanner->OpenNextRange(*key_range));
+  bool eos = false;
+  while (!eos) {
+    gscoped_ptr<RowBatch> row_batch(new RowBatch(
+          row_desc(), runtime_state_->batch_size(), mem_tracker()));
+    RETURN_IF_ERROR(scanner->GetNext(row_batch.get(), &eos));
+    while (!done_) {
+      scanner->KeepKuduScannerAlive();
+      if (materialized_row_batches_->AddBatchWithTimeout(row_batch.get(), 1000000)) {
+        ignore_result(row_batch.release());
+        break;
+      }
+    }
+  }
+  // Mark the current scan range as complete.
+  if (eos) scan_ranges_complete_counter()->Add(1);
+  return Status::OK();
+}
+
+void KuduScanNode::ScannerThread(const string& name, const TKuduKeyRange* initial_range) {
+  DCHECK(initial_range != NULL);
   SCOPED_THREAD_COUNTER_MEASUREMENT(scanner_thread_counters());
   SCOPED_TIMER(runtime_state_->total_cpu_timer());
 
+  // Set to true if this thread observes that the number of optional threads has been
+  // exceeded and is exiting early.
+  bool optional_thread_exiting = false;
   KuduScanner scanner(this, runtime_state_);
   Status status = scanner.Open();
-  if (!status.ok()) goto done;
-
-  while (!done_) {
-    status = scanner.OpenNextRange(*key_range);
-    if (!status.ok()) goto done;
 
-    // Keep looping through all the ranges.
-    bool eos = false;
-    while (!eos) {
-      // Keep looping through all the rows.
-      gscoped_ptr<RowBatch> row_batch(new RowBatch(
-          row_desc(), runtime_state_->batch_size(), mem_tracker()));
-      status = scanner.GetNext(row_batch.get(), &eos);
-      if (!status.ok()) goto done;
-      while (true) {
-        if (done_) goto done;
-        scanner.KeepKuduScannerAlive();
-        if (materialized_row_batches_->AddBatchWithTimeout(row_batch.get(), 1000000)) {
-          ignore_result(row_batch.release());
+  if (status.ok()) {
+    const TKuduKeyRange* key_range = initial_range;
+    while (!done_ && key_range != NULL) {
+      status = ProcessRange(&scanner, key_range);
+      if (!status.ok()) break;
+
+      // Check if the number of optional threads has been exceeded.
+      if (runtime_state_->resource_pool()->optional_exceeded()) {
+        unique_lock<mutex> l(lock_);
+        // Don't exit if this is the last thread. Otherwise, the scan will indicate it's
+        // done before all ranges have been processed.
+        if (num_active_scanners_ > 1) {
+          --num_active_scanners_;
+          optional_thread_exiting = true;
           break;
         }
       }
+      key_range = GetNextKeyRange();
     }
-    // Mark the current scan range as complete.
-    scan_ranges_complete_counter()->Add(1);
-    if (runtime_state_->resource_pool()->optional_exceeded()) goto done;
-    key_range = GetNextKeyRange();
-    if (key_range == NULL) goto done;
+    scanner.Close();
   }
 
-done:
-  VLOG(1) << "Thread done: " << name;
-  scanner.Close();
-  runtime_state_->resource_pool()->ReleaseThreadToken(false);
-
-  unique_lock<mutex> l(lock_);
-  if (!status.ok()) {
-    if (status_.ok()) {
-      status_ = status;
+  {
+    unique_lock<mutex> l(lock_);
+    if (!status.ok()) {
+      if (status_.ok()) {
+        status_ = status;
+        done_ = true;
+      }
+    }
+    // Decrement num_active_scanners_ unless handling the case of an early exit when
+    // optional threads have been exceeded, in which case it already was decremented.
+    if (!optional_thread_exiting) --num_active_scanners_;
+    if (num_active_scanners_ == 0) {
       done_ = true;
+      materialized_row_batches_->Shutdown();
     }
   }
-  --num_active_scanners_;
-  if (num_active_scanners_ == 0) {
-    // If we got here and we are the last thread, we're all done.
-    done_ = true;
-    materialized_row_batches_->Shutdown();
-  }
+
+  // lock_ is released before calling ThreadResourceMgr::ReleaseThreadToken() which
+  // invokes ThreadTokenAvailableCb() which attempts to take the same lock.
+  VLOG(1) << "Thread done: " << name;
+  runtime_state_->resource_pool()->ReleaseThreadToken(false);
 }
 
 }  // namespace impala

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/857b94d0/be/src/exec/kudu-scan-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scan-node.h b/be/src/exec/kudu-scan-node.h
index 329f52a..5dfb309 100644
--- a/be/src/exec/kudu-scan-node.h
+++ b/be/src/exec/kudu-scan-node.h
@@ -163,11 +163,15 @@ class KuduScanNode : public ScanNode {
   void ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool);
 
   /// Main function for scanner thread which executes a KuduScanner. Begins by processing
-  /// 'initial_range', once that range is completed it fetches more ranges with 'GetNextKeyRange()'
-  /// until there are no more ranges to fetch, an error occurred or the limit has been reached.
-  /// Scanned batches are enqueued in 'materialized_row_batches_'.
+  /// 'initial_range', and continues processing ranges returned by 'GetNextKeyRange()'
+  /// until there are no more ranges, an error occurs, or the limit is reached.
   void ScannerThread(const string& name, const TKuduKeyRange* initial_range);
 
+  /// Processes a single scan range. Row batches are fetched using 'scanner' and enqueued
+  /// in 'materialized_row_batches_' until the scanner reports eos for 'key_range', an
+  /// error occurs, or the limit is reached.
+  Status ProcessRange(KuduScanner* scanner, const TKuduKeyRange* key_range);
+
   /// Returns the next partition key range to read. Thread safe. Returns NULL if there are
   /// no more ranges.
   TKuduKeyRange* GetNextKeyRange();