You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jb...@apache.org on 2016/11/29 22:46:37 UTC
[1/3] incubator-impala git commit: IMPALA-4542: Fix use-after-free in
some BE tests
Repository: incubator-impala
Updated Branches:
refs/heads/master e2cde13a2 -> 26805809f
IMPALA-4542: Fix use-after-free in some BE tests
Change RuntimeState::ReleaseResources() to use cached ExecEnv pointer,
rather than singleton.
Change-Id: I5302d665756183289edf227588da1c094e197584
Testing: Ran affected BE tests under ASAN.
Reviewed-on: http://gerrit.cloudera.org:8080/5243
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/8adc49e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/8adc49e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/8adc49e5
Branch: refs/heads/master
Commit: 8adc49e51c44aa3a4b5334a93b64ae5a21e400b9
Parents: e2cde13
Author: Henry Robinson <he...@cloudera.com>
Authored: Mon Nov 28 10:49:29 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 29 16:36:43 2016 +0000
----------------------------------------------------------------------
be/src/runtime/runtime-state.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8adc49e5/be/src/runtime/runtime-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc
index d0e1172..7954f22 100644
--- a/be/src/runtime/runtime-state.cc
+++ b/be/src/runtime/runtime-state.cc
@@ -311,7 +311,7 @@ void RuntimeState::ReleaseResources() {
if (desc_tbl_ != nullptr) desc_tbl_->ClosePartitionExprs(this);
if (filter_bank_ != nullptr) filter_bank_->Close();
if (resource_pool_ != nullptr) {
- ExecEnv::GetInstance()->thread_mgr()->UnregisterPool(resource_pool_);
+ exec_env_->thread_mgr()->UnregisterPool(resource_pool_);
}
}
}
[3/3] incubator-impala git commit: IMPALA-4512: Add a script that
builds Impala on stock Ubuntu 14.04.
Posted by jb...@apache.org.
IMPALA-4512: Add a script that builds Impala on stock Ubuntu 14.04.
This is a simpler alternative to bootstrap_development.sh - it
acquires enough dependencies to build, but does not attempt to load
the test data or even build the tests. This is sometimes a lightweight
testing method used by Apache PPMC members who are voting on a release
of an incubating project.
Change-Id: If34e398052a61dfda9825b1cf3a918eb61736048
Reviewed-on: http://gerrit.cloudera.org:8080/5154
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/26805809
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/26805809
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/26805809
Branch: refs/heads/master
Commit: 26805809f7fb91537b8068058cb3f7b5dd4fd5de
Parents: 9032952
Author: Jim Apple <jb...@cloudera.com>
Authored: Sat Nov 19 18:18:24 2016 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Nov 29 22:10:58 2016 +0000
----------------------------------------------------------------------
README.md | 2 +-
bin/bootstrap_build.sh | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/26805809/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 9c2d9b1..797e7c5 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ Impala only supports Linux at the moment.
## Build Instructions
-./buildall.sh -notests
+See bin/bootstrap_build.sh.
## Export Control Notice
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/26805809/bin/bootstrap_build.sh
----------------------------------------------------------------------
diff --git a/bin/bootstrap_build.sh b/bin/bootstrap_build.sh
new file mode 100755
index 0000000..c3bd22b
--- /dev/null
+++ b/bin/bootstrap_build.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script builds Impala from scratch. It is known to work on Ubuntu 14.04. To run it
+# you need to have:
+#
+# 1. At least 8GB of free disk space
+# 4. A connection to the internet (parts of the build download dependencies)
+#
+# To bootstrap a more complete development environment that includes not only building
+# Impala but also running and testing it, see bootstrap_development.sh in this directory.
+
+# Set up some logging and exit conditions:
+set -euxo pipefail
+
+# Install dependencies:
+sudo apt-get update
+sudo apt-get --yes install g++ gcc git libsasl2-dev libssl-dev make maven openjdk-7-jdk \
+ python-dev python-setuptools
+
+export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/
+./buildall.sh -notests -so
[2/3] incubator-impala git commit: IMPALA-4000: Restricted Sentry
authorization for Kudu Tables
Posted by jb...@apache.org.
IMPALA-4000: Restricted Sentry authorization for Kudu Tables
At this time, there is no comprehensive way of enforcing a Sentry
authorization policy against tables stored in Kudu. The following
behavior was implemented in this patch:
- Only the ALL privilege level can be granted to Kudu tables.
Finer-grained levels such as only SELECT or only INSERT are not
supported.
- Column level permissions on Kudu tables are not supported.
- Only users with ALL privileges on SERVER may create external Kudu
tables.
Change-Id: I183f08ad8ce80deee011a6b90ad67b9cefc0452c
Reviewed-on: http://gerrit.cloudera.org:8080/5047
Reviewed-by: Taras Bobrovytsky <tb...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/90329527
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/90329527
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/90329527
Branch: refs/heads/master
Commit: 9032952758b00c172a63fb83f6eb1c638f20de5f
Parents: 8adc49e
Author: Taras Bobrovytsky <tb...@cloudera.com>
Authored: Thu Nov 10 19:05:02 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 29 17:06:16 2016 +0000
----------------------------------------------------------------------
.../java/org/apache/impala/analysis/Analyzer.java | 1 +
.../org/apache/impala/analysis/CreateTableStmt.java | 15 +++++++++++++--
.../org/apache/impala/analysis/PrivilegeSpec.java | 12 ++++++++++++
.../authorization/PrivilegeRequestBuilder.java | 8 ++++++++
.../apache/impala/analysis/AnalyzeAuthStmtsTest.java | 11 +++++++++++
.../apache/impala/analysis/AuthorizationTest.java | 13 +++++++++++++
6 files changed, 58 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/90329527/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index fa03434..3ab367c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -151,6 +151,7 @@ public class Analyzer {
private String authErrorMsg_;
// If false, privilege requests will not be registered in the analyzer.
+ // Note: it's not the purpose of this flag to control if security is enabled in general.
private boolean enablePrivChecks_ = true;
// By default, all registered semi-joined tuples are invisible, i.e., their slots
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/90329527/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
index b896acc..f2db81b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.avro.Schema;
import org.apache.avro.SchemaParseException;
+import org.apache.impala.authorization.PrivilegeRequestBuilder;
import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.RowFormat;
import org.apache.impala.common.AnalysisException;
@@ -197,7 +198,7 @@ public class CreateTableStmt extends StatementBase {
analyzeKuduTableProperties(analyzer);
if (isExternal()) {
- analyzeExternalKuduTableParams();
+ analyzeExternalKuduTableParams(analyzer);
} else {
analyzeManagedKuduTableParams(analyzer);
}
@@ -238,7 +239,17 @@ public class CreateTableStmt extends StatementBase {
/**
* Analyzes and checks parameters specified for external Kudu tables.
*/
- private void analyzeExternalKuduTableParams() throws AnalysisException {
+ private void analyzeExternalKuduTableParams(Analyzer analyzer)
+ throws AnalysisException {
+ if (analyzer.getAuthzConfig().isEnabled()) {
+ // Today there is no comprehensive way of enforcing a Sentry authorization policy
+ // against tables stored in Kudu. This is why only users with ALL privileges on
+ // SERVER may create external Kudu tables. See IMPALA-4000 for details.
+ String authzServer = analyzer.getAuthzConfig().getServerName();
+ Preconditions.checkNotNull(authzServer);
+ analyzer.registerPrivReq(new PrivilegeRequestBuilder().onServer(
+ authzServer).all().toRequest());
+ }
AnalysisUtils.throwIfNull(getTblProperties().get(KuduTable.KEY_TABLE_NAME),
String.format("Table property %s must be specified when creating " +
"an external Kudu table.", KuduTable.KEY_TABLE_NAME));
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/90329527/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java b/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
index f064604..b999cda 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.DataSourceTable;
+import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.RolePrivilege;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.TableLoadingException;
@@ -282,6 +283,17 @@ public class PrivilegeSpec implements ParseNode {
"to issue a GRANT/REVOKE statement.", tableName_.toString()));
}
Preconditions.checkNotNull(table);
+ if (table instanceof KuduTable) {
+ // We only support the ALL privilege on Kudu tables since many of the finer-grained
+ // levels (DELETE/UPDATE) are not available. See IMPALA-4000 for details.
+ if (privilegeLevel_ != TPrivilegeLevel.ALL) {
+ throw new AnalysisException("Kudu tables only support the ALL privilege level.");
+ }
+ if (scope_ == TPrivilegeScope.COLUMN) {
+ throw new AnalysisException("Column-level privileges on Kudu " +
+ "tables are not supported.");
+ }
+ }
return table;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/90329527/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java b/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java
index 8cf146c..f47c52f 100644
--- a/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java
+++ b/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java
@@ -54,6 +54,14 @@ public class PrivilegeRequestBuilder {
}
/**
+ * Sets the authorizeable object to be a server.
+ */
+ public PrivilegeRequestBuilder onServer(String serverName) {
+ authorizeable_ = new AuthorizeableServer(serverName);
+ return this;
+ }
+
+ /**
* Sets the authorizeable object to be a database.
*/
public PrivilegeRequestBuilder onDb(String dbName) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/90329527/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
index c37047f..07a1c0a 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
@@ -160,6 +160,13 @@ public class AnalyzeAuthStmtsTest extends AnalyzerTest {
AnalysisError(String.format("%s INSERT ON URI 'hdfs:////abc//123' %s myrole",
formatArgs), "Only 'ALL' privilege may be applied at URI scope in privilege " +
"spec.");
+ // IMPALA-4000: Insert privilege on a Kudu table
+ AnalysisError(String.format(
+ "%s SELECT ON TABLE functional_kudu.alltypessmall %s myrole", formatArgs),
+ "Kudu tables only support the ALL privilege level.");
+ AnalysisError(String.format(
+ "%s INSERT ON TABLE functional_kudu.alltypessmall %s myrole", formatArgs),
+ "Kudu tables only support the ALL privilege level.");
// SELECT privilege
AnalyzesOk(String.format("%s SELECT ON TABLE alltypessmall %s myrole", formatArgs),
@@ -196,6 +203,10 @@ public class AnalyzeAuthStmtsTest extends AnalyzerTest {
AnalysisError(String.format("%s SELECT (id, bool_col) ON TABLE " +
"functional.alltypes_hive_view %s myrole", formatArgs), "Column-level " +
"privileges on views are not supported.");
+ // IMPALA-4000: Column-level privileges on a KUDU table
+ AnalysisError(String.format("%s SELECT (id, bool_col) ON TABLE " +
+ "functional_kudu.alltypessmall %s myrole", formatArgs),
+ "Kudu tables only support the ALL privilege level.");
// Columns/table that don't exist
AnalysisError(String.format("%s SELECT (invalid_col) ON TABLE " +
"functional.alltypes %s myrole", formatArgs), "Error setting column-level " +
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/90329527/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
index 38aff7d..431ac01 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
@@ -903,6 +903,17 @@ public class AuthorizationTest {
"User '%s' does not have privileges to execute 'CREATE' on: " +
"nodb.alltypes");
+ // IMPALA-4000: Only users with ALL privileges on SERVER may create external Kudu
+ // tables.
+ AuthzError("create external table tpch.kudu_tbl stored as kudu " +
+ "TBLPROPERTIES ('kudu.master_addresses'='127.0.0.1', 'kudu.table_name'='tbl')",
+ "User '%s' does not have privileges to access: server1");
+
+ // IMPALA-4000: ALL privileges on SERVER are not required to create managed tables.
+ AuthzOk("create table tpch.kudu_tbl (i int, j int, primary key (i))" +
+ " DISTRIBUTE BY HASH (i) INTO 9 BUCKETS stored as kudu TBLPROPERTIES " +
+ "('kudu.master_addresses'='127.0.0.1')");
+
// User does not have permission to create table at the specified location..
AuthzError("create table tpch.new_table (i int) location " +
"'hdfs://localhost:20500/test-warehouse/alltypes'",
@@ -2154,6 +2165,8 @@ public class AuthorizationTest {
AuthzOk(fe, context, "select * from functional.alltypesagg");
AuthzOk(fe, context, "select * from functional.alltypes");
AuthzOk(fe, context, "invalidate metadata");
+ AuthzOk(fe, context, "create external table tpch.kudu_tbl stored as kudu " +
+ "TBLPROPERTIES ('kudu.master_addresses'='127.0.0.1', 'kudu.table_name'='tbl')");
}
private void TestWithIncorrectConfig(AuthorizationConfig authzConfig, User user)