You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/03/20 20:44:36 UTC
[01/11] impala git commit: IMPALA-6683: Fix infinite loop after
restarting the catalog
Repository: impala
Updated Branches:
refs/heads/master 8dde41e80 -> 1d38c584a
IMPALA-6683: Fix infinite loop after restarting the catalog
Currently the catalog service ID topic item includes the ID string.
It causes the coexistence of multiple catalog service ID topic items
after the catalogd restarts. Impalad therefore keeps detecting the
change of catalog service ID and requests a full catalog update. This
patch uses "CATALOG_SERVICE_ID" as the topic item name instead.
With the patch impalad prints only one line of catalog change log after
the catalog restarts.
Change-Id: I1ee6c6477458e0f4dd31b12daa9ed5f146d84e7b
Reviewed-on: http://gerrit.cloudera.org:8080/9684
Reviewed-by: Alex Behm <al...@cloudera.com>
Reviewed-by: Dimitris Tsirogiannis <dt...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/0532ff97
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/0532ff97
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/0532ff97
Branch: refs/heads/master
Commit: 0532ff97c80aad08e800fede78348653e57a872f
Parents: 8dde41e
Author: Tianyi Wang <tw...@cloudera.com>
Authored: Thu Mar 15 18:25:54 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Mar 16 08:20:40 2018 +0000
----------------------------------------------------------------------
fe/src/main/java/org/apache/impala/catalog/Catalog.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/0532ff97/fe/src/main/java/org/apache/impala/catalog/Catalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Catalog.java b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
index bee5be5..6f9cdb5 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Catalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
@@ -575,8 +575,7 @@ public abstract class Catalog {
case DATA_SOURCE:
return "DATA_SOURCE:" + catalogObject.getData_source().getName().toLowerCase();
case CATALOG:
- return "CATALOG:" +
- TUniqueIdUtil.PrintId(catalogObject.getCatalog().catalog_service_id);
+ return "CATALOG_SERVICE_ID";
default:
throw new IllegalStateException(
"Unsupported catalog object type: " + catalogObject.getType());
[06/11] impala git commit: IMPALA-6589: remove invalid DCHECK in
parquet reader
Posted by ta...@apache.org.
IMPALA-6589: remove invalid DCHECK in parquet reader
The DCHECK was only valid if the Parquet file metadata is internally
consistent, with the number of values reported by the metadata
matching the number of encoded levels.
The DCHECK was intended to directly detect misuse of the RleBatchDecoder
interface, which would lead to incorrect results. However, our other
test coverage for reading Parquet files is sufficient to test the
correctness of level decoding.
Testing:
Added a minimal corrupt test file that reproduces the issue.
Change-Id: Idd6e09f8c8cca8991be5b5b379f6420adaa97daa
Reviewed-on: http://gerrit.cloudera.org:8080/9556
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e148c1a7
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e148c1a7
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e148c1a7
Branch: refs/heads/master
Commit: e148c1a7c3cf67e84cb6be736ed709dd683b7188
Parents: af6769d
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Thu Mar 8 09:32:45 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sat Mar 17 02:52:19 2018 +0000
----------------------------------------------------------------------
be/src/exec/parquet-column-readers.cc | 25 ++++++++++++++++---
be/src/exec/parquet-column-readers.h | 6 +++++
testdata/data/README | 6 +++++
.../data/num_values_def_levels_mismatch.parquet | Bin 0 -> 252 bytes
.../parquet-num-values-def-levels-mismatch.test | 6 +++++
tests/query_test/test_scanners.py | 11 ++++++++
6 files changed, 50 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/e148c1a7/be/src/exec/parquet-column-readers.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/parquet-column-readers.cc b/be/src/exec/parquet-column-readers.cc
index 7cf89ba..61839af 100644
--- a/be/src/exec/parquet-column-readers.cc
+++ b/be/src/exec/parquet-column-readers.cc
@@ -172,7 +172,7 @@ bool ParquetLevelDecoder::FillCache(int batch_size, int* num_cached_levels) {
DCHECK(!CacheHasNext());
DCHECK(num_cached_levels != nullptr);
DCHECK_GE(max_level_, 0);
- DCHECK_EQ(num_cached_levels_ % 32, 0) << "Last batch was not a multiple of 32";
+ DCHECK_GE(*num_cached_levels, 0);
cached_level_idx_ = 0;
if (max_level_ == 0) {
// No levels to read, e.g., because the field is required. The cache was
@@ -580,6 +580,7 @@ class ScalarColumnReader : public BaseScalarColumnReader {
parent_->parse_status_ = Status(TErrorCode::PARQUET_DICT_DECODE_FAILURE, filename(),
slot_desc_->type().DebugString(), stream_->file_offset());
}
+
void __attribute__((noinline)) SetPlainDecodeError() {
parent_->parse_status_ = Status(TErrorCode::PARQUET_CORRUPT_PLAIN_VALUE, filename(),
slot_desc_->type().DebugString(), stream_->file_offset());
@@ -1189,15 +1190,17 @@ bool BaseScalarColumnReader::NextLevels() {
// The compiler can optimize these two conditions into a single branch by treating
// def_level_ as unsigned.
if (UNLIKELY(def_level_ < 0 || def_level_ > max_def_level())) {
- parent_->parse_status_.MergeStatus(Status(Substitute("Corrupt Parquet file '$0': "
- "invalid def level $1 > max def level $2 for column '$3'", filename(),
- def_level_, max_def_level(), schema_element().name)));
+ SetLevelDecodeError("def", def_level_, max_def_level());
return false;
}
if (ADVANCE_REP_LEVEL && max_rep_level() > 0) {
// Repetition level is only present if this column is nested in any collection type.
rep_level_ = rep_levels_.ReadLevel();
+ if (UNLIKELY(rep_level_ < 0 || rep_level_ > max_rep_level())) {
+ SetLevelDecodeError("rep", rep_level_, max_rep_level());
+ return false;
+ }
// Reset position counter if we are at the start of a new parent collection.
if (rep_level_ <= max_rep_level() - 1) pos_current_value_ = 0;
}
@@ -1219,6 +1222,20 @@ bool BaseScalarColumnReader::NextPage() {
return true;
}
+void BaseScalarColumnReader::SetLevelDecodeError(const char* level_name,
+ int decoded_level, int max_level) {
+ if (decoded_level < 0) {
+ DCHECK_EQ(decoded_level, HdfsParquetScanner::INVALID_LEVEL);
+ parent_->parse_status_.MergeStatus(Status(Substitute("Corrupt Parquet file '$0': "
+ "could not read all $1 levels for column '$2'", filename(),
+ level_name, schema_element().name)));
+ } else {
+ parent_->parse_status_.MergeStatus(Status(Substitute("Corrupt Parquet file '$0': "
+ "invalid $1 level $2 > max $1 level $3 for column '$4'", filename(),
+ level_name, decoded_level, max_level, schema_element().name)));
+ }
+}
+
bool CollectionColumnReader::NextLevels() {
DCHECK(!children_.empty());
DCHECK_LE(rep_level_, new_collection_rep_level());
http://git-wip-us.apache.org/repos/asf/impala/blob/e148c1a7/be/src/exec/parquet-column-readers.h
----------------------------------------------------------------------
diff --git a/be/src/exec/parquet-column-readers.h b/be/src/exec/parquet-column-readers.h
index aa90bb8..fbc98ec 100644
--- a/be/src/exec/parquet-column-readers.h
+++ b/be/src/exec/parquet-column-readers.h
@@ -502,6 +502,12 @@ class BaseScalarColumnReader : public ParquetColumnReader {
return page_encoding != parquet::Encoding::PLAIN_DICTIONARY
&& slot_desc_ != nullptr && slot_desc_->type().IsVarLenStringType();
}
+
+ /// Slow-path status construction code for def/rep decoding errors. 'level_name' is
+ /// either "rep" or "def", 'decoded_level' is the value returned from
+ /// ParquetLevelDecoder::ReadLevel() and 'max_level' is the maximum allowed value.
+ void __attribute__((noinline)) SetLevelDecodeError(const char* level_name,
+ int decoded_level, int max_level);
};
/// Collections are not materialized directly in parquet files; only scalar values appear
http://git-wip-us.apache.org/repos/asf/impala/blob/e148c1a7/testdata/data/README
----------------------------------------------------------------------
diff --git a/testdata/data/README b/testdata/data/README
index f02e70b..928d435 100644
--- a/testdata/data/README
+++ b/testdata/data/README
@@ -162,3 +162,9 @@ compression codec. The data in the file is the whole of the "alltypestiny" data
the same columns: id int, bool_col boolean, tinyint_col tinyint, smallint_col smallint,
int_col int, bigint_col bigint, float_col float, double_col double,
date_string_col string, string_col string, timestamp_col timestamp, year int, month int
+
+num_values_def_levels_mismatch.parquet:
+A file with a single boolean column with page metadata reporting 2 values but only def
+levels for a single literal value. Generated by hacking Impala's parquet writer to
+increment page.header.data_page_header.num_values. This caused Impala to hit a DCHECK
+(IMPALA-6589).
http://git-wip-us.apache.org/repos/asf/impala/blob/e148c1a7/testdata/data/num_values_def_levels_mismatch.parquet
----------------------------------------------------------------------
diff --git a/testdata/data/num_values_def_levels_mismatch.parquet b/testdata/data/num_values_def_levels_mismatch.parquet
new file mode 100644
index 0000000..92f6e60
Binary files /dev/null and b/testdata/data/num_values_def_levels_mismatch.parquet differ
http://git-wip-us.apache.org/repos/asf/impala/blob/e148c1a7/testdata/workloads/functional-query/queries/QueryTest/parquet-num-values-def-levels-mismatch.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/parquet-num-values-def-levels-mismatch.test b/testdata/workloads/functional-query/queries/QueryTest/parquet-num-values-def-levels-mismatch.test
new file mode 100644
index 0000000..47fbc1a
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/parquet-num-values-def-levels-mismatch.test
@@ -0,0 +1,6 @@
+====
+---- QUERY
+select * from num_values_def_levels_mismatch
+---- CATCH
+could not read all def levels for column '_c0'
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/e148c1a7/tests/query_test/test_scanners.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index a9ba5b8..d2014d1 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -430,6 +430,17 @@ class TestParquet(ImpalaTestSuite):
"/testdata/data/bad_codec.parquet", tbl_loc])
self.run_test_case('QueryTest/parquet-bad-codec', vector, unique_database)
+ def test_num_values_def_levels_mismatch(self, vector, unique_database):
+ """IMPALA-6589: test the bad num_values handled correctly. """
+ self.client.execute(("""CREATE TABLE {0}.num_values_def_levels_mismatch (_c0 BOOLEAN)
+ STORED AS PARQUET""").format(unique_database))
+ tbl_loc = get_fs_path("/test-warehouse/%s.db/%s" % (unique_database,
+ "num_values_def_levels_mismatch"))
+ check_call(['hdfs', 'dfs', '-copyFromLocal', os.environ['IMPALA_HOME'] +
+ "/testdata/data/num_values_def_levels_mismatch.parquet", tbl_loc])
+ self.run_test_case('QueryTest/parquet-num-values-def-levels-mismatch',
+ vector, unique_database)
+
@SkipIfS3.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
[04/11] impala git commit: IMPALA-6655: Add owner information on
database creation
Posted by ta...@apache.org.
IMPALA-6655: Add owner information on database creation
Add owner information on database creation.
> create database foo;
> describe database extended foo;
+---------+----------+---------+
| name | location | comment |
+---------+----------+---------+
| foo | | |
| Owner: | | |
| | user1 | USER |
+---------+----------+---------+
Testing:
- Ran end-to-end query and metadata tests
Change-Id: Id74ec9bd3cb7954999305e9cd9085cbf50921a78
Reviewed-on: http://gerrit.cloudera.org:8080/9637
Reviewed-by: Fredy Wijaya <fw...@cloudera.com>
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/41a516f9
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/41a516f9
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/41a516f9
Branch: refs/heads/master
Commit: 41a516f949223a090bf4e2702993d760c3fa87d5
Parents: 2f50818
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Tue Mar 13 16:31:23 2018 -0500
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Mar 16 19:28:37 2018 +0000
----------------------------------------------------------------------
common/thrift/JniCatalog.thrift | 3 +++
.../java/org/apache/impala/analysis/CreateDbStmt.java | 11 +++++++++++
.../org/apache/impala/service/CatalogOpExecutor.java | 3 +++
.../functional-query/queries/QueryTest/describe-db.test | 10 ++++++++++
tests/common/impala_test_suite.py | 3 ++-
5 files changed, 29 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/41a516f9/common/thrift/JniCatalog.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/JniCatalog.thrift b/common/thrift/JniCatalog.thrift
index 4edf4d2..371475b 100644
--- a/common/thrift/JniCatalog.thrift
+++ b/common/thrift/JniCatalog.thrift
@@ -88,6 +88,9 @@ struct TCreateDbParams {
// Do not throw an error if a database of the same name already exists.
4: optional bool if_not_exists
+
+ // Owner of the database
+ 5: required string owner
}
// Parameters of CREATE DATA SOURCE commands
http://git-wip-us.apache.org/repos/asf/impala/blob/41a516f9/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
index c803910..f71f269 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
@@ -17,6 +17,7 @@
package org.apache.impala.analysis;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.Db;
@@ -32,6 +33,8 @@ public class CreateDbStmt extends StatementBase {
private final HdfsUri location_;
private final String comment_;
private final boolean ifNotExists_;
+ // Database owner. Set during analysis.
+ private String owner_;
/**
* Creates a database with the given name.
@@ -74,6 +77,7 @@ public class CreateDbStmt extends StatementBase {
params.setComment(getComment());
params.setLocation(location_ == null ? null : location_.toString());
params.setIf_not_exists(getIfNotExists());
+ params.setOwner(getOwner());
return params;
}
@@ -96,5 +100,12 @@ public class CreateDbStmt extends StatementBase {
if (location_ != null) {
location_.analyze(analyzer, Privilege.ALL, FsAction.READ_WRITE);
}
+ owner_ = analyzer.getUser().getName();
}
+
+ /**
+ * Can only be called after analysis, returns the owner of this database (the user from
+ * the current session).
+ */
+ public String getOwner() { return Preconditions.checkNotNull(owner_); }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/41a516f9/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index c3b15a1..9ccfa50 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
@@ -956,6 +957,8 @@ public class CatalogOpExecutor {
if (params.getLocation() != null) {
db.setLocationUri(params.getLocation());
}
+ db.setOwnerName(params.getOwner());
+ db.setOwnerType(PrincipalType.USER);
if (LOG.isTraceEnabled()) LOG.trace("Creating database " + dbName);
Db newDb = null;
synchronized (metastoreDdlLock_) {
http://git-wip-us.apache.org/repos/asf/impala/blob/41a516f9/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
index 8f55325..027e19a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
@@ -31,6 +31,8 @@ string, string, string
---- QUERY
describe database extended impala_test_desc_db1
---- RESULTS
+'','$USER','USER'
+'Owner: ','',''
'impala_test_desc_db1','$NAMENODE/test-warehouse/impala_test_desc_db1.db',''
---- TYPES
string, string, string
@@ -38,6 +40,8 @@ string, string, string
---- QUERY
describe database extended impala_test_desc_db2
---- RESULTS
+'','$USER','USER'
+'Owner: ','',''
'impala_test_desc_db2','$NAMENODE/test-warehouse/impala_test_desc_db2.db','test comment'
---- TYPES
string, string, string
@@ -45,6 +49,8 @@ string, string, string
---- QUERY
describe database extended impala_test_desc_db3
---- RESULTS
+'','$USER','USER'
+'Owner: ','',''
'impala_test_desc_db3','$NAMENODE/testdb',''
---- TYPES
string, string, string
@@ -52,6 +58,8 @@ string, string, string
---- QUERY
describe database extended impala_test_desc_db4
---- RESULTS
+'','$USER','USER'
+'Owner: ','',''
'impala_test_desc_db4','$NAMENODE/test2.db','test comment'
---- TYPES
string, string, string
@@ -59,6 +67,8 @@ string, string, string
---- QUERY
describe database formatted impala_test_desc_db4
---- RESULTS
+'','$USER','USER'
+'Owner: ','',''
'impala_test_desc_db4','$NAMENODE/test2.db','test comment'
---- TYPES
string, string, string
http://git-wip-us.apache.org/repos/asf/impala/blob/41a516f9/tests/common/impala_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index bdd524f..28d9c2e 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -291,7 +291,8 @@ class ImpalaTestSuite(BaseTestSuite):
replace_filenames_with_placeholder = False
test_section[section_name] = test_section[section_name] \
.replace('$NAMENODE', NAMENODE) \
- .replace('$IMPALA_HOME', IMPALA_HOME)
+ .replace('$IMPALA_HOME', IMPALA_HOME) \
+ .replace('$USER', getuser())
if use_db:
test_section[section_name] = test_section[section_name].replace('$DATABASE', use_db)
verify_raw_results(test_section, result, vector.get_value('table_format').file_format,
[10/11] impala git commit: IMPALA-6610: Improve LDAP auth fail
warning message in impala-shell
Posted by ta...@apache.org.
IMPALA-6610: Improve LDAP auth fail warning message in impala-shell
The value of LDAP password in Impala shell contains extra line break
causes authentication failure, but the user can't detect the cause of
the failure.
I fixed the issue by adding inspection to the password for common
pitfalls and issuing a warning in the shell when authentication fails.
Change-Id: Ie570166aea62af223905b7f0124e9efb15a88ac7
Reviewed-on: http://gerrit.cloudera.org:8080/9506
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e6aadc9c
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e6aadc9c
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e6aadc9c
Branch: refs/heads/master
Commit: e6aadc9c35c0a5adfde7c888dd349c46752764e1
Parents: 4fbe4cb
Author: davidxdh <xu...@zte.com.cn>
Authored: Tue Mar 6 17:11:16 2018 +0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Mar 20 06:47:30 2018 +0000
----------------------------------------------------------------------
shell/impala_shell.py | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/e6aadc9c/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index abbe9eb..4e57677 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -818,6 +818,11 @@ class ImpalaShell(object, cmd.Cmd):
print_to_stderr("Socket error %s: %s" % (code, e))
self.prompt = self.DISCONNECTED_PROMPT
except Exception, e:
+ if options.ldap_password_cmd and \
+ options.ldap_password and \
+ options.ldap_password.endswith('\n'):
+ print_to_stderr("Warning: LDAP password contains a trailing newline. "
+ "Did you use 'echo' instead of 'echo -n'?")
print_to_stderr("Error connecting: %s, %s" % (type(e).__name__, e))
# A secure connection may still be open. So we explicitly close it.
self.imp_client.close_connection()
[05/11] impala git commit: IMPALA-6690: Fix pip_download.py on python
2.6
Posted by ta...@apache.org.
IMPALA-6690: Fix pip_download.py on python 2.6
IMPALA-6682 used set literal syntax in pip_download.py, which is
introduced in python 2.7. This patch changes it to set constructor.
It's tested on python 2.6.9.
Change-Id: I82b4116ee056f605c8aadf39a8b92b78313cb8bf
Reviewed-on: http://gerrit.cloudera.org:8080/9694
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/af6769d9
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/af6769d9
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/af6769d9
Branch: refs/heads/master
Commit: af6769d95a35f66af191a54e676401411b766988
Parents: 41a516f
Author: Tianyi Wang <tw...@cloudera.com>
Authored: Fri Mar 16 14:40:40 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sat Mar 17 01:32:22 2018 +0000
----------------------------------------------------------------------
infra/python/deps/pip_download.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/af6769d9/infra/python/deps/pip_download.py
----------------------------------------------------------------------
diff --git a/infra/python/deps/pip_download.py b/infra/python/deps/pip_download.py
index 5a79b80..65f55be 100755
--- a/infra/python/deps/pip_download.py
+++ b/infra/python/deps/pip_download.py
@@ -44,7 +44,7 @@ def check_digest(filename, algorithm, expected_digest):
supported_algorithms = hashlib.algorithms_available
except AttributeError:
# Fallback to hardcoded set if hashlib.algorithms_available doesn't exist.
- supported_algorithms = {'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'}
+ supported_algorithms = set(['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'])
if algorithm not in supported_algorithms:
print 'Hash algorithm {0} is not supported by hashlib'.format(algorithm)
return False
[09/11] impala git commit: IMPALA-6697: Downgrade setuptools to be
compatible with Python 2.6
Posted by ta...@apache.org.
IMPALA-6697: Downgrade setuptools to be compatible with Python 2.6
Change-Id: I0d4727b7a5911269b82287ed9ce759f1e211f386
Reviewed-on: http://gerrit.cloudera.org:8080/9713
Reviewed-by: Philip Zeyliger <ph...@cloudera.com>
Tested-by: Lars Volker <lv...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/4fbe4cb2
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/4fbe4cb2
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/4fbe4cb2
Branch: refs/heads/master
Commit: 4fbe4cb208c3d5a044fc64f151d496475524eda9
Parents: b1ef7de
Author: Lars Volker <lv...@cloudera.com>
Authored: Sun Mar 18 10:29:41 2018 -0700
Committer: Lars Volker <lv...@cloudera.com>
Committed: Sun Mar 18 23:31:17 2018 +0000
----------------------------------------------------------------------
infra/python/deps/requirements.txt | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/4fbe4cb2/infra/python/deps/requirements.txt
----------------------------------------------------------------------
diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt
index 29cc9ce..abf5d7d 100644
--- a/infra/python/deps/requirements.txt
+++ b/infra/python/deps/requirements.txt
@@ -55,7 +55,8 @@ python-magic == 0.4.11
pywebhdfs == 0.3.2
pbr == 1.8.1
requests == 2.7.0
-setuptools == 39.0.0
+# Newer versions of setuptools don't support Python 2.6
+setuptools == 36.8.0
setuptools-scm == 1.15.4
sh == 1.11
sqlparse == 0.1.15
[08/11] impala git commit: IMPALA-6695: Fix PyPi regex,
update setuptools version
Posted by ta...@apache.org.
IMPALA-6695: Fix PyPi regex, update setuptools version
pytest-runner, which is required by kudu-python requires are more recent
version of setuptools. Adding an explicit dependency required an update
to the regular expression to parse PyPi URLs.
Change-Id: Ia67189f81a31a9a5a0ed80cd4d6661762ef427b2
Reviewed-on: http://gerrit.cloudera.org:8080/9711
Reviewed-by: Philip Zeyliger <ph...@cloudera.com>
Tested-by: Lars Volker <lv...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/b1ef7de0
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/b1ef7de0
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/b1ef7de0
Branch: refs/heads/master
Commit: b1ef7de0e75b50f6268d73114938ed94d03ad939
Parents: 5c8da5d
Author: Lars Volker <lv...@cloudera.com>
Authored: Sun Mar 18 00:05:42 2018 -0700
Committer: Lars Volker <lv...@cloudera.com>
Committed: Sun Mar 18 16:39:32 2018 +0000
----------------------------------------------------------------------
infra/python/deps/pip_download.py | 2 +-
infra/python/deps/requirements.txt | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/b1ef7de0/infra/python/deps/pip_download.py
----------------------------------------------------------------------
diff --git a/infra/python/deps/pip_download.py b/infra/python/deps/pip_download.py
index 65f55be..2e84426 100755
--- a/infra/python/deps/pip_download.py
+++ b/infra/python/deps/pip_download.py
@@ -87,7 +87,7 @@ def get_package_info(pkg_name, pkg_version):
# downloading an extra package before running this script. Since the HTML is guaranteed
# to be formatted according to PEP 503, this is acceptable.
pkg_info = urlopen(url).read()
- regex = r'<a href=\".*?packages/(.*?)#(.*?)=(.*?)\".*?>(.*?)<\/a>'
+ regex = r'<a .*?href=\".*?packages/(.*?)#(.*?)=(.*?)\".*?>(.*?)<\/a>'
for match in re.finditer(regex, pkg_info):
path = match.group(1)
hash_algorithm = match.group(2)
http://git-wip-us.apache.org/repos/asf/impala/blob/b1ef7de0/infra/python/deps/requirements.txt
----------------------------------------------------------------------
diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt
index 4c65d5a..29cc9ce 100644
--- a/infra/python/deps/requirements.txt
+++ b/infra/python/deps/requirements.txt
@@ -55,6 +55,7 @@ python-magic == 0.4.11
pywebhdfs == 0.3.2
pbr == 1.8.1
requests == 2.7.0
+setuptools == 39.0.0
setuptools-scm == 1.15.4
sh == 1.11
sqlparse == 0.1.15
[11/11] impala git commit: IMPALA-6643: Add REFRESH fine-grained
privilege
Posted by ta...@apache.org.
IMPALA-6643: Add REFRESH fine-grained privilege
Before this patch, ALL privilege was required to execute INVALIDATE
METADATA and having any privilege allowed executing REFRESH <table>
and INVALIDATE METADATA <table>. With this patch, REFRESH privilege
is now required to execute INVALIDATE METADATA or REFRESH statement.
These are the new GRANT/REVOKE statements introduced at server,
database, and table scopes.
GRANT REFRESH on SERVER svr TO ROLE testrole;
GRANT REFRESH on DATABASE db TO ROLE testrole;
GRANT REFRESH on TABLE db.tbl TO ROLE testrole;
REVOKE REFRESH on SERVER svr FROM ROLE testrole;
REVOKE REFRESH on DATABASE db FROM ROLE testrole;
REVOKE REFRESH on TABLE db.tbl FROM ROLE testrole;
Testing:
- Ran front-end tests
Cherry-picks: not for 2.x
Change-Id: I4c3c5a51fe493d39fd719c7a388d4d5760049ce4
Reviewed-on: http://gerrit.cloudera.org:8080/9589
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/1d38c584
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/1d38c584
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/1d38c584
Branch: refs/heads/master
Commit: 1d38c584aef2d0f852af8ee659157d00eb3b061a
Parents: e6aadc9
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Mon Mar 12 16:25:47 2018 -0500
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Mar 20 20:37:35 2018 +0000
----------------------------------------------------------------------
common/thrift/CatalogObjects.thrift | 3 +-
fe/src/main/cup/sql-parser.cup | 2 +
.../apache/impala/analysis/PrivilegeSpec.java | 7 +-
.../impala/analysis/ResetMetadataStmt.java | 10 +-
.../authorization/AuthorizationChecker.java | 11 +-
.../apache/impala/authorization/Privilege.java | 52 +++++--
.../impala/analysis/AnalyzeAuthStmtsTest.java | 18 ++-
.../impala/analysis/AuthorizationTest.java | 137 +++++++++++++++----
.../org/apache/impala/analysis/ParserTest.java | 6 +
fe/src/test/resources/authz-policy.ini.template | 9 +-
10 files changed, 204 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/common/thrift/CatalogObjects.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift
index 0f1ea5d..f4aefdb 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -467,7 +467,8 @@ enum TPrivilegeScope {
enum TPrivilegeLevel {
ALL,
INSERT,
- SELECT
+ SELECT,
+ REFRESH
}
// Represents a privilege in an authorization policy. Privileges contain the level
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/fe/src/main/cup/sql-parser.cup
----------------------------------------------------------------------
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index 9802778..55ebc5d 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -960,6 +960,8 @@ privilege ::=
{: RESULT = TPrivilegeLevel.SELECT; :}
| KW_INSERT
{: RESULT = TPrivilegeLevel.INSERT; :}
+ | KW_REFRESH
+ {: RESULT = TPrivilegeLevel.REFRESH; :}
| KW_ALL
{: RESULT = TPrivilegeLevel.ALL; :}
;
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java b/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
index a21af93..cbc3c80 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
@@ -188,9 +188,10 @@ public class PrivilegeSpec implements ParseNode {
switch (scope_) {
case SERVER:
- if (privilegeLevel_ != TPrivilegeLevel.ALL) {
- throw new AnalysisException("Only 'ALL' privilege may be applied at " +
- "SERVER scope in privilege spec.");
+ if (privilegeLevel_ != TPrivilegeLevel.ALL &&
+ privilegeLevel_ != TPrivilegeLevel.REFRESH) {
+ throw new AnalysisException("Only 'ALL' or 'REFRESH' privilege " +
+ "may be applied at SERVER scope in privilege spec.");
}
break;
case DATABASE:
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
index e070d51..a985734 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
@@ -97,7 +97,7 @@ public class ResetMetadataStmt extends StatementBase {
// Verify the user has privileges to access this table. Will throw if the parent
// database does not exists. Don't call getTable() to avoid loading the table
// metadata if it is not yet in this impalad's catalog cache.
- if (!analyzer.dbContainsTable(dbName, tableName_.getTbl(), Privilege.ANY)) {
+ if (!analyzer.dbContainsTable(dbName, tableName_.getTbl(), Privilege.REFRESH)) {
// Only throw an exception when the table does not exist for refresh statements
// since 'invalidate metadata' should add/remove tables created/dropped external
// to Impala.
@@ -110,10 +110,14 @@ public class ResetMetadataStmt extends StatementBase {
} else {
// Verify the user has privileges to access this table.
analyzer.registerPrivReq(new PrivilegeRequestBuilder()
- .onTable(dbName, tableName_.getTbl()).any().toRequest());
+ .onTable(dbName, tableName_.getTbl()).allOf(Privilege.REFRESH)
+ .toRequest());
}
+ } else if (database_ != null) {
+ analyzer.registerPrivReq(new PrivilegeRequestBuilder()
+ .onDb(database_).allOf(Privilege.REFRESH).toRequest());
} else {
- analyzer.registerPrivReq(new PrivilegeRequest(Privilege.ALL));
+ analyzer.registerPrivReq(new PrivilegeRequest(Privilege.REFRESH));
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java b/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
index 43e86cf..b6fc299 100644
--- a/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
+++ b/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
@@ -22,12 +22,12 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.lang.reflect.ConstructorUtils;
+import org.apache.impala.authorization.Privilege.SentryAction;
import org.apache.impala.catalog.AuthorizationException;
import org.apache.impala.catalog.AuthorizationPolicy;
import org.apache.impala.common.InternalException;
import org.apache.sentry.core.common.ActiveRoleSet;
import org.apache.sentry.core.common.Subject;
-import org.apache.sentry.core.model.db.DBModelAction;
import org.apache.sentry.core.model.db.DBModelAuthorizable;
import org.apache.sentry.policy.db.SimpleDBPolicyEngine;
import org.apache.sentry.provider.cache.SimpleCacheProviderBackend;
@@ -137,6 +137,11 @@ public class AuthorizationChecker {
throw new AuthorizationException(String.format(
"User '%s' does not have privileges to access: %s",
user.getName(), privilegeRequest.getName()));
+ } else if (privilege == Privilege.REFRESH) {
+ throw new AuthorizationException(String.format(
+ "User '%s' does not have privileges to execute " +
+ "'INVALIDATE METADATA/REFRESH' on: %s", user.getName(),
+ privilegeRequest.getName()));
} else {
throw new AuthorizationException(String.format(
"User '%s' does not have privileges to execute '%s' on: %s",
@@ -160,7 +165,7 @@ public class AuthorizationChecker {
return true;
}
- EnumSet<DBModelAction> actions = request.getPrivilege().getHiveActions();
+ EnumSet<SentryAction> actions = request.getPrivilege().getSentryActions();
List<DBModelAuthorizable> authorizeables = Lists.newArrayList(
server_.getHiveAuthorizeableHierarchy());
@@ -172,7 +177,7 @@ public class AuthorizationChecker {
// The Hive Access API does not currently provide a way to check if the user
// has any privileges on a given resource.
if (request.getPrivilege().getAnyOf()) {
- for (DBModelAction action: actions) {
+ for (SentryAction action: actions) {
if (provider_.hasAccess(new Subject(user.getShortName()), authorizeables,
EnumSet.of(action), ActiveRoleSet.ALL)) {
return true;
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/fe/src/main/java/org/apache/impala/authorization/Privilege.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/authorization/Privilege.java b/fe/src/main/java/org/apache/impala/authorization/Privilege.java
index 453558b..f82008c 100644
--- a/fe/src/main/java/org/apache/impala/authorization/Privilege.java
+++ b/fe/src/main/java/org/apache/impala/authorization/Privilege.java
@@ -19,45 +19,67 @@ package org.apache.impala.authorization;
import java.util.EnumSet;
-import org.apache.sentry.core.model.db.DBModelAction;
+import org.apache.sentry.core.common.Action;
/**
- * Maps an Impala Privilege to one or more Hive Access "Actions".
+ * Maps an Impala Privilege to one or more Sentry "Actions".
*/
public enum Privilege {
- ALL(DBModelAction.ALL, false),
- ALTER(DBModelAction.ALL, false),
- DROP(DBModelAction.ALL, false),
- CREATE(DBModelAction.ALL, false),
- INSERT(DBModelAction.INSERT, false),
- SELECT(DBModelAction.SELECT, false),
+ ALL(SentryAction.ALL, false),
+ ALTER(SentryAction.ALL, false),
+ DROP(SentryAction.ALL, false),
+ CREATE(SentryAction.ALL, false),
+ INSERT(SentryAction.INSERT, false),
+ SELECT(SentryAction.SELECT, false),
+ REFRESH(SentryAction.REFRESH, false),
// Privileges required to view metadata on a server object.
- VIEW_METADATA(EnumSet.of(DBModelAction.INSERT, DBModelAction.SELECT), true),
+ VIEW_METADATA(EnumSet.of(SentryAction.INSERT, SentryAction.SELECT,
+ SentryAction.REFRESH), true),
// Special privilege that is used to determine if the user has any valid privileges
// on a target object.
- ANY(EnumSet.allOf(DBModelAction.class), true),
+ ANY(EnumSet.allOf(SentryAction.class), true),
;
- private final EnumSet<DBModelAction> actions;
+ private final EnumSet<SentryAction> actions;
// Determines whether to check if the user has ANY the privileges defined in the
// actions list or whether to check if the user has ALL of the privileges in the
// actions list.
private final boolean anyOf_;
- private Privilege(EnumSet<DBModelAction> actions, boolean anyOf) {
+ /**
+ * This enum provides a list of Sentry actions used in Impala.
+ */
+ public enum SentryAction implements Action {
+ SELECT("select"),
+ INSERT("insert"),
+ REFRESH("refresh"),
+ ALL("*");
+
+ private final String value;
+
+ SentryAction(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
+ }
+
+ private Privilege(EnumSet<SentryAction> actions, boolean anyOf) {
this.actions = actions;
this.anyOf_ = anyOf;
}
- private Privilege(DBModelAction action, boolean anyOf) {
+ private Privilege(SentryAction action, boolean anyOf) {
this(EnumSet.of(action), anyOf);
}
/*
- * Returns the set of Hive Access Actions mapping to this Privilege.
+ * Returns the set of Sentry Access Actions mapping to this Privilege.
*/
- public EnumSet<DBModelAction> getHiveActions() {
+ public EnumSet<SentryAction> getSentryActions() {
return actions;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
index ddb95f1..f749e1f 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
@@ -165,7 +165,8 @@ public class AnalyzeAuthStmtsTest extends AnalyzerTest {
AnalyzesOk(String.format("%s INSERT ON DATABASE functional %s myrole",
formatArgs));
AnalysisError(String.format("%s INSERT ON SERVER %s myrole", formatArgs),
- "Only 'ALL' privilege may be applied at SERVER scope in privilege spec.");
+ "Only 'ALL' or 'REFRESH' privilege may be applied at SERVER scope " +
+ "in privilege spec.");
AnalysisError(String.format("%s INSERT ON URI 'hdfs:////abc//123' %s myrole",
formatArgs), "Only 'ALL' privilege may be applied at URI scope in privilege " +
"spec.");
@@ -180,7 +181,8 @@ public class AnalyzeAuthStmtsTest extends AnalyzerTest {
AnalyzesOk(String.format("%s SELECT ON DATABASE functional %s myrole",
formatArgs));
AnalysisError(String.format("%s SELECT ON SERVER %s myrole", formatArgs),
- "Only 'ALL' privilege may be applied at SERVER scope in privilege spec.");
+ "Only 'ALL' or 'REFRESH' privilege may be applied at SERVER scope " +
+ "in privilege spec.");
AnalysisError(String.format("%s SELECT ON URI 'hdfs:////abc//123' %s myrole",
formatArgs), "Only 'ALL' privilege may be applied at URI scope in privilege " +
"spec.");
@@ -219,6 +221,18 @@ public class AnalyzeAuthStmtsTest extends AnalyzerTest {
"functional.does_not_exist %s myrole", formatArgs), "Error setting " +
"privileges for table 'functional.does_not_exist'. Verify that the table " +
"exists and that you have permissions to issue a GRANT/REVOKE statement.");
+
+ // REFRESH privilege
+ AnalyzesOk(String.format(
+ "%s REFRESH ON TABLE functional.alltypes %s myrole", formatArgs));
+ AnalyzesOk(String.format(
+ "%s REFRESH ON DATABASE functional %s myrole", formatArgs));
+ AnalyzesOk(String.format("%s REFRESH ON SERVER %s myrole", formatArgs));
+ AnalyzesOk(String.format("%s REFRESH ON SERVER server1 %s myrole",
+ formatArgs));
+ AnalysisError(String.format(
+ "%s REFRESH ON URI 'hdfs:////abc//123' %s myrole", formatArgs),
+ "Only 'ALL' privilege may be applied at URI scope in privilege spec.");
}
AnalysisContext authDisabledCtx = createAuthDisabledAnalysisCtx();
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
index a3ff4c1..e517172 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
@@ -85,9 +85,9 @@ public class AuthorizationTest extends FrontendTestBase {
// ALL permission on 'tpch' database and 'newdb' database
// ALL permission on 'functional_seq_snap' database
// SELECT permissions on all tables in 'tpcds' database
- // SELECT permissions on 'functional.alltypesagg' (no INSERT permissions)
+ // SELECT, REFRESH permissions on 'functional.alltypesagg' (no INSERT permissions)
// SELECT permissions on 'functional.complex_view' (no INSERT permissions)
- // SELECT permissions on 'functional.view_view' (no INSERT permissions)
+ // SELECT, REFRESH permissions on 'functional.view_view' (no INSERT permissions)
// SELECT permissions on columns ('id', 'int_col', and 'year') on
// 'functional.alltypessmall' (no SELECT permissions on 'functional.alltypessmall')
// SELECT permissions on columns ('id', 'int_struct_col', 'struct_array_col',
@@ -101,6 +101,7 @@ public class AuthorizationTest extends FrontendTestBase {
// No permissions on database 'functional_rc'
// Only column level permissions in 'functional_avro':
// SELECT permissions on columns ('id') on 'functional_avro.alltypessmall'
+ // REFRESH permissions on 'functional_text_lzo' database
public final static String AUTHZ_POLICY_FILE = "/test-warehouse/authz-policy.ini";
public final static User USER = new User(System.getProperty("user.name"));
@@ -238,6 +239,42 @@ public class AuthorizationTest extends FrontendTestBase {
privilege.setTable_name(AuthorizeableTable.ANY_TABLE_NAME);
sentryService.grantRolePrivilege(USER, roleName, privilege);
+ // refresh_functional_text_lzo
+ roleName = "refresh_functional_text_lzo";
+ sentryService.createRole(USER, roleName, true);
+ sentryService.grantRoleToGroup(USER, roleName, USER.getName());
+
+ privilege = new TPrivilege("", TPrivilegeLevel.REFRESH,
+ TPrivilegeScope.DATABASE, false);
+ privilege.setServer_name("server1");
+ privilege.setDb_name("functional_text_lzo");
+ privilege.setTable_name(AuthorizeableTable.ANY_TABLE_NAME);
+ sentryService.grantRolePrivilege(USER, roleName, privilege);
+
+ // refresh_functional_alltypesagg
+ roleName = "refresh_functional_alltypesagg";
+ sentryService.createRole(USER, roleName, true);
+ sentryService.grantRoleToGroup(USER, roleName, USER.getName());
+
+ privilege = new TPrivilege("", TPrivilegeLevel.REFRESH,
+ TPrivilegeScope.TABLE, false);
+ privilege.setServer_name("server1");
+ privilege.setDb_name("functional");
+ privilege.setTable_name("alltypesagg");
+ sentryService.grantRolePrivilege(USER, roleName, privilege);
+
+ // refresh_functional_view_view
+ roleName = "refresh_functional_view_view";
+ sentryService.createRole(USER, roleName, true);
+ sentryService.grantRoleToGroup(USER, roleName, USER.getName());
+
+ privilege = new TPrivilege("", TPrivilegeLevel.REFRESH,
+ TPrivilegeScope.TABLE, false);
+ privilege.setServer_name("server1");
+ privilege.setDb_name("functional");
+ privilege.setTable_name("view_view");
+ sentryService.grantRolePrivilege(USER, roleName, privilege);
+
// all newdb w/ all on URI
roleName = "all_newdb";
sentryService.createRole(USER, roleName, true);
@@ -800,45 +837,64 @@ public class AuthorizationTest extends FrontendTestBase {
@Test
public void TestResetMetadata() throws ImpalaException {
- // Positive cases (user has privileges on these tables/views).
+ // Positive cases (user has REFRESH privilege on these tables/views).
AuthzOk("invalidate metadata functional.alltypesagg");
AuthzOk("refresh functional.alltypesagg");
AuthzOk("invalidate metadata functional.view_view");
AuthzOk("refresh functional.view_view");
// Positive cases for checking refresh partition
AuthzOk("refresh functional.alltypesagg partition (year=2010, month=1, day=1)");
- AuthzOk("refresh functional.alltypes partition (year=2009, month=1)");
AuthzOk("refresh functional_seq_snap.alltypes partition (year=2009, month=1)");
+ // User has REFRESH privilege on functional_text_lzo database, =
+ // but no privilege at the table level.
+ AuthzOk("invalidate metadata functional_text_lzo.alltypes");
+ AuthzOk("refresh functional_text_lzo.alltypes");
+ AuthzOk("refresh functions functional_text_lzo");
AuthzError("invalidate metadata unknown_db.alltypessmall",
- "User '%s' does not have privileges to access: unknown_db.alltypessmall");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: unknown_db.alltypessmall");
AuthzError("invalidate metadata functional_seq.alltypessmall",
- "User '%s' does not have privileges to access: functional_seq.alltypessmall");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional_seq.alltypessmall");
AuthzError("invalidate metadata functional.alltypes_view",
- "User '%s' does not have privileges to access: functional.alltypes_view");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional.alltypes_view");
AuthzError("invalidate metadata functional.unknown_table",
- "User '%s' does not have privileges to access: functional.unknown_table");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional.unknown_table");
AuthzError("invalidate metadata functional.alltypessmall",
- "User '%s' does not have privileges to access: functional.alltypessmall");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional.alltypessmall");
AuthzError("refresh functional.alltypessmall",
- "User '%s' does not have privileges to access: functional.alltypessmall");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional.alltypessmall");
AuthzError("refresh functional.alltypes_view",
- "User '%s' does not have privileges to access: functional.alltypes_view");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional.alltypes_view");
// Only column-level privileges on the table
- AuthzError("invalidate metadata functional.alltypestiny", "User '%s' does not " +
- "have privileges to access: functional.alltypestiny");
+ AuthzError("invalidate metadata functional.alltypestiny",
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional.alltypestiny");
// Only column-level privileges on the table
- AuthzError("refresh functional.alltypestiny", "User '%s' does not have " +
- "privileges to access: functional.alltypestiny");
+ AuthzError("refresh functional.alltypestiny",
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional.alltypestiny");
AuthzError("invalidate metadata",
- "User '%s' does not have privileges to access: server");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: server");
AuthzError(
"refresh functional_rc.alltypesagg partition (year=2010, month=1, day=1)",
- "User '%s' does not have privileges to access: functional_rc.alltypesagg");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional_rc.alltypesagg");
AuthzError(
"refresh functional_rc.alltypesagg partition (year=2010, month=1, day=9999)",
- "User '%s' does not have privileges to access: functional_rc.alltypesagg");
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional_rc.alltypesagg");
+ AuthzError("refresh functions functional_rc",
+ "User '%s' does not have privileges to execute 'INVALIDATE METADATA/REFRESH' " +
+ "on: functional_rc");
// TODO: Add test support for dynamically changing privileges for
// file-based policy.
@@ -847,11 +903,35 @@ public class AuthorizationTest extends FrontendTestBase {
try {
sentryService.grantRoleToGroup(USER, "admin", USER.getName());
- ((ImpaladTestCatalog) ctx_.catalog).reset();
+ ctx_.catalog.reset();
AuthzOk("invalidate metadata");
+ AuthzOk("invalidate metadata functional.testtbl");
+ AuthzOk("refresh functional.testtbl");
+ AuthzOk("refresh functional.alltypesagg partition (year=2010, month=1, day=1)");
+ AuthzOk("refresh functions functional");
} finally {
sentryService.revokeRoleFromGroup(USER, "admin", USER.getName());
- ((ImpaladTestCatalog) ctx_.catalog).reset();
+ ctx_.catalog.reset();
+ }
+
+ // User has REFRESH privilege on server.
+ String roleName = "refresh_role";
+ try {
+ sentryService.createRole(USER, roleName, true);
+ TPrivilege privilege = new TPrivilege("", TPrivilegeLevel.REFRESH,
+ TPrivilegeScope.SERVER, false);
+ privilege.setServer_name("server1");
+ sentryService.grantRolePrivilege(USER, roleName, privilege);
+ sentryService.grantRoleToGroup(USER, roleName, USER.getName());
+ ctx_.catalog.reset();
+ AuthzOk("invalidate metadata");
+ AuthzOk("invalidate metadata functional.testtbl");
+ AuthzOk("refresh functional.testtbl");
+ AuthzOk("refresh functional.alltypesagg partition (year=2010, month=1, day=1)");
+ AuthzOk("refresh functions functional");
+ } finally {
+ sentryService.dropRole(USER, roleName, true);
+ ctx_.catalog.reset();
}
}
@@ -1437,6 +1517,8 @@ public class AuthorizationTest extends FrontendTestBase {
@Test
public void TestDescribeDb() throws ImpalaException {
AuthzOk("describe database functional_seq_snap");
+ // User has REFRESH privilege on functional_text_lzo database.
+ AuthzOk("describe database functional_text_lzo");
// Database doesn't exist.
AuthzError("describe database nodb",
@@ -1448,6 +1530,7 @@ public class AuthorizationTest extends FrontendTestBase {
@Test
public void TestDescribe() throws ImpalaException {
+ // User has SELECT and REFRESH privileges on functional.alltypesagg table.
AuthzOk("describe functional.alltypesagg");
AuthzOk("describe functional.alltypes");
AuthzOk("describe functional.complex_view");
@@ -1725,6 +1808,9 @@ public class AuthorizationTest extends FrontendTestBase {
AuthzOk("show databases");
AuthzOk("show tables in _impala_builtins");
AuthzOk("show functions in _impala_builtins");
+ // User has REFRESH privilege on functional_text_lzo database.
+ AuthzOk("show tables in functional_text_lzo");
+ AuthzOk("show functions in functional_text_lzo");
// Database exists, user does not have access.
AuthzError("show tables in functional_rc",
@@ -1748,6 +1834,7 @@ public class AuthorizationTest extends FrontendTestBase {
// Show partitions and show table/column stats.
String[] statsQuals = new String[] { "partitions", "table stats", "column stats" };
for (String qual: statsQuals) {
+ // User has SELECT and REFRESH privileges on functional.alltypesagg table.
AuthzOk(String.format("show %s functional.alltypesagg", qual));
AuthzOk(String.format("show %s functional.alltypes", qual));
// User does not have access to db/table.
@@ -1765,6 +1852,8 @@ public class AuthorizationTest extends FrontendTestBase {
// Show files
String[] partitions = new String[] { "", "partition(month=10, year=2010)" };
for (String partition: partitions) {
+ // User has SELECT and REFRESH privileges on functional.alltypesagg table.
+ AuthzOk(String.format("show files in functional.alltypesagg %s", partition));
AuthzOk(String.format("show files in functional.alltypes %s", partition));
// User does not have access to db/table.
AuthzError(String.format("show files in nodb.tbl %s", partition),
@@ -1784,7 +1873,8 @@ public class AuthorizationTest extends FrontendTestBase {
// These are the only dbs that should show up because they are the only
// dbs the user has any permissions on.
List<String> expectedDbs = Lists.newArrayList("default", "functional",
- "functional_avro", "functional_parquet", "functional_seq_snap", "tpcds", "tpch");
+ "functional_avro", "functional_parquet", "functional_seq_snap",
+ "functional_text_lzo", "tpcds", "tpch");
List<Db> dbs = fe_.getDbs(PatternMatcher.createHivePatternMatcher("*"), USER);
assertEquals(expectedDbs, extractDbNames(dbs));
@@ -1846,11 +1936,11 @@ public class AuthorizationTest extends FrontendTestBase {
@Test
public void TestShowCreateTable() throws ImpalaException {
+ // User has SELECT and REFRESH privileges functional.alltypesagg table.
AuthzOk("show create table functional.alltypesagg");
AuthzOk("show create table functional.alltypes");
// Have permissions on view and underlying table.
AuthzOk("show create table functional_seq_snap.alltypes_view");
-
// Unqualified table name.
AuthzError("show create table alltypes",
"User '%s' does not have privileges to access: default.alltypes");
@@ -1947,7 +2037,8 @@ public class AuthorizationTest extends FrontendTestBase {
req.get_schemas_req.setSchemaName("%");
TResultSet resp = fe_.execHiveServer2MetadataOp(req);
List<String> expectedDbs = Lists.newArrayList("default", "functional",
- "functional_avro", "functional_parquet", "functional_seq_snap", "tpcds", "tpch");
+ "functional_avro", "functional_parquet", "functional_seq_snap",
+ "functional_text_lzo", "tpcds", "tpch");
assertEquals(expectedDbs.size(), resp.rows.size());
for (int i = 0; i < resp.rows.size(); ++i) {
assertEquals(expectedDbs.get(i),
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index b51d148..aefa8e2 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -3575,6 +3575,12 @@ public class ParserTest extends FrontendTestBase {
formatStr));
ParserError(String.format("%s SELECT (a, b) ON URI 'foo' %s myRole", formatStr));
+ // REFRESH privilege.
+ ParsesOk(String.format("%s REFRESH ON SERVER %s myRole", formatStr));
+ ParsesOk(String.format("%s REFRESH ON SERVER foo %s myRole", formatStr));
+ ParsesOk(String.format("%s REFRESH ON DATABASE foo %s myRole", formatStr));
+ ParsesOk(String.format("%s REFRESH ON TABLE foo %s myRole", formatStr));
+
// Server scope does not accept a name.
ParsesOk(String.format("%s ALL ON SERVER %s myRole", formatStr));
ParsesOk(String.format("%s INSERT ON SERVER %s myRole", formatStr));
http://git-wip-us.apache.org/repos/asf/impala/blob/1d38c584/fe/src/test/resources/authz-policy.ini.template
----------------------------------------------------------------------
diff --git a/fe/src/test/resources/authz-policy.ini.template b/fe/src/test/resources/authz-policy.ini.template
index 18abdb6..f3c7c1f 100644
--- a/fe/src/test/resources/authz-policy.ini.template
+++ b/fe/src/test/resources/authz-policy.ini.template
@@ -24,7 +24,9 @@ ${USER} = all_tpch, all_newdb, all_functional_seq_snap, select_tpcds,\
select_functional_alltypesagg, insert_functional_alltypes,\
select_functional_complex_view, select_functional_view_view,\
insert_parquet, new_table_uri, tpch_data_uri, select_column_level_functional,\
- select_column_level_functional_avro, upper_case_uri
+ select_column_level_functional_avro, upper_case_uri,\
+ refresh_functional_text_lzo, refresh_functional_alltypesagg,\
+ refresh_functional_view_view
auth_to_local_group = test_role
server_admin = all_server
@@ -47,6 +49,11 @@ select_functional_complex_view =\
select_functional_view_view =\
server=server1->db=functional->table=view_view->action=select
insert_parquet = server=server1->db=functional_parquet->table=*->action=insert
+refresh_functional_text_lzo = server=server1->db=functional_text_lzo->action=refresh
+refresh_functional_alltypesagg =\
+ server=server1->db=functional->table=alltypesagg->action=refresh
+refresh_functional_view_view =\
+ server=server1->db=functional->table=view_view->action=refresh
select_column_level_functional =\
server=server1->db=functional->table=alltypessmall->column=id->action=select,\
server=server1->db=functional->table=alltypessmall->column=int_col->action=select,\
[07/11] impala git commit: Consistently use Java 1.7 compiler.
Posted by ta...@apache.org.
Consistently use Java 1.7 compiler.
We use Java 1.7 in fe/pom.xml, where most of our Java code is. For
consistency, this updates the rest of our Maven configurations to use
the same version of Java. A change I'm working with uses
try-with-resources in HBase splitting, which is how I ran into
this.
Testing: ran core tests
Change-Id: I6cecddf367f00185a14a8b08c03456e3b756bd70
Reviewed-on: http://gerrit.cloudera.org:8080/9600
Reviewed-by: Philip Zeyliger <ph...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/5c8da5d1
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/5c8da5d1
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/5c8da5d1
Branch: refs/heads/master
Commit: 5c8da5d13a0fc1289428aad2a5e207b2c5b0fc8e
Parents: e148c1a
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Mon Mar 12 21:58:56 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sat Mar 17 04:08:53 2018 +0000
----------------------------------------------------------------------
ext-data-source/api/pom.xml | 4 ++--
ext-data-source/sample/pom.xml | 4 ++--
ext-data-source/test/pom.xml | 4 ++--
testdata/pom.xml | 4 ++--
4 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/5c8da5d1/ext-data-source/api/pom.xml
----------------------------------------------------------------------
diff --git a/ext-data-source/api/pom.xml b/ext-data-source/api/pom.xml
index 9240fb1..15bfd55 100644
--- a/ext-data-source/api/pom.xml
+++ b/ext-data-source/api/pom.xml
@@ -61,8 +61,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
- <source>1.6</source>
- <target>1.6</target>
+ <source>1.7</source>
+ <target>1.7</target>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/impala/blob/5c8da5d1/ext-data-source/sample/pom.xml
----------------------------------------------------------------------
diff --git a/ext-data-source/sample/pom.xml b/ext-data-source/sample/pom.xml
index f6a8e54..c935549 100644
--- a/ext-data-source/sample/pom.xml
+++ b/ext-data-source/sample/pom.xml
@@ -66,8 +66,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
- <source>1.6</source>
- <target>1.6</target>
+ <source>1.7</source>
+ <target>1.7</target>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/impala/blob/5c8da5d1/ext-data-source/test/pom.xml
----------------------------------------------------------------------
diff --git a/ext-data-source/test/pom.xml b/ext-data-source/test/pom.xml
index 5fdd7e9..b8e2601 100644
--- a/ext-data-source/test/pom.xml
+++ b/ext-data-source/test/pom.xml
@@ -57,8 +57,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
- <source>1.6</source>
- <target>1.6</target>
+ <source>1.7</source>
+ <target>1.7</target>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/impala/blob/5c8da5d1/testdata/pom.xml
----------------------------------------------------------------------
diff --git a/testdata/pom.xml b/testdata/pom.xml
index ae584d9..f00da3b 100644
--- a/testdata/pom.xml
+++ b/testdata/pom.xml
@@ -123,8 +123,8 @@ under the License.
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
- <source>1.6</source>
- <target>1.6</target>
+ <source>1.7</source>
+ <target>1.7</target>
</configuration>
</plugin>
[02/11] impala git commit: [DOCS] Changed to
--use_local_tz_for_unix_timestamp_conversions
Posted by ta...@apache.org.
[DOCS] Changed to --use_local_tz_for_unix_timestamp_conversions
Change-Id: Id75ec73031b97aa8a3c61ccdbaea39db008b4093
Reviewed-on: http://gerrit.cloudera.org:8080/9620
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/b62ecb67
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/b62ecb67
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/b62ecb67
Branch: refs/heads/master
Commit: b62ecb6795cca22a6641f10d00d097c4957aa5fc
Parents: 0532ff9
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Tue Mar 13 17:47:38 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Mar 16 16:36:14 2018 +0000
----------------------------------------------------------------------
docs/shared/impala_common.xml | 10 ++++++----
docs/topics/impala_datetime_functions.xml | 2 +-
docs/topics/impala_timestamp.xml | 8 ++++----
3 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/b62ecb67/docs/shared/impala_common.xml
----------------------------------------------------------------------
diff --git a/docs/shared/impala_common.xml b/docs/shared/impala_common.xml
index 4a96565..c272e4f 100644
--- a/docs/shared/impala_common.xml
+++ b/docs/shared/impala_common.xml
@@ -1251,7 +1251,7 @@ drop database temp;
<p rev="2.2.0" id="timezone_conversion_caveat">
The way this function deals with time zones when converting to or from <codeph>TIMESTAMP</codeph>
- values is affected by the <codeph>-use_local_tz_for_unix_timestamp_conversions</codeph> startup flag for the
+ values is affected by the <codeph>--use_local_tz_for_unix_timestamp_conversions</codeph> startup flag for the
<cmdname>impalad</cmdname> daemon. See <xref href="../topics/impala_timestamp.xml#timestamp"/> for details about
how Impala handles time zone considerations for the <codeph>TIMESTAMP</codeph> data type.
</p>
@@ -1409,11 +1409,13 @@ drop database temp;
</p>
<p>
- <ph id="cast_int_to_timestamp">Casting an integer or floating-point value <codeph>N</codeph> to
+ <ph id="cast_int_to_timestamp">
+ Casting an integer or floating-point value <codeph>N</codeph> to
<codeph>TIMESTAMP</codeph> produces a value that is <codeph>N</codeph> seconds past the start of the epoch
date (January 1, 1970). By default, the result value represents a date and time in the UTC time zone.
- If the setting <codeph>-use_local_tz_for_unix_timestamp_conversions=true</codeph> is in effect,
- the resulting <codeph>TIMESTAMP</codeph> represents a date and time in the local time zone.</ph>
+ If the setting <codeph>--use_local_tz_for_unix_timestamp_conversions=true</codeph> is in effect,
+ the resulting <codeph>TIMESTAMP</codeph> represents a date and time in the local time zone.
+ </ph>
</p>
<p id="redaction_yes" rev="2.2.0">
http://git-wip-us.apache.org/repos/asf/impala/blob/b62ecb67/docs/topics/impala_datetime_functions.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_datetime_functions.xml b/docs/topics/impala_datetime_functions.xml
index d9c77dc..d40e28b 100644
--- a/docs/topics/impala_datetime_functions.xml
+++ b/docs/topics/impala_datetime_functions.xml
@@ -52,7 +52,7 @@ under the License.
<p rev="2.2.0">
Some of these functions are affected by the setting of the
- <codeph>-use_local_tz_for_unix_timestamp_conversions</codeph> startup flag for the
+ <codeph>--use_local_tz_for_unix_timestamp_conversions</codeph> startup flag for the
<cmdname>impalad</cmdname> daemon. This setting is off by default, meaning that
functions such as <codeph>from_unixtime()</codeph> and <codeph>unix_timestamp()</codeph>
consider the input values to always represent the UTC time zone.
http://git-wip-us.apache.org/repos/asf/impala/blob/b62ecb67/docs/topics/impala_timestamp.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_timestamp.xml b/docs/topics/impala_timestamp.xml
index 94e9c69..320f99d 100644
--- a/docs/topics/impala_timestamp.xml
+++ b/docs/topics/impala_timestamp.xml
@@ -163,21 +163,21 @@ insert into auction_details
If you have data files written by Hive, those <codeph>TIMESTAMP</codeph> values represent the local timezone
of the host where the data was written, potentially leading to inconsistent results when processed by Impala.
To avoid compatibility problems or having to code workarounds, you can specify one or both of these
- <cmdname>impalad</cmdname> startup flags: <codeph>-use_local_tz_for_unix_timestamp_conversions=true</codeph>
+ <cmdname>impalad</cmdname> startup flags: <codeph>--use_local_tz_for_unix_timestamp_conversions=true</codeph>
<codeph>-convert_legacy_hive_parquet_utc_timestamps=true</codeph>. Although
<codeph>-convert_legacy_hive_parquet_utc_timestamps</codeph> is turned off by default to avoid performance overhead, where practical
turn it on when processing <codeph>TIMESTAMP</codeph> columns in Parquet files written by Hive, to avoid unexpected behavior.
</p>
<p rev="2.2.0">
- The <codeph>-use_local_tz_for_unix_timestamp_conversions</codeph> setting affects conversions from
+ The <codeph>--use_local_tz_for_unix_timestamp_conversions</codeph> setting affects conversions from
<codeph>TIMESTAMP</codeph> to <codeph>BIGINT</codeph>, or from <codeph>BIGINT</codeph>
to <codeph>TIMESTAMP</codeph>. By default, Impala treats all <codeph>TIMESTAMP</codeph> values as UTC,
to simplify analysis of time-series data from different geographic regions. When you enable the
- <codeph>-use_local_tz_for_unix_timestamp_conversions</codeph> setting, these operations
+ <codeph>--use_local_tz_for_unix_timestamp_conversions</codeph> setting, these operations
treat the input values as if they are in the local tie zone of the host doing the processing.
See <xref href="impala_datetime_functions.xml#datetime_functions"/> for the list of functions
- affected by the <codeph>-use_local_tz_for_unix_timestamp_conversions</codeph> setting.
+ affected by the <codeph>--use_local_tz_for_unix_timestamp_conversions</codeph> setting.
</p>
<p>
[03/11] impala git commit: IMPALA-6675: Default to
--compact_catalog_topic=true.
Posted by ta...@apache.org.
IMPALA-6675: Default to --compact_catalog_topic=true.
Testing:
- Ran a few queries locally
- Ran test_compact_catalog_updates.py locally
Mostafa's perf evaluation:
- 130 node cluster
- Load metadata after invalidate for 4 tables, each
with 100K partitions and 1 million files
Results compaction on vs. compaction off
- 5.7x reduction in topic size and Network
- 30% reduction in Catalog+Statestore CPU
- 15% speedup in query time
- Compaction of topic takes ~22s in the Catalog
- Time spent by Statestore sending the topics is
reduced from 90s to 17s
- Max topic update duration reduced from 72s to 11s
Change-Id: I39a2dd42a21ef448b85278a8cef3c1d0112b844f
Reviewed-on: http://gerrit.cloudera.org:8080/9661
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/2f508183
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/2f508183
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/2f508183
Branch: refs/heads/master
Commit: 2f508183cb4dfefe1c67bf8c1988f350700763fe
Parents: b62ecb6
Author: Alex Behm <al...@cloudera.com>
Authored: Thu Mar 15 09:57:10 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Mar 16 19:12:46 2018 +0000
----------------------------------------------------------------------
be/src/common/global-flags.cc | 2 +-
tests/custom_cluster/test_compact_catalog_updates.py | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/2f508183/be/src/common/global-flags.cc
----------------------------------------------------------------------
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 3ed0a0f..b62170d 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -88,7 +88,7 @@ DEFINE_bool(abort_on_config_error, true, "Abort Impala startup if there are impr
DEFINE_bool(disable_mem_pools, false, "Set to true to disable memory pooling. "
"This can be used to help diagnose memory corruption issues.");
-DEFINE_bool(compact_catalog_topic, false, "If true, catalog updates sent via the "
+DEFINE_bool(compact_catalog_topic, true, "If true, catalog updates sent via the "
"statestore are compacted before transmission. This saves network bandwidth at the"
" cost of a small quantity of CPU time. Enable this option in cluster with large"
" catalogs. It must be enabled on both the catalog service, and all Impala demons.");
http://git-wip-us.apache.org/repos/asf/impala/blob/2f508183/tests/custom_cluster/test_compact_catalog_updates.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_compact_catalog_updates.py b/tests/custom_cluster/test_compact_catalog_updates.py
index c824ad5..05252d5 100644
--- a/tests/custom_cluster/test_compact_catalog_updates.py
+++ b/tests/custom_cluster/test_compact_catalog_updates.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
#
-# Test Catalog behavior when --compact_catalog_topic is true.
+# Test Catalog behavior when --compact_catalog_topic is false.
import pytest
@@ -23,10 +23,10 @@ from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
class TestCompactCatalogUpdates(CustomClusterTestSuite):
@pytest.mark.execute_serially
- @CustomClusterTestSuite.with_args(impalad_args="--compact_catalog_topic=true",
- catalogd_args="--compact_catalog_topic=true")
- def test_compact_catalog_topic_updates(self):
- """Start Impala cluster with compact catalog update topics enabled and run a set
+ @CustomClusterTestSuite.with_args(impalad_args="--compact_catalog_topic=false",
+ catalogd_args="--compact_catalog_topic=false")
+ def test_non_compact_catalog_topic_updates(self):
+ """Start Impala cluster with compact catalog update topics disabled and run a set
of smoke tests to verify that catalog updates are received properly."""
try: