You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/10/02 17:33:21 UTC
[1/8] impala git commit: IMPALA-7595: Check the validity of the time
part of Parquet timestamps
Repository: impala
Updated Branches:
refs/heads/master ddef2cb9b -> ea2809f5d
IMPALA-7595: Check the validity of the time part of Parquet timestamps
Before this fix Impala did not check whether a timestamp's time part
is out of the valid [0, 24 hour) range when reading Parquet files,
so these timestamps were memcopied as they were to slots, leading to
results like:
1970-01-01 -00:00:00.000000001
1970-01-01 24:00:00
Different parts of Impala treat these timestamp differently:
- string conversion leads to invalid representation that cannot be
converted back to timestamp
- timezone conversions handle the overflowing time part and give
a valid timestamp result (at least since CCTZ, I did not check
older versions of Impala)
- Parquet writing inserts these timestamp as they are, so the
resulting Parquet file will also contain corrupt timestamps
The fix adds a check that converts these corrupt timestamps to NULL,
similarly to the handling of timestamp outside the [1400..10000)
range. A new error code is added for this case. If both the date
and the time part is corrupt, then error about corrupt time is
returned.
Testing:
- added a new scanner test that reads a corrupted Parquet file
with edge values
Change-Id: Ibc0ae651b6a0a028c61a15fd069ef9e904231058
Reviewed-on: http://gerrit.cloudera.org:8080/11521
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/81084111
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/81084111
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/81084111
Branch: refs/heads/master
Commit: 810841115a4f62dffd219cca8a9fbd34ea73e37c
Parents: ddef2cb
Author: Csaba Ringhofer <cs...@cloudera.com>
Authored: Wed Sep 26 18:14:13 2018 +0200
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Mon Oct 1 13:20:40 2018 +0000
----------------------------------------------------------------------
be/src/exec/parquet-column-readers.cc | 13 ++++++++++---
be/src/runtime/timestamp-value.h | 7 +++++++
common/thrift/generate_error_codes.py | 4 ++++
testdata/data/README | 7 +++++++
testdata/data/out_of_range_time_of_day.parquet | Bin 0 -> 427 bytes
.../out-of-range-timestamp-abort-on-error.test | 7 +++++++
.../out-of-range-timestamp-continue-on-error.test | 14 ++++++++++++++
tests/query_test/test_scanners.py | 9 ++++++++-
8 files changed, 57 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/81084111/be/src/exec/parquet-column-readers.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/parquet-column-readers.cc b/be/src/exec/parquet-column-readers.cc
index 74603e7..9a9d704 100644
--- a/be/src/exec/parquet-column-readers.cc
+++ b/be/src/exec/parquet-column-readers.cc
@@ -694,9 +694,16 @@ inline bool ScalarColumnReader<TimestampValue, parquet::Type::INT96, true>
template <>
bool ScalarColumnReader<TimestampValue, parquet::Type::INT96, true>::ValidateValue(
TimestampValue* val) const {
- if (UNLIKELY(!TimestampValue::IsValidDate(val->date()))) {
- ErrorMsg msg(TErrorCode::PARQUET_TIMESTAMP_OUT_OF_RANGE,
- filename(), node_.element->name);
+ if (UNLIKELY(!TimestampValue::IsValidDate(val->date())
+ || !TimestampValue::IsValidTime(val->time()))) {
+ // If both are corrupt, invalid time takes precedence over invalid date, because
+ // invalid date may come from a more or less functional encoder that does not respect
+ // the 1400..9999 limit, while an invalid time is a good indicator of buggy encoder
+ // or memory garbage.
+ TErrorCode::type errorCode = TimestampValue::IsValidTime(val->time())
+ ? TErrorCode::PARQUET_TIMESTAMP_OUT_OF_RANGE
+ : TErrorCode::PARQUET_TIMESTAMP_INVALID_TIME_OF_DAY;
+ ErrorMsg msg(errorCode, filename(), node_.element->name);
Status status = parent_->state_->LogOrReturnError(msg);
if (!status.ok()) parent_->parse_status_ = status;
return false;
http://git-wip-us.apache.org/repos/asf/impala/blob/81084111/be/src/runtime/timestamp-value.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/timestamp-value.h b/be/src/runtime/timestamp-value.h
index 56f103e..840a5ae 100644
--- a/be/src/runtime/timestamp-value.h
+++ b/be/src/runtime/timestamp-value.h
@@ -194,6 +194,13 @@ class TimestampValue {
&& date.day_number() <= MAX_DAY_NUMBER;
}
+ /// Verifies that the time is not negative and is less than a whole day.
+ static inline bool IsValidTime(const boost::posix_time::time_duration& time) {
+ static const int64_t NANOS_PER_DAY = 1'000'000'000LL*60*60*24;
+ return !time.is_negative()
+ && time.total_nanoseconds() < NANOS_PER_DAY;
+ }
+
/// Formats the timestamp using the given date/time context and places the result in the
/// string buffer. The size of the buffer should be at least dt_ctx.fmt_out_len + 1. A
/// string terminator will be appended to the string.
http://git-wip-us.apache.org/repos/asf/impala/blob/81084111/common/thrift/generate_error_codes.py
----------------------------------------------------------------------
diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py
index 84859b1..768a569 100755
--- a/common/thrift/generate_error_codes.py
+++ b/common/thrift/generate_error_codes.py
@@ -366,6 +366,10 @@ error_codes = (
("CANCELLED_INTERNALLY", 119, "Cancelled in $0"),
("SERVER_SHUTTING_DOWN", 120, "Server is being shut down: $0."),
+
+ ("PARQUET_TIMESTAMP_INVALID_TIME_OF_DAY", 121,
+ "Parquet file '$0' column '$1' contains a timestamp with invalid time of day. "
+ "The time of day should be 0 <= and < 24 hour (in nanoseconds)."),
)
import sys
http://git-wip-us.apache.org/repos/asf/impala/blob/81084111/testdata/data/README
----------------------------------------------------------------------
diff --git a/testdata/data/README b/testdata/data/README
index 2cf756e..599b009 100644
--- a/testdata/data/README
+++ b/testdata/data/README
@@ -223,3 +223,10 @@ Contains a single row. It is used to test IMPALA-7559 which only occurs when all
in a column chunk are the same timestamp and the file is written with parquet-mr (which
is used by Hive).
+out_of_range_time_of_day.parquet:
+IMPALA-7595: Parquet file that contains timestamps where the time part is out of the
+valid range [0..24H). Before the fix, select * returned these values:
+1970-01-01 -00:00:00.000000001 (invalid - negative time of day)
+1970-01-01 00:00:00
+1970-01-01 23:59:59.999999999
+1970-01-01 24:00:00 (invalid - time of day should be less than a whole day)
http://git-wip-us.apache.org/repos/asf/impala/blob/81084111/testdata/data/out_of_range_time_of_day.parquet
----------------------------------------------------------------------
diff --git a/testdata/data/out_of_range_time_of_day.parquet b/testdata/data/out_of_range_time_of_day.parquet
new file mode 100644
index 0000000..8c1034f
Binary files /dev/null and b/testdata/data/out_of_range_time_of_day.parquet differ
http://git-wip-us.apache.org/repos/asf/impala/blob/81084111/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-abort-on-error.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-abort-on-error.test b/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-abort-on-error.test
index c5783b1..180a75e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-abort-on-error.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-abort-on-error.test
@@ -6,3 +6,10 @@ SELECT * FROM out_of_range_timestamp;
---- CATCH
Parquet file '$NAMENODE/test-warehouse/$DATABASE.db/out_of_range_timestamp/out_of_range_timestamp.parquet' column 'ts' contains an out of range timestamp. The valid date range is 1400-01-01..9999-12-31.
====
+---- QUERY
+# IMPALA-7595: Test Parquet timestamp columns where the time part is
+# out of the valid range [0..24H).
+SELECT * FROM out_of_range_time_of_day;
+---- CATCH
+Parquet file '$NAMENODE/test-warehouse/$DATABASE.db/out_of_range_time_of_day/out_of_range_time_of_day.parquet' column 'd' contains a timestamp with invalid time of day. The time of day should be 0 <= and < 24 hour (in nanoseconds)
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/81084111/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-continue-on-error.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-continue-on-error.test b/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-continue-on-error.test
index 1fe185d..09364dd 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-continue-on-error.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/out-of-range-timestamp-continue-on-error.test
@@ -13,3 +13,17 @@ NULL
---- ERRORS
Parquet file '$NAMENODE/test-warehouse/$DATABASE.db/out_of_range_timestamp/out_of_range_timestamp.parquet' column 'ts' contains an out of range timestamp. The valid date range is 1400-01-01..9999-12-31. (1 of 2 similar)
====
+---- QUERY
+# IMPALA-7595: Test Parquet timestamp columns where the time part is
+# out of the valid range [0..24H).
+SELECT * FROM out_of_range_time_of_day;
+---- TYPES
+TIMESTAMP
+---- RESULTS
+NULL
+1970-01-01 00:00:00
+1970-01-01 23:59:59.999999999
+NULL
+---- ERRORS
+Parquet file '$NAMENODE/test-warehouse/$DATABASE.db/out_of_range_time_of_day/out_of_range_time_of_day.parquet' column 'd' contains a timestamp with invalid time of day. The time of day should be 0 <= and < 24 hour (in nanoseconds). (1 of 2 similar)
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/81084111/tests/query_test/test_scanners.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 8ee0eb7..ab724d2 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -306,9 +306,16 @@ class TestParquet(ImpalaTestSuite):
self.run_test_case('QueryTest/parquet-abort-on-error', vector)
def test_timestamp_out_of_range(self, vector, unique_database):
- """IMPALA-4363: Test scanning parquet files with an out of range timestamp."""
+ """IMPALA-4363: Test scanning parquet files with an out of range timestamp.
+ Also tests IMPALA-7595: Test Parquet timestamp columns where the time part
+ is out of the valid range [0..24H).
+ """
+ # out of range date part
create_table_from_parquet(self.client, unique_database, "out_of_range_timestamp")
+ # out of range time part
+ create_table_from_parquet(self.client, unique_database, "out_of_range_time_of_day")
+
vector.get_value('exec_option')['abort_on_error'] = 0
self.run_test_case('QueryTest/out-of-range-timestamp-continue-on-error',
vector, unique_database)
[8/8] impala git commit: Revert "IMPALA-7527: add fetch-from-catalogd
cache info to profile"
Posted by ta...@apache.org.
Revert "IMPALA-7527: add fetch-from-catalogd cache info to profile"
Update to profile conflicts with downstream dependency (change 2/2).
This reverts commit 8c330adf409aa74857b23ba345f0c710a1f25a32.
Change-Id: Ide56f2cd3ee6a34f716b6b465f6fb5fb944e7db8
Reviewed-on: http://gerrit.cloudera.org:8080/11560
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ea2809f5
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ea2809f5
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ea2809f5
Branch: refs/heads/master
Commit: ea2809f5ddcf48f3f41dcd12743e8a17b4ea8cd7
Parents: d3cf6d3
Author: Vuk Ercegovac <ve...@cloudera.com>
Authored: Mon Oct 1 15:59:17 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Oct 2 06:49:46 2018 +0000
----------------------------------------------------------------------
be/src/service/client-request-state.cc | 10 --
be/src/service/client-request-state.h | 8 -
be/src/service/impala-server.cc | 1 -
be/src/util/runtime-profile.h | 2 +-
common/thrift/Frontend.thrift | 6 +-
.../catalog/local/CatalogdMetaProvider.java | 92 +----------
.../org/apache/impala/service/Frontend.java | 13 +-
.../apache/impala/service/FrontendProfile.java | 163 -------------------
.../catalog/local/CatalogdMetaProviderTest.java | 20 ---
tests/custom_cluster/test_local_catalog.py | 5 +-
10 files changed, 19 insertions(+), 301 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/be/src/service/client-request-state.cc
----------------------------------------------------------------------
diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc
index f59d65b..ca871ab 100644
--- a/be/src/service/client-request-state.cc
+++ b/be/src/service/client-request-state.cc
@@ -77,7 +77,6 @@ ClientRequestState::ClientRequestState(
coord_exec_called_(false),
// Profile is assigned name w/ id after planning
profile_(RuntimeProfile::Create(&profile_pool_, "Query")),
- frontend_profile_(RuntimeProfile::Create(&profile_pool_, "Frontend")),
server_profile_(RuntimeProfile::Create(&profile_pool_, "ImpalaServer")),
summary_profile_(RuntimeProfile::Create(&profile_pool_, "Summary")),
frontend_(frontend),
@@ -119,8 +118,6 @@ ClientRequestState::ClientRequestState(
"Sql Statement", query_ctx_.client_request.stmt);
summary_profile_->AddInfoString("Coordinator",
TNetworkAddressToString(exec_env->GetThriftBackendAddress()));
-
- profile_->AddChild(frontend_profile_);
}
ClientRequestState::~ClientRequestState() {
@@ -141,13 +138,6 @@ Status ClientRequestState::SetResultCache(QueryResultSet* cache,
return Status::OK();
}
-void ClientRequestState::SetFrontendProfile(TRuntimeProfileNode profile) {
- // Should we defer creating and adding the child until here? probably.
- TRuntimeProfileTree prof_tree;
- prof_tree.nodes.emplace_back(std::move(profile));
- frontend_profile_->Update(prof_tree);
-}
-
Status ClientRequestState::Exec(TExecRequest* exec_request) {
MarkActive();
exec_request_ = *exec_request;
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/be/src/service/client-request-state.h
----------------------------------------------------------------------
diff --git a/be/src/service/client-request-state.h b/be/src/service/client-request-state.h
index d28ca3f..9442a0d 100644
--- a/be/src/service/client-request-state.h
+++ b/be/src/service/client-request-state.h
@@ -66,11 +66,6 @@ class ClientRequestState {
~ClientRequestState();
- /// Sets the profile that is produced by the frontend. The frontend creates the
- /// profile during planning and returns it to the backend via TExecRequest,
- /// which then sets the frontend profile.
- void SetFrontendProfile(TRuntimeProfileNode profile);
-
/// Based on query type, this either initiates execution of a exec_request or submits
/// the query to the Admission controller for asynchronous admission control. When this
/// returns the operation state is either RUNNING_STATE or PENDING_STATE.
@@ -357,8 +352,6 @@ class ClientRequestState {
/// The ClientRequestState builds three separate profiles.
/// * profile_ is the top-level profile which houses the other
/// profiles, plus the query timeline
- /// * frontend_profile_ is the profile emitted by the frontend
- // during planning.
/// * summary_profile_ contains mostly static information about the
/// query, including the query statement, the plan and the user who submitted it.
/// * server_profile_ tracks time spent inside the ImpalaServer,
@@ -377,7 +370,6 @@ class ClientRequestState {
/// - Query Status
/// - Error logs
RuntimeProfile* const profile_;
- RuntimeProfile* const frontend_profile_;
RuntimeProfile* const server_profile_;
RuntimeProfile* const summary_profile_;
RuntimeProfile::Counter* row_materialization_timer_;
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index af9414c..649f1fc 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -965,7 +965,6 @@ Status ImpalaServer::ExecuteInternal(
(*request_state)->set_user_profile_access(result.user_has_profile_access);
(*request_state)->summary_profile()->AddEventSequence(
result.timeline.name, result.timeline);
- (*request_state)->SetFrontendProfile(result.profile);
if (result.__isset.result_set_metadata) {
(*request_state)->set_result_metadata(result.result_set_metadata);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/be/src/util/runtime-profile.h
----------------------------------------------------------------------
diff --git a/be/src/util/runtime-profile.h b/be/src/util/runtime-profile.h
index a6b06ba..cae2462 100644
--- a/be/src/util/runtime-profile.h
+++ b/be/src/util/runtime-profile.h
@@ -263,7 +263,7 @@ class RuntimeProfile { // NOLINT: This struct is not packed, but there are not s
Counter* inactive_timer() { return counter_map_[INACTIVE_TIME_COUNTER_NAME]; }
int64_t local_time() { return local_time_ns_; }
- /// Prints the contents of the profile in a name: value format.
+ /// Prints the counters in a name: value format.
/// Does not hold locks when it makes any function calls.
void PrettyPrint(std::ostream* s, const std::string& prefix="") const;
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/common/thrift/Frontend.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index 48e915a..f8cd472 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -634,7 +634,6 @@ struct TExecRequest {
10: optional TSetQueryOptionRequest set_query_option_request
// Timeline of planner's operation, for profiling
- // TODO(todd): should integrate this with the 'profile' member instead.
11: optional RuntimeProfile.TEventSequence timeline
// If false, the user that runs this statement doesn't have access to the runtime
@@ -642,11 +641,8 @@ struct TExecRequest {
// that has a view for which the user doesn't have access to the underlying tables.
12: optional bool user_has_profile_access
- // Profile information from the planning process.
- 13: optional RuntimeProfile.TRuntimeProfileNode profile
-
// Set iff stmt_type is ADMIN_FN.
- 14: optional TAdminRequest admin_request
+ 13: optional TAdminRequest admin_request
}
// Parameters to FeSupport.cacheJar().
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
index 62f1d3e..2684c33 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
@@ -48,7 +48,6 @@ import org.apache.impala.common.InternalException;
import org.apache.impala.common.Pair;
import org.apache.impala.common.Reference;
import org.apache.impala.service.FeSupport;
-import org.apache.impala.service.FrontendProfile;
import org.apache.impala.thrift.CatalogLookupStatus;
import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.thrift.TCatalogInfoSelector;
@@ -67,7 +66,6 @@ import org.apache.impala.thrift.TPartialPartitionInfo;
import org.apache.impala.thrift.TTable;
import org.apache.impala.thrift.TTableInfoSelector;
import org.apache.impala.thrift.TUniqueId;
-import org.apache.impala.thrift.TUnit;
import org.apache.impala.thrift.TUpdateCatalogCacheRequest;
import org.apache.impala.thrift.TUpdateCatalogCacheResponse;
import org.apache.impala.util.ListMap;
@@ -83,7 +81,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@@ -188,25 +185,6 @@ public class CatalogdMetaProvider implements MetaProvider {
*/
private static final Object DB_LIST_CACHE_KEY = new Object();
- private static final String CATALOG_FETCH_PREFIX = "CatalogFetch";
- private static final String DB_LIST_STATS_CATEGORY = "DatabaseList";
- private static final String DB_METADATA_STATS_CATEGORY = "Databases";
- private static final String TABLE_NAMES_STATS_CATEGORY = "TableNames";
- private static final String TABLE_METADATA_CACHE_CATEGORY = "Tables";
- private static final String PARTITION_LIST_STATS_CATEGORY = "PartitionLists";
- private static final String PARTITIONS_STATS_CATEGORY = "Partitions";
- private static final String COLUMN_STATS_STATS_CATEGORY = "ColumnStats";
- private static final String GLOBAL_CONFIGURATION_STATS_CATEGORY = "Config";
- private static final String FUNCTION_LIST_STATS_CATEGORY = "FunctionLists";
- private static final String FUNCTIONS_STATS_CATEGORY = "Functions";
- private static final String RPC_STATS_CATEGORY = "RPCs";
- private static final String RPC_REQUESTS =
- CATALOG_FETCH_PREFIX + "." + RPC_STATS_CATEGORY + ".Requests";
- private static final String RPC_BYTES =
- CATALOG_FETCH_PREFIX + "." + RPC_STATS_CATEGORY + ".Bytes";
- private static final String RPC_TIME =
- CATALOG_FETCH_PREFIX + "." + RPC_STATS_CATEGORY + ".Time";
-
/**
* File descriptors store replicas using a compressed format that references hosts
* by index in a "host index" list rather than by their full addresses. Since we cache
@@ -312,20 +290,11 @@ public class CatalogdMetaProvider implements MetaProvider {
TGetPartialCatalogObjectRequest req)
throws TException {
TGetPartialCatalogObjectResponse resp;
- byte[] ret = null;
- Stopwatch sw = new Stopwatch().start();
+ byte[] ret;
try {
ret = FeSupport.GetPartialCatalogObject(new TSerializer().serialize(req));
} catch (InternalException e) {
throw new TException(e);
- } finally {
- sw.stop();
- FrontendProfile profile = FrontendProfile.getCurrentOrNull();
- if (profile != null) {
- profile.addToCounter(RPC_REQUESTS, TUnit.NONE, 1);
- profile.addToCounter(RPC_BYTES, TUnit.BYTES, ret == null ? 0 : ret.length);
- profile.addToCounter(RPC_TIME, TUnit.TIME_MS, sw.elapsed(TimeUnit.MILLISECONDS));
- }
}
resp = new TGetPartialCatalogObjectResponse();
new TDeserializer().deserialize(resp, ret);
@@ -374,15 +343,13 @@ public class CatalogdMetaProvider implements MetaProvider {
@SuppressWarnings("unchecked")
private <CacheKeyType, ValueType> ValueType loadWithCaching(String itemString,
- String statsCategory, CacheKeyType key,
- final Callable<ValueType> loadCallable) throws TException {
+ CacheKeyType key, final Callable<ValueType> loadCallable) throws TException {
// TODO(todd): there a race here if an invalidation comes in while we are
// fetching here. Perhaps we need some locking, or need to remember the
// version numbers of the invalidation messages and ensure that we don't
// 'put' an element with a too-old version? See:
// https://softwaremill.com/race-condition-cache-guava-caffeine/
final Reference<Boolean> hit = new Reference<>(true);
- Stopwatch sw = new Stopwatch().start();
try {
return (ValueType)cache_.get(key, new Callable<ValueType>() {
@Override
@@ -395,39 +362,13 @@ public class CatalogdMetaProvider implements MetaProvider {
Throwables.propagateIfPossible(e.getCause(), TException.class);
throw new RuntimeException(e);
} finally {
- sw.stop();
- addStatsToProfile(statsCategory, /*numHits=*/hit.getRef() ? 1 : 0,
- /*numMisses=*/hit.getRef() ? 0 : 1, sw);
LOG.trace("Request for {}: {}", itemString, hit.getRef() ? "hit" : "miss");
}
}
- /**
- * Adds basic statistics to the query's profile when accessing cache entries.
- * For each cache request, the number of hits, misses, and elapsed time is aggregated.
- * Cache requests for different types of cache entries, such as function names vs.
- * table names, are differentiated by a 'statsCategory'.
- */
- private void addStatsToProfile(String statsCategory, int numHits, int numMisses,
- Stopwatch stopwatch) {
- FrontendProfile profile = FrontendProfile.getCurrentOrNull();
- if (profile == null) return;
- final String prefix = CATALOG_FETCH_PREFIX + "." +
- Preconditions.checkNotNull(statsCategory) + ".";
- profile.addToCounter(prefix + "Requests", TUnit.NONE, numHits + numMisses);;
- profile.addToCounter(prefix + "Time", TUnit.TIME_MS,
- stopwatch.elapsed(TimeUnit.MILLISECONDS));
- if (numHits > 0) {
- profile.addToCounter(prefix + "Hits", TUnit.NONE, numHits);
- }
- if (numMisses > 0) {
- profile.addToCounter(prefix + "Misses", TUnit.NONE, numMisses);
- }
- }
-
@Override
public ImmutableList<String> loadDbList() throws TException {
- return loadWithCaching("database list", DB_LIST_STATS_CATEGORY, DB_LIST_CACHE_KEY,
+ return loadWithCaching("database list", DB_LIST_CACHE_KEY,
new Callable<ImmutableList<String>>() {
@Override
public ImmutableList<String> call() throws Exception {
@@ -474,7 +415,6 @@ public class CatalogdMetaProvider implements MetaProvider {
@Override
public Database loadDb(final String dbName) throws TException {
return loadWithCaching("database metadata for " + dbName,
- DB_METADATA_STATS_CATEGORY,
new DbCacheKey(dbName, DbCacheKey.DbInfoType.HMS_METADATA),
new Callable<Database>() {
@Override
@@ -493,7 +433,6 @@ public class CatalogdMetaProvider implements MetaProvider {
public ImmutableList<String> loadTableNames(final String dbName)
throws MetaException, UnknownDBException, TException {
return loadWithCaching("table names for database " + dbName,
- TABLE_NAMES_STATS_CATEGORY,
new DbCacheKey(dbName, DbCacheKey.DbInfoType.TABLE_NAMES),
new Callable<ImmutableList<String>>() {
@Override
@@ -535,7 +474,6 @@ public class CatalogdMetaProvider implements MetaProvider {
TableCacheKey cacheKey = new TableCacheKey(dbName, tableName);
TableMetaRefImpl ref = loadWithCaching(
"table metadata for " + dbName + "." + tableName,
- TABLE_METADATA_CACHE_CATEGORY,
cacheKey,
new Callable<TableMetaRefImpl>() {
@Override
@@ -555,7 +493,6 @@ public class CatalogdMetaProvider implements MetaProvider {
@Override
public List<ColumnStatisticsObj> loadTableColumnStatistics(final TableMetaRef table,
List<String> colNames) throws TException {
- Stopwatch sw = new Stopwatch().start();
List<ColumnStatisticsObj> ret = Lists.newArrayListWithCapacity(colNames.size());
// Look up in cache first, keeping track of which ones are missing.
// We can't use 'loadWithCaching' since we need to fetch several entries batched
@@ -597,9 +534,6 @@ public class CatalogdMetaProvider implements MetaProvider {
NEGATIVE_COLUMN_STATS_SENTINEL);
}
}
- sw.stop();
- addStatsToProfile(COLUMN_STATS_STATS_CATEGORY,
- hitCount + negativeHitCount, missingCols.size(), sw);
LOG.trace("Request for column stats of {}: hit {}/ neg hit {} / miss {}",
table, hitCount, negativeHitCount, missingCols.size());
return ret;
@@ -610,8 +544,8 @@ public class CatalogdMetaProvider implements MetaProvider {
public List<PartitionRef> loadPartitionList(final TableMetaRef table)
throws TException {
PartitionListCacheKey key = new PartitionListCacheKey((TableMetaRefImpl) table);
- return (List<PartitionRef>) loadWithCaching("partition list for " + table,
- PARTITION_LIST_STATS_CATEGORY, key, new Callable<List<PartitionRef>>() {
+ return (List<PartitionRef>) loadWithCaching(
+ "partition list for " + table, key, new Callable<List<PartitionRef>>() {
/** Called to load cache for cache misses */
@Override
public List<PartitionRef> call() throws Exception {
@@ -640,13 +574,13 @@ public class CatalogdMetaProvider implements MetaProvider {
throws MetaException, TException {
Preconditions.checkArgument(table instanceof TableMetaRefImpl);
TableMetaRefImpl refImpl = (TableMetaRefImpl)table;
- Stopwatch sw = new Stopwatch().start();
+
// Load what we can from the cache.
Map<PartitionRef, PartitionMetadata> refToMeta = loadPartitionsFromCache(refImpl,
hostIndex, partitionRefs);
- final int numHits = refToMeta.size();
- final int numMisses = partitionRefs.size() - numHits;
+ LOG.trace("Request for partitions of {}: hit {}/{}", table, refToMeta.size(),
+ partitionRefs.size());
// Load the remainder from the catalogd.
List<PartitionRef> missingRefs = Lists.newArrayList();
@@ -660,10 +594,6 @@ public class CatalogdMetaProvider implements MetaProvider {
// Write back to the cache.
storePartitionsInCache(refImpl, hostIndex, fromCatalogd);
}
- sw.stop();
- addStatsToProfile(PARTITIONS_STATS_CATEGORY, refToMeta.size(), numMisses, sw);
- LOG.trace("Request for partitions of {}: hit {}/{}", table, refToMeta.size(),
- partitionRefs.size());
// Convert the returned map to be by-name instead of by-ref.
Map<String, PartitionMetadata> nameToMeta = Maps.newHashMapWithExpectedSize(
@@ -794,9 +724,7 @@ public class CatalogdMetaProvider implements MetaProvider {
@Override
public String loadNullPartitionKeyValue() throws MetaException, TException {
return (String) loadWithCaching("null partition key value",
- GLOBAL_CONFIGURATION_STATS_CATEGORY,
- NULL_PARTITION_KEY_VALUE_CACHE_KEY,
- new Callable<String>() {
+ NULL_PARTITION_KEY_VALUE_CACHE_KEY, new Callable<String>() {
/** Called to load cache for cache misses */
@Override
public String call() throws Exception {
@@ -808,7 +736,6 @@ public class CatalogdMetaProvider implements MetaProvider {
@Override
public List<String> loadFunctionNames(final String dbName) throws TException {
return loadWithCaching("function names for database " + dbName,
- FUNCTION_LIST_STATS_CATEGORY,
new DbCacheKey(dbName, DbCacheKey.DbInfoType.FUNCTION_NAMES),
new Callable<ImmutableList<String>>() {
@Override
@@ -828,7 +755,6 @@ public class CatalogdMetaProvider implements MetaProvider {
final String functionName) throws TException {
ImmutableList<TFunction> thriftFuncs = loadWithCaching(
"function " + dbName + "." + functionName,
- FUNCTIONS_STATS_CATEGORY,
new FunctionsCacheKey(dbName, functionName),
new Callable<ImmutableList<TFunction>>() {
@Override
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/fe/src/main/java/org/apache/impala/service/Frontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index d31b573..39983ea 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -1061,14 +1061,11 @@ public class Frontend {
throws ImpalaException {
// Timeline of important events in the planning process, used for debugging
// and profiling.
- try (FrontendProfile.Scope scope = FrontendProfile.createNewWithScope()) {
- EventSequence timeline = new EventSequence("Query Compilation");
- TExecRequest result = getTExecRequest(queryCtx, timeline, explainString);
- timeline.markEvent("Planning finished");
- result.setTimeline(timeline.toThrift());
- result.setProfile(FrontendProfile.getCurrent().emitAsThrift());
- return result;
- }
+ EventSequence timeline = new EventSequence("Query Compilation");
+ TExecRequest result = getTExecRequest(queryCtx, timeline, explainString);
+ timeline.markEvent("Planning finished");
+ result.setTimeline(timeline.toThrift());
+ return result;
}
/**
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/fe/src/main/java/org/apache/impala/service/FrontendProfile.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/FrontendProfile.java b/fe/src/main/java/org/apache/impala/service/FrontendProfile.java
deleted file mode 100644
index 3344bf6..0000000
--- a/fe/src/main/java/org/apache/impala/service/FrontendProfile.java
+++ /dev/null
@@ -1,163 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.apache.impala.service;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-import javax.annotation.concurrent.ThreadSafe;
-
-import org.apache.impala.thrift.TCounter;
-import org.apache.impala.thrift.TRuntimeProfileNode;
-import org.apache.impala.thrift.TUnit;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.errorprone.annotations.concurrent.GuardedBy;
-
-/**
- * Wrapper class for creating a runtime profile within the frontend.
- *
- * In order to avoid plumbing an object through all code that might want to emit counters
- * into the profile, this class provides some support for storing a current profile in
- * a thread-local variable.
- *
- * This class is thread-safe.
- */
-@ThreadSafe
-public class FrontendProfile {
- private static final String ROOT_COUNTER_NAME = "";
-
- private static ThreadLocal<FrontendProfile> THREAD_LOCAL =
- new ThreadLocal<>();
-
- @GuardedBy("this")
- private TRuntimeProfileNode profile_;
-
- /**
- * Name-based access to the counters in the profile_.counters List<TCounter>.
- */
- @GuardedBy("this")
- private final Map<String, TCounter> countersByName_ = new HashMap<>();
-
- FrontendProfile() {
- profile_ = new TRuntimeProfileNode("Frontend",
- /*num_children=*/ 0,
- /*counters=*/new ArrayList<>(),
- /*metadata=*/-1L, // TODO(todd) what is this used for? why is it required?
- /*indent=*/false,
- /*info_strings=*/new HashMap<>(),
- /*info_strings_display_order*/new ArrayList<>(),
- /*child_counters_map=*/ImmutableMap.of(ROOT_COUNTER_NAME, new HashSet<>()));
- }
-
- /**
- * Create a new profile, setting it as the current thread-local profile for the
- * length of the current scope. This is meant to be used in a try-with-resources
- * statement. Supports at most one scope per thread. No nested scopes are currently
- * allowed.
- */
- public static Scope createNewWithScope() {
- return new Scope(new FrontendProfile());
- }
-
- /**
- * Get the profile attached to the current thread, throw IllegalStateException if there
- * is none.
- */
- @Nonnull
- public static FrontendProfile getCurrent() {
- FrontendProfile prof = THREAD_LOCAL.get();
- Preconditions.checkState(prof != null, "no profile in scope");
- return prof;
- }
-
- /**
- * Get the profile attached to the current thread, or null if there is no current
- * profile.
- */
- @Nullable
- public static FrontendProfile getCurrentOrNull() {
- return THREAD_LOCAL.get();
- }
-
- /**
- * Return the profile in Thrift format. This may be called only once, and after it is
- * called, no further methods may be used on this PlannerProfile object. Any attempts
- * to do so will result in IllegalStateExceptions.
- */
- public synchronized TRuntimeProfileNode emitAsThrift() {
- Preconditions.checkState(profile_ != null, "already emitted profile");
- TRuntimeProfileNode ret = profile_;
- profile_ = null;
- return ret;
- }
-
- /**
- * Add an informational key/value string pair to the profile. These are written out
- * as is to the user. Subsequent calls with the same key will overwrite previous ones.
- */
- public synchronized void addInfoString(String key, String val) {
- Preconditions.checkState(profile_ != null, "already emitted profile");
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(val);
- if (profile_.getInfo_strings().put(key, val) == null) {
- // If it's a new info string instead of replacing an existing one,
- // we need to also include it in the 'ordering' list.
- profile_.getInfo_strings_display_order().add(key);
- }
- }
-
- /**
- * Add 'delta' to the counter with the given name and unit. Counters are created
- * on-demand.
- */
- public synchronized void addToCounter(String name, TUnit unit, long delta) {
- Preconditions.checkState(profile_ != null, "already emitted profile");
- TCounter counter = countersByName_.get(Preconditions.checkNotNull(name));
- if (counter == null) {
- // Need to create the counter.
- counter = new TCounter(name, unit, 0);
- countersByName_.put(name, counter);
- profile_.counters.add(counter);
- // Currently we don't support hierarchical counters in the frontend.
- profile_.child_counters_map.get(ROOT_COUNTER_NAME).add(name);
- }
- counter.value += delta;
- }
-
-
- public static class Scope implements AutoCloseable {
- private final FrontendProfile oldThreadLocalValue_;
-
- private Scope(FrontendProfile profile) {
- oldThreadLocalValue_ = THREAD_LOCAL.get();
- // TODO: remove when allowing nested scopes.
- Preconditions.checkState(oldThreadLocalValue_ == null);
- THREAD_LOCAL.set(profile);
- }
-
- @Override
- public void close() {
- THREAD_LOCAL.set(oldThreadLocalValue_);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/fe/src/test/java/org/apache/impala/catalog/local/CatalogdMetaProviderTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/catalog/local/CatalogdMetaProviderTest.java b/fe/src/test/java/org/apache/impala/catalog/local/CatalogdMetaProviderTest.java
index de6dd07..74264c6 100644
--- a/fe/src/test/java/org/apache/impala/catalog/local/CatalogdMetaProviderTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/local/CatalogdMetaProviderTest.java
@@ -19,7 +19,6 @@ package org.apache.impala.catalog.local;
import static org.junit.Assert.*;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -32,13 +31,11 @@ import org.apache.impala.catalog.local.MetaProvider.PartitionRef;
import org.apache.impala.catalog.local.MetaProvider.TableMetaRef;
import org.apache.impala.common.Pair;
import org.apache.impala.service.FeSupport;
-import org.apache.impala.service.FrontendProfile;
import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.thrift.TCatalogObject;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TDatabase;
import org.apache.impala.thrift.TNetworkAddress;
-import org.apache.impala.thrift.TRuntimeProfileNode;
import org.apache.impala.thrift.TTable;
import org.apache.impala.util.ListMap;
import org.junit.Test;
@@ -224,21 +221,4 @@ public class CatalogdMetaProviderTest {
assertEquals(0, stats.hitCount());
assertEquals(1, stats.missCount());
}
-
- @Test
- public void testProfile() throws Exception {
- FrontendProfile profile;
- try (FrontendProfile.Scope scope = FrontendProfile.createNewWithScope()) {
- provider_.loadTable("functional", "alltypes");
- profile = FrontendProfile.getCurrent();
- }
- TRuntimeProfileNode prof = profile.emitAsThrift();
- assertEquals(3, prof.counters.size());
- Collections.sort(prof.counters);
- assertEquals("TCounter(name:CatalogFetch.Tables.Hits, unit:NONE, value:1)",
- prof.counters.get(0).toString());
- assertEquals("TCounter(name:CatalogFetch.Tables.Requests, unit:NONE, value:1)",
- prof.counters.get(1).toString());
- assertEquals("CatalogFetch.Tables.Time", prof.counters.get(2).name);
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/impala/blob/ea2809f5/tests/custom_cluster/test_local_catalog.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index 14a9a54..b78ef88 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -306,8 +306,9 @@ class TestCompactCatalogUpdates(CustomClusterTestSuite):
for _ in xrange(0, 10):
for query in queries_to_test:
ret = self.execute_query_expect_success(client, query)
- assert ret.runtime_profile.count("Frontend:") == 1
- assert ret.runtime_profile.count("CatalogFetch") > 1
+ # TODO: re-enable checks when counters are put back into profile
+ # assert ret.runtime_profile.count("Frontend:") == 1
+ # assert ret.runtime_profile.count("CatalogFetch") > 1
cache_metrics = self.get_catalog_cache_metrics(impalad)
cache_hit_rate = cache_metrics[cache_hit_rate_metric_key]
cache_miss_rate = cache_metrics[cache_miss_rate_metric_key]
[7/8] impala git commit: IMPALA-7629: Re-enable erroneously disabled
TestClientSsl tests.
Posted by ta...@apache.org.
IMPALA-7629: Re-enable erroneously disabled TestClientSsl tests.
The fix for IMPALA-6990 had a bug, disabling some tests erroneously.
With this change, the tests run on Ubuntu16:04 like so:
tests/custom_cluster/test_client_ssl.py::TestClientSsl::test_ssl[] PASSED
tests/custom_cluster/test_client_ssl.py::TestClientSsl::test_tls_ecdh[] PASSED
tests/custom_cluster/test_client_ssl.py::TestClientSsl::test_tls_v12[] PASSED
tests/custom_cluster/test_client_ssl.py::TestClientSsl::test_wildcard_ssl[] xfail
tests/custom_cluster/test_client_ssl.py::TestClientSsl::test_wildcard_san_ssl[] xfail
The xfails are all "Inconsistent wildcard support on target platforms".
On centos7:
custom_cluster/test_client_ssl.py::TestClientSsl::test_ssl[] PASSED
custom_cluster/test_client_ssl.py::TestClientSsl::test_tls_ecdh[] SKIPPED
custom_cluster/test_client_ssl.py::TestClientSsl::test_tls_v12[] SKIPPED
custom_cluster/test_client_ssl.py::TestClientSsl::test_wildcard_ssl[] xfail
custom_cluster/test_client_ssl.py::TestClientSsl::test_wildcard_san_ssl[] xfail
On centos6:
custom_cluster/test_client_ssl.py::TestClientSsl::test_ssl[] PASSED
custom_cluster/test_client_ssl.py::TestClientSsl::test_tls_ecdh[] SKIPPED
custom_cluster/test_client_ssl.py::TestClientSsl::test_tls_v12[] SKIPPED
custom_cluster/test_client_ssl.py::TestClientSsl::test_wildcard_ssl[] SKIPPED
custom_cluster/test_client_ssl.py::TestClientSsl::test_wildcard_san_ssl[] SKIPPED
I used "curl --silent https://.../consoleText | grep test_client_ssl | sed -e 's/\[.*\]/[]/'"
to extract these from Jenkins output.
Change-Id: I64879b8af39f967b0059797e7b36421ce0e58bed
Reviewed-on: http://gerrit.cloudera.org:8080/11530
Reviewed-by: Philip Zeyliger <ph...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d3cf6d32
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d3cf6d32
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d3cf6d32
Branch: refs/heads/master
Commit: d3cf6d325779fff4ab0ba9db53411c510aa0f717
Parents: ade399c
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Wed Sep 26 15:50:47 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Oct 2 01:54:41 2018 +0000
----------------------------------------------------------------------
tests/custom_cluster/test_client_ssl.py | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/d3cf6d32/tests/custom_cluster/test_client_ssl.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py
index 81afd52..b6f7e04 100644
--- a/tests/custom_cluster/test_client_ssl.py
+++ b/tests/custom_cluster/test_client_ssl.py
@@ -32,11 +32,14 @@ from tests.shell.util import run_impala_shell_cmd, run_impala_shell_cmd_no_expec
REQUIRED_MIN_OPENSSL_VERSION = 0x10001000L
REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12 = (2,7,9)
-SKIP_SSL_MSG = "Legacy OpenSSL module detected"
-HAS_LEGACY_OPENSSL = getattr(ssl, "OPENSSL_VERSION_NUMBER", None)
-if HAS_LEGACY_OPENSSL is not None:
+_openssl_version_number = getattr(ssl, "OPENSSL_VERSION_NUMBER", None)
+if _openssl_version_number is None:
+ SKIP_SSL_MSG = "Legacy OpenSSL module detected"
+elif _openssl_version_number < REQUIRED_MIN_OPENSSL_VERSION:
SKIP_SSL_MSG = "Only have OpenSSL version %X, but test requires %X" % (
ssl.OPENSSL_VERSION_NUMBER, REQUIRED_MIN_OPENSSL_VERSION)
+else:
+ SKIP_SSL_MSG = None
class TestClientSsl(CustomClusterTestSuite):
"""Tests for a client using SSL (particularly, the Impala Shell) """
@@ -119,7 +122,7 @@ class TestClientSsl(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args(impalad_args=TLS_ECDH_ARGS,
statestored_args=TLS_ECDH_ARGS,
catalogd_args=TLS_ECDH_ARGS)
- @pytest.mark.skipif(HAS_LEGACY_OPENSSL, reason=SKIP_SSL_MSG)
+ @pytest.mark.skipif(SKIP_SSL_MSG is not None, reason=SKIP_SSL_MSG)
@pytest.mark.skipif(sys.version_info < REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12,
reason="Working around IMPALA-7628. TODO: is the right workaround?")
def test_tls_ecdh(self, vector):
@@ -140,7 +143,7 @@ class TestClientSsl(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args(impalad_args=TLS_V12_ARGS,
statestored_args=TLS_V12_ARGS,
catalogd_args=TLS_V12_ARGS)
- @pytest.mark.skipif(HAS_LEGACY_OPENSSL, reason=SKIP_SSL_MSG)
+ @pytest.mark.skipif(SKIP_SSL_MSG is not None, reason=SKIP_SSL_MSG)
@pytest.mark.skipif(sys.version_info < REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12, \
reason="Python version too old to allow Thrift client to use TLSv1.2")
def test_tls_v12(self, vector):
@@ -150,7 +153,7 @@ class TestClientSsl(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args(impalad_args=SSL_WILDCARD_ARGS,
statestored_args=SSL_WILDCARD_ARGS,
catalogd_args=SSL_WILDCARD_ARGS)
- @pytest.mark.skipif(HAS_LEGACY_OPENSSL, reason=SKIP_SSL_MSG)
+ @pytest.mark.skipif(SKIP_SSL_MSG is not None, reason=SKIP_SSL_MSG)
@pytest.mark.xfail(run=True, reason="Inconsistent wildcard support on target platforms")
def test_wildcard_ssl(self, vector):
""" Test for IMPALA-3159: Test with a certificate which has a wildcard for the
@@ -164,7 +167,7 @@ class TestClientSsl(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args(impalad_args=SSL_WILDCARD_SAN_ARGS,
statestored_args=SSL_WILDCARD_SAN_ARGS,
catalogd_args=SSL_WILDCARD_SAN_ARGS)
- @pytest.mark.skipif(HAS_LEGACY_OPENSSL, reason=SKIP_SSL_MSG)
+ @pytest.mark.skipif(SKIP_SSL_MSG is not None, reason=SKIP_SSL_MSG)
@pytest.mark.xfail(run=True, reason="Inconsistent wildcard support on target platforms")
def test_wildcard_san_ssl(self, vector):
""" Test for IMPALA-3159: Test with a certificate which has a wildcard as a SAN. """
[4/8] impala git commit: IMPALA-7646: SHOW GRANT USER does not work
for kerberos cluster
Posted by ta...@apache.org.
IMPALA-7646: SHOW GRANT USER does not work for kerberos cluster
This patch fixes the SHOW GRANT USER statement to properly check
that the requesting user short name matches the name in the
SHOW GRANT USER statement to determine whether or not an admin
check is required for showing the privileges. Previous to this
patch, the full kerberos user name, e.g. foo_user@REALM was
compared against "SHOW GRANT USER foo_user" and did not match
do admin privileges were required.
Testing:
- Ran all fe and custom cluster tests.
- Validated against kerberized cluster.
Change-Id: Iba4c627b72c8cbc323be25917698a75d153afd31
Reviewed-on: http://gerrit.cloudera.org:8080/11553
Reviewed-by: Fredy Wijaya <fw...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/a381483e
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/a381483e
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/a381483e
Branch: refs/heads/master
Commit: a381483e658f824d1639096e03f1a23b2a216c41
Parents: d918b2a
Author: Adam Holley <gi...@holleyism.com>
Authored: Mon Oct 1 11:23:57 2018 -0500
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Mon Oct 1 22:48:46 2018 +0000
----------------------------------------------------------------------
fe/src/main/java/org/apache/impala/service/Frontend.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/a381483e/fe/src/main/java/org/apache/impala/service/Frontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index ec6189f..d31b573 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -508,7 +508,7 @@ public class Frontend {
if (showGrantPrincipalStmt.getPrincipal().getPrincipalType()
== TPrincipalType.USER) {
requiresAdmin = !showGrantPrincipalStmt.getPrincipal().getName().equals(
- analysis.getAnalyzer().getUser().getName());
+ analysis.getAnalyzer().getUser().getShortName());
} else {
requiresAdmin = Sets.intersection(groupNames, showGrantPrincipalStmt
.getPrincipal().getGrantGroups()).isEmpty();
[5/8] impala git commit: IMPALA-7520: Fix NullPointerException in
SentryProxy
Posted by ta...@apache.org.
IMPALA-7520: Fix NullPointerException in SentryProxy
Prior to this patch, the code in SentryProxy could throw a
NullPointerException when trying to retrieve a set of privileges for a
given role name. I was able to manually reproduce the issue by doing
the following steps:
1. Get all Sentry role privileges: [a, b] --> in SentryProxy
2. Add a sleep statement before getting all Sentry roles --> in SentryProxy
3. Add a new Sentry role: [c] --> Externally via Sentry CLI
4. Get all Sentry roles: [a, b, c] --> in SentryProxy
Role c was added in step 3.
5. Get Sentry role privileges for role c: NPE --> in SentryProxy
The fix is to add a null guard when retrieving Sentry privileges for a
given role name and let the new role get updated in the next Sentry
refresh.
Testing:
- Ran all FE tests
- Ran all authorization E2E tests
- Manually tested it by temporarily modifying the SentryProxy code and
did not see the NullPointerException
Change-Id: I36af840056a4d037fb5c7b1d9a167c0eb8526a11
Reviewed-on: http://gerrit.cloudera.org:8080/11552
Reviewed-by: Fredy Wijaya <fw...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/de39b033
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/de39b033
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/de39b033
Branch: refs/heads/master
Commit: de39b0331e1e000162a93bfb90888e2dfbadcd13
Parents: a381483
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Mon Oct 1 08:38:00 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Oct 2 00:30:18 2018 +0000
----------------------------------------------------------------------
fe/src/main/java/org/apache/impala/util/SentryProxy.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/de39b033/fe/src/main/java/org/apache/impala/util/SentryProxy.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/SentryProxy.java b/fe/src/main/java/org/apache/impala/util/SentryProxy.java
index b47d0ce..f74e033 100644
--- a/fe/src/main/java/org/apache/impala/util/SentryProxy.java
+++ b/fe/src/main/java/org/apache/impala/util/SentryProxy.java
@@ -209,7 +209,7 @@ public class SentryProxy {
*/
private Set<String> refreshUserPrivileges() throws ImpalaException {
// Assume all users should be removed. Then query the Policy Service and remove
- // roles from this set that actually exist.
+ // users from this set that actually exist.
Set<String> usersToRemove = catalog_.getAuthPolicy().getAllUserNames();
Map<String, Set<TSentryPrivilege>> allUsersPrivileges =
sentryPolicyService_.listAllUsersPrivileges(processUser_);
@@ -242,8 +242,11 @@ public class SentryProxy {
// deleted from this set and we are left with the set of privileges that need
// to be removed.
Set<String> privilegesToRemove = principal.getPrivilegeNames();
+ Set<TSentryPrivilege> sentryPrivileges = allPrincipalPrivileges.get(
+ principal.getName());
+ if (sentryPrivileges == null) return;
// Check all the privileges that are part of this principal.
- for (TSentryPrivilege sentryPriv: allPrincipalPrivileges.get(principal.getName())) {
+ for (TSentryPrivilege sentryPriv: sentryPrivileges) {
TPrivilege thriftPriv =
SentryPolicyService.sentryPrivilegeToTPrivilege(sentryPriv, principal);
String privilegeName = PrincipalPrivilege.buildPrivilegeName(thriftPriv);
[6/8] impala git commit: IMPALA-7532: Add catalogd client backoff
time into impalad CLI options
Posted by ta...@apache.org.
IMPALA-7532: Add catalogd client backoff time into impalad CLI options
Impala may fail queries or fail to start if the connection to catalogd
cannot be estabilished. Impala already has a retrial mechanism but the
backoff time is currently 0. This patch adds an option
"catalog_client_rpc_retry_interval_ms" for it, defaulting to 10 seconds.
Change-Id: I924c1f2fd37021f4c8fb6b46aa278ac4b1aee131
Reviewed-on: http://gerrit.cloudera.org:8080/11543
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ade399c0
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ade399c0
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ade399c0
Branch: refs/heads/master
Commit: ade399c08f5a74f87d192b47d2b65c3b56d05f7c
Parents: de39b03
Author: Tianyi Wang <tw...@cloudera.com>
Authored: Fri Sep 28 15:41:26 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Oct 2 00:53:49 2018 +0000
----------------------------------------------------------------------
be/src/runtime/exec-env.cc | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/ade399c0/be/src/runtime/exec-env.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index dd46789..5e15990 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -129,6 +129,8 @@ DEFINE_int32(backend_client_rpc_timeout_ms, 300000, "(Advanced) The underlying "
DEFINE_int32(catalog_client_connection_num_retries, 3, "Retry catalog connections.");
DEFINE_int32(catalog_client_rpc_timeout_ms, 0, "(Advanced) The underlying TSocket "
"send/recv timeout in milliseconds for a catalog client RPC.");
+DEFINE_int32(catalog_client_rpc_retry_interval_ms, 10000, "(Advanced) The time to wait "
+ "before retrying when the catalog RPC client fails to connect to catalogd.");
const static string DEFAULT_FS = "fs.defaultFS";
@@ -155,7 +157,8 @@ ExecEnv::ExecEnv(int backend_port, int krpc_port,
FLAGS_backend_client_rpc_timeout_ms, FLAGS_backend_client_rpc_timeout_ms, "",
!FLAGS_ssl_client_ca_certificate.empty())),
catalogd_client_cache_(
- new CatalogServiceClientCache(FLAGS_catalog_client_connection_num_retries, 0,
+ new CatalogServiceClientCache(FLAGS_catalog_client_connection_num_retries,
+ FLAGS_catalog_client_rpc_retry_interval_ms,
FLAGS_catalog_client_rpc_timeout_ms, FLAGS_catalog_client_rpc_timeout_ms, "",
!FLAGS_ssl_client_ca_certificate.empty())),
htable_factory_(new HBaseTableFactory()),
[2/8] impala git commit: IMPALA-7570: [DOCS] Added a table of all
built-in impala_functions
Posted by ta...@apache.org.
IMPALA-7570: [DOCS] Added a table of all built-in impala_functions
- Cleaned up no-value added texts.
- Added a table of built-in functions that users can use to get a link
to functions.
Because the functions are listed as <dlentry>, the above list
of functions has to be manually maintained. When there is a new function
or a removed function, update the above list. The link format is:
<xref href="xml file name#concept id/dlentry id">FUNCTION NAME</xref>
For example:
<xref href="impala_datetime_functions.xml#datetime_functions/weekofyear"
>WEEKOFYEAR</xref>
Change-Id: I2f6b024bc218a9158249f161fd16be10f16d19db
Reviewed-on: http://gerrit.cloudera.org:8080/11441
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/10bffe2f
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/10bffe2f
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/10bffe2f
Branch: refs/heads/master
Commit: 10bffe2f934720b848acd6b2fe8ce10b282d891d
Parents: 8108411
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Thu Sep 13 17:04:08 2018 -0700
Committer: Alex Rodoni <ar...@cloudera.com>
Committed: Mon Oct 1 17:52:05 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_functions.xml | 1469 ++++++++++++++++++++++++++++++---
1 file changed, 1366 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/10bffe2f/docs/topics/impala_functions.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_functions.xml b/docs/topics/impala_functions.xml
index ad51389..fe26d31 100644
--- a/docs/topics/impala_functions.xml
+++ b/docs/topics/impala_functions.xml
@@ -21,7 +21,13 @@ under the License.
<concept id="builtins">
<title id="title_functions">Impala Built-In Functions</title>
- <titlealts audience="PDF"><navtitle>Built-In Functions</navtitle></titlealts>
+
+ <titlealts audience="PDF">
+
+ <navtitle>Built-In Functions</navtitle>
+
+ </titlealts>
+
<prolog>
<metadata>
<data name="Category" value="Impala"/>
@@ -35,17 +41,10 @@ under the License.
<conbody>
- <!-- To do:
- Opportunity to conref some material between here and the "Functions" topic under "Schema Objects".
- -->
-
<p>
- Impala supports several categories of built-in functions. These functions let you perform mathematical
- calculations, string manipulation, date calculations, and other kinds of data transformations directly in
- <codeph>SELECT</codeph> statements. The built-in functions let a SQL query return results with all
- formatting, calculating, and type conversions applied, rather than performing time-consuming postprocessing
- in another application. By applying function calls where practical, you can make a SQL query that is as
- convenient as an expression in a procedural programming language or a formula in a spreadsheet.
+ Impala supports several categories of built-in functions. These functions let you perform
+ mathematical calculations, string manipulation, date calculations, and other kinds of data
+ transformations directly in SQL statements.
</p>
<p>
@@ -74,106 +73,1370 @@ under the License.
</li>
<li>
- Aggregation functions, explained in <xref href="impala_aggregate_functions.xml#aggregate_functions"/>.
+ <xref href="impala_aggregate_functions.xml#aggregate_functions"/>.
</li>
- </ul>
-
- <p>
- You call any of these functions through the <codeph>SELECT</codeph> statement. For most functions, you can
- omit the <codeph>FROM</codeph> clause and supply literal values for any required arguments:
- </p>
-
-<codeblock>select abs(-1);
-+---------+
-| abs(-1) |
-+---------+
-| 1 |
-+---------+
-
-select concat('The rain ', 'in Spain');
-+---------------------------------+
-| concat('the rain ', 'in spain') |
-+---------------------------------+
-| The rain in Spain |
-+---------------------------------+
-
-select power(2,5);
-+-------------+
-| power(2, 5) |
-+-------------+
-| 32 |
-+-------------+
-</codeblock>
-
- <p>
- When you use a <codeph>FROM</codeph> clause and specify a column name as a function argument, the function is
- applied for each item in the result set:
- </p>
-
-<!-- TK: make real output for these; change the queries if necessary to use tables I already have. -->
-
-<codeblock>select concat('Country = ',country_code) from all_countries where population > 100000000;
-select round(price) as dollar_value from product_catalog where price between 0.0 and 100.0;
-</codeblock>
-
- <p>
- Typically, if any argument to a built-in function is <codeph>NULL</codeph>, the result value is also
- <codeph>NULL</codeph>:
- </p>
-<codeblock>select cos(null);
-+-----------+
-| cos(null) |
-+-----------+
-| NULL |
-+-----------+
-
-select power(2,null);
-+----------------+
-| power(2, null) |
-+----------------+
-| NULL |
-+----------------+
-
-select concat('a',null,'b');
-+------------------------+
-| concat('a', null, 'b') |
-+------------------------+
-| NULL |
-+------------------------+
-</codeblock>
-
- <p conref="../shared/impala_common.xml#common/aggr1"/>
-
-<codeblock conref="../shared/impala_common.xml#common/aggr2"/>
-
- <p conref="../shared/impala_common.xml#common/aggr3"/>
-
- <p>
- Aggregate functions are a special category with different rules. These functions calculate a return value
- across all the items in a result set, so they do require a <codeph>FROM</codeph> clause in the query:
- </p>
+ <li>
+ <xref href="impala_analytic_functions.xml#analytic_functions"/>
+ </li>
-<!-- TK: make real output for these; change the queries if necessary to use tables I already have. -->
+ <li>
+ <xref href="impala_bit_functions.xml#bit_functions"/>
+ </li>
-<codeblock>select count(product_id) from product_catalog;
-select max(height), avg(height) from census_data where age > 20;
-</codeblock>
+ <li>
+ <xref href="impala_misc_functions.xml#misc_functions"/>
+ </li>
+ </ul>
<p>
- Aggregate functions also ignore <codeph>NULL</codeph> values rather than returning a <codeph>NULL</codeph>
- result. For example, if some rows have <codeph>NULL</codeph> for a particular column, those rows are ignored
- when computing the AVG() for that column. Likewise, specifying <codeph>COUNT(col_name)</codeph> in a query
- counts only those rows where <codeph>col_name</codeph> contains a non-<codeph>NULL</codeph> value.
- </p>
-
- <p rev="2.0.0">
- Analytic functions are a variation on aggregate functions. Instead of returning a single value, or an
- identical value for each group of rows, they can compute values that vary based on a <q>window</q> consisting
- of other rows around them in the result set.
+ The following is a list of built-in functions supported in Impala:
</p>
- <p outputclass="toc"/>
+ <table id="impala_functions" frame="none" colsep="0" rowsep="0">
+ <tgroup cols="1">
+ <colspec colnum="1" colname="col1"/>
+ <tbody>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/abs"
+ >ABS</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/acos"
+ >ACOS</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/add_months"
+ >ADD_MONTHS</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/adddate"
+ >ADDDATE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_appx_median.xml#appx_median">APPX_MEDIAN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/ascii"
+ >ASCII</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/asin"
+ >ASIN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/atan"
+ >ATAN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/atan2"
+ >ATAN2</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_avg.xml#avg">AVG</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#avg_analytic">AVG - Analytic
+ Function</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/base64decode"
+ >BASE64DECODE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/base64encode"
+ >BASE64ENCODE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/bitand"
+ >BITAND</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/bin"
+ >BIN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/bitnot"
+ >BITNOT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/bitor"
+ >BITOR</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/bitxor"
+ >BITXOR</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/btrim"
+ >BTRIM</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/case"
+ >CASE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/case2"
+ >CASE
+ WHEN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conversion_functions.xml#conversion_functions/cast"
+ >CAST</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/ceil">CEIL, CEILING,
+ DCEIL</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/char_length"
+ >CHAR_LENGTH</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/chr"
+ >CHR</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/coalesce"
+ >COALESCE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/concat"
+ >CONCAT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/concat_ws"
+ >CONCAT_WS</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/conv"
+ >CONV</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/cos"
+ >COS</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/cosh"
+ >COSH</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/cot"
+ >COT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_count.xml#count">COUNT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#count_analytic">COUNT - Analytic
+ Function</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/countset"
+ >COUNTSET</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#cume_dist"
+ >CUME_DIST</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_misc_functions.xml#misc_functions/current_database"
+ >CURRENT_DATABASE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/current_timestamp"
+ >CURRENT_TIMESTAMP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/date_add"
+ >DATE_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/date_part"
+ >DATE_PART</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/date_sub"
+ >DATE_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/date_trunc"
+ >DATE_TRUNC</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/datediff"
+ >DATEDIFF</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_datetime_functions.xml#datetime_functions/day"
+ >DAY</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/dayname"
+ >DAYNAME</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/dayofweek"
+ >DAYOFWEEK</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/dayofyear"
+ >DAYOFYEAR</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/days_add"
+ >DAYS_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/days_sub"
+ >DAYS_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/decode"
+ >DECODE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/degrees"
+ >DEGREES</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#dense_rank"
+ >DENSE_RANK</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/e">E</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_misc_functions.xml#misc_functions/effective_user"
+ >EFFECTIVE_USER</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/exp"
+ >EXP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/extract"
+ >EXTRACT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/factorial"
+ >FACTORIAL</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/find_in_set"
+ >FIND_IN_SET</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#first_value"
+ >FIRST_VALUE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/floor">FLOOR, DFLOOR</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/fmod"
+ >FMOD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/fnv_hash"
+ >FNV_HASH</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/from_unixtime"
+ >FROM_UNIXTIME</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/from_timestamp"
+ >FROM_TIMESTAMP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/from_utc_timestamp"
+ >FROM_UTC_TIMESTAMP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/getbit"
+ >GETBIT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/greatest"
+ >GREATEST</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/group_concat"
+ >GROUP_CONCAT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_group_concat.xml#group_concat">GROUP_CONCAT - Analytic
+ Function</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/hex"
+ >HEX</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_datetime_functions.xml#datetime_functions/hour"
+ >HOUR</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/hours_add"
+ >HOURS_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/hours_sub"
+ >HOURS_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/if"
+ >IF</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/ifnull"
+ >IFNULL</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/initcap"
+ >INITCAP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/instr"
+ >INSTR</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/int_months_between"
+ >INT_MONTHS_BETWEEN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/is_inf"
+ >IS_INF</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/is_nan"
+ >IS_NAN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/isfalse"
+ >ISFALSE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/isnotfalse"
+ >ISNOTFALSE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/isnottrue"
+ >ISNOTTRUE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/isnull"
+ >ISNULL</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/istrue"
+ >ISTRUE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#lag">LAG</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#last_value"
+ >LAST_VALUE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#lead">LEAD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/least"
+ >LEAST</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/left"
+ >LEFT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/length"
+ >LENGTH</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/ln">LN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/locate"
+ >LOCATE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/log"
+ >LOG</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/log10"
+ >LOG10</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/log2"
+ >LOG2</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/lower"
+ >LOWER,
+ LCASE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/lpad"
+ >LPAD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/ltrim"
+ >LTRIM</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_max.xml#max">MAX</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#max_analytic">MAX - Analytic
+ Function</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/max_int"
+ >MAX_INT,
+ MAX_TINYINT, MAX_SMALLINT, MAX_BIGINT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/microseconds_add"
+ >MICROSECONDS_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/microseconds_sub"
+ >MICROSECONDS_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/millisecond"
+ >MILLISECOND</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/milliseconds_add"
+ >MILLISECONDS_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/milliseconds_sub"
+ >MILLISECONDS_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_min.xml#min">MIN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#min_analytic">MIN - Analytic
+ Function</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/min_int"
+ >MIN_INT,
+ MIN_TINYINT, MIN_SMALLINT, MIN_BIGINT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/minute"
+ >MINUTE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/minutes_add"
+ >MINUTES_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/minutes_sub"
+ >MINUTES_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/mod"
+ >MOD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/month"
+ >MONTH</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/monthname"
+ >MONTHNAME</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/months_add"
+ >MONTHS_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/months_between"
+ >MONTHS_BETWEEN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/months_sub"
+ >MONTHS_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/murmur_hash"
+ >MURMUR_HASH</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/nanoseconds_add"
+ >NANOSECONDS_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/nanoseconds_sub"
+ >NANOSECONDS_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_ndv.xml#ndv">NDV</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/negative"
+ >NEGATIVE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/next_day"
+ >NEXT_DAY</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/nonnullvalue"
+ >NONNULLVALUE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_datetime_functions.xml#datetime_functions/now"
+ >NOW</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#ntile">NTILE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/nullif"
+ >NULLIF</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/nullifzero"
+ >NULLIFZERO</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/nullvalue"
+ >NULLVALUE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/nvl"
+ >NVL</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/nvl2"
+ >NVL2</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#over">OVER Clause</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/parse_url"
+ >PARSE_URL</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#percent_rank"
+ >PERCENT_RANK</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/pi">PI</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_misc_functions.xml#misc_functions/pid"
+ >PID</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/pmod"
+ >PMOD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/positive"
+ >POSITIVE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/pow">POW, POWER, DPOW,
+ FPOW</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/precision"
+ >PRECISION</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/quarter"
+ >QUARTER</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/quotient"
+ >QUOTIENT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/radians"
+ >RADIANS</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/rand">RAND, RANDOM</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#rank">RANK</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/regexp_escape"
+ >REGEXP_ESCAPE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/regexp_extract"
+ >REGEXP_EXTRACT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/regexp_like"
+ >REGEXP_LIKE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/regexp_replace"
+ >REGEXP_REPLACE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/repeat"
+ >REPEAT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/replace"
+ >REPLACE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/reverse"
+ >REVERSE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/right"
+ >RIGHT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/rotateleft"
+ >ROTATELEFT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/rotateright"
+ >ROTATERIGHT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/round">ROUND, DROUND</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#row_number"
+ >ROW_NUMBER</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/rpad"
+ >RPAD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/rtrim"
+ >RTRIM</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/scale"
+ >SCALE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/second"
+ >SECOND</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/seconds_add"
+ >SECONDS_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/seconds_sub"
+ >SECONDS_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/setbit"
+ >SETBIT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/shiftleft"
+ >SHIFTLEFT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_bit_functions.xml#bit_functions/shiftright"
+ >SHIFTRIGHT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/sign"
+ >SIGN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/sin"
+ >SIN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/sinh"
+ >SINH</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_misc_functions.xml#misc_functions/sleep"
+ >SLEEP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/space"
+ >SPACE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/split_part"
+ >SPLIT_PART</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/sqrt"
+ >SQRT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_stddev.xml#stddev">STDDEV, STDDEV_SAMP, STDDEV_POP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/strleft"
+ >STRLEFT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/strright"
+ >STRRIGHT</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/subdate"
+ >SUBDATE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/substr"
+ >SUBSTR,
+ SUBSTRING</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_sum.xml#sum">SUM</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_analytic_functions.xml#sum_analytic">SUM - Analytic
+ Function</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/tan"
+ >TAN</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/tanh"
+ >TANH</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/timeofday"
+ >TIMEOFDAY</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/timestamp_cmp"
+ >TIMESTAMP_CMP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/to_date"
+ >TO_DATE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/to_timestamp"
+ >TO_TIMESTAMP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/to_utc_timestamp"
+ >TO_UTC_TIMESTAMP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_string_functions.xml#string_functions/translate"
+ >TRANSLATE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/trim"
+ >TRIM</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/trunc"
+ >TRUNC</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/truncate"
+ >TRUNCATE,
+ DTRUNC, TRUNC</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conversion_functions.xml#conversion_functions/typeof"
+ >TYPEOF</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/unhex"
+ >UNHEX</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/unix_timestamp"
+ >UNIX_TIMESTAMP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_string_functions.xml#string_functions/upper"
+ >UPPER,
+ UCASE</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_misc_functions.xml#misc_functions/user"
+ >USER</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/utc_timestamp"
+ >UTC_TIMESTAMP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_misc_functions.xml#misc_functions/uuid"
+ >UUID</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_variance.xml#variance">VARIANCE, VARIANCE_SAMP, VARIANCE_POP,
+ VAR_SAMP, VAR_POP</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_misc_functions.xml#misc_functions/version"
+ >VERSION</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/weekofyear"
+ >WEEKOFYEAR</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/weeks_add"
+ >WEEKS_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/weeks_sub"
+ >WEEKS_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_math_functions.xml#math_functions/width_bucket"
+ >WIDTH_BUCKET</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref href="impala_datetime_functions.xml#datetime_functions/year"
+ >YEAR</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/years_add"
+ >YEARS_ADD</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_datetime_functions.xml#datetime_functions/years_sub"
+ >YEARS_SUB</xref>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <xref
+ href="impala_conditional_functions.xml#conditional_functions/zeroifnull"
+ >ZEROIFNULL</xref>
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
</conbody>
[3/8] impala git commit: Revert "IMPALA-7622: adds profile metrics
when fetching incremental stats"
Posted by ta...@apache.org.
Revert "IMPALA-7622: adds profile metrics when fetching incremental stats"
Breaks downstream dependence on profile (1/2 of changes).
This reverts commit 235748316c5cada5c58b3e84a4e20ee57f1c4a49.
Change-Id: I80b4c0e4b8487572285ac788ab0195896f221842
Reviewed-on: http://gerrit.cloudera.org:8080/11551
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d918b2ae
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d918b2ae
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d918b2ae
Branch: refs/heads/master
Commit: d918b2aeb582ca465dc3e5066a77a7b4dab39641
Parents: 10bffe2
Author: Vuk Ercegovac <ve...@cloudera.com>
Authored: Mon Oct 1 10:34:01 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Mon Oct 1 21:33:43 2018 +0000
----------------------------------------------------------------------
.../impala/analysis/ComputeStatsStmt.java | 43 +----------------
tests/common/custom_cluster_test_suite.py | 2 +-
tests/custom_cluster/test_pull_stats.py | 51 --------------------
3 files changed, 3 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/d918b2ae/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
index 24f387c..36f88f2 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
@@ -46,18 +46,15 @@ import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.service.BackendConfig;
import org.apache.impala.service.CatalogOpExecutor;
-import org.apache.impala.service.FrontendProfile;
import org.apache.impala.thrift.TComputeStatsParams;
import org.apache.impala.thrift.TErrorCode;
import org.apache.impala.thrift.TGetPartitionStatsResponse;
import org.apache.impala.thrift.TPartitionStats;
import org.apache.impala.thrift.TTableName;
-import org.apache.impala.thrift.TUnit;
import org.apache.log4j.Logger;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -117,21 +114,6 @@ public class ComputeStatsStmt extends StatementBase {
private static String AVRO_SCHEMA_MSG_SUFFIX = "Please re-create the table with " +
"column definitions, e.g., using the result of 'SHOW CREATE TABLE'";
- // Metrics collected when fetching incremental statistics from Catalogd. All metrics
- // are per query.
- private static final String STATS_FETCH_PREFIX = "StatsFetch";
- // Time (ms) needed to fetch all partitions stats from catalogd.
- private static final String STATS_FETCH_TIME = STATS_FETCH_PREFIX + ".Time";
- // Number of compressed bytes received for all partitions.
- private static final String STATS_FETCH_COMPRESSED_BYTES =
- STATS_FETCH_PREFIX + ".CompressedBytes";
- // Number of partitions sent from Catalogd.
- private static final String STATS_FETCH_TOTAL_PARTITIONS =
- STATS_FETCH_PREFIX + ".TotalPartitions";
- // Number of partitions sent from Catalogd that include statistics.
- private static final String STATS_FETCH_NUM_PARTITIONS_WITH_STATS =
- STATS_FETCH_PREFIX + ".NumPartitionsWithStats";
-
protected final TableName tableName_;
protected final TableSampleClause sampleParams_;
@@ -645,6 +627,8 @@ public class ComputeStatsStmt extends StatementBase {
* - incremental statistics are present
* - the partition is whitelisted in 'partitions'
* - the partition is present in the local impalad catalog
+ * TODO(vercegovac): Add metrics to track time spent for these rpc's when fetching
+ * from catalog. Look into adding to timeline.
* TODO(vercegovac): Look into parallelizing the fetch while child-queries are
* running. Easiest would be to move this fetch to the backend.
*/
@@ -654,10 +638,6 @@ public class ComputeStatsStmt extends StatementBase {
Preconditions.checkState(BackendConfig.INSTANCE.pullIncrementalStatistics()
&& !RuntimeEnv.INSTANCE.isTestEnv());
if (partitions.isEmpty()) return Collections.emptyMap();
- Stopwatch sw = new Stopwatch().start();
- int numCompressedBytes = 0;
- int totalPartitions = 0;
- int numPartitionsWithStats = 0;
try {
TGetPartitionStatsResponse response =
analyzer.getCatalog().getPartitionStats(table.getTableName());
@@ -677,19 +657,16 @@ public class ComputeStatsStmt extends StatementBase {
// local catalogs are returned.
Map<Long, TPartitionStats> partitionStats =
Maps.newHashMapWithExpectedSize(partitions.size());
- totalPartitions = partitions.size();
for (FeFsPartition part: partitions) {
ByteBuffer compressedStats = response.partition_stats.get(
FeCatalogUtils.getPartitionName(part));
if (compressedStats != null) {
byte[] compressedStatsBytes = new byte[compressedStats.remaining()];
- numCompressedBytes += compressedStatsBytes.length;
compressedStats.get(compressedStatsBytes);
TPartitionStats remoteStats =
PartitionStatsUtil.partStatsFromCompressedBytes(
compressedStatsBytes, part);
if (remoteStats != null && remoteStats.isSetIntermediate_col_stats()) {
- ++numPartitionsWithStats;
partitionStats.put(part.getId(), remoteStats);
}
}
@@ -698,26 +675,10 @@ public class ComputeStatsStmt extends StatementBase {
} catch (Exception e) {
Throwables.propagateIfInstanceOf(e, AnalysisException.class);
throw new AnalysisException("Error fetching partition statistics", e);
- } finally {
- recordFetchMetrics(numCompressedBytes, totalPartitions, numPartitionsWithStats, sw);
}
}
/**
- * Adds metrics to the frontend profile when fetching incremental stats from catalogd.
- */
- private static void recordFetchMetrics(int numCompressedBytes,
- int totalPartitions, int numPartitionsWithStats, Stopwatch stopwatch) {
- FrontendProfile profile = FrontendProfile.getCurrentOrNull();
- if (profile == null) return;
- profile.addToCounter(STATS_FETCH_COMPRESSED_BYTES, TUnit.BYTES, numCompressedBytes);
- profile.addToCounter(STATS_FETCH_TOTAL_PARTITIONS, TUnit.NONE, totalPartitions);
- profile.addToCounter(STATS_FETCH_NUM_PARTITIONS_WITH_STATS, TUnit.NONE,
- numPartitionsWithStats);
- profile.addToCounter(STATS_FETCH_TIME, TUnit.TIME_MS, stopwatch.elapsedMillis());
- }
-
- /**
* Analyzes the TABLESAMPLE clause and computes the files sample to set
* 'effectiveSamplePerc_'.
* Returns the TABLESAMPLE SQL to be used for all child queries or an empty string if
http://git-wip-us.apache.org/repos/asf/impala/blob/d918b2ae/tests/common/custom_cluster_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py
index fd2c69e..8fc24c2 100644
--- a/tests/common/custom_cluster_test_suite.py
+++ b/tests/common/custom_cluster_test_suite.py
@@ -187,7 +187,7 @@ class CustomClusterTestSuite(ImpalaTestSuite):
if pytest.config.option.pull_incremental_statistics:
cmd.append("--impalad_args=%s --catalogd_args=%s" %
- ("--pull_incremental_statistics", "--pull_incremental_statistics"))
+ ("--pull_incremental_statistcs", "--pull_incremental_statistics"))
default_query_option_kvs = []
# Put any defaults first, then any arguments after that so they can override defaults.
http://git-wip-us.apache.org/repos/asf/impala/blob/d918b2ae/tests/custom_cluster/test_pull_stats.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_pull_stats.py b/tests/custom_cluster/test_pull_stats.py
index b852f3d..e470ead 100644
--- a/tests/custom_cluster/test_pull_stats.py
+++ b/tests/custom_cluster/test_pull_stats.py
@@ -31,54 +31,3 @@ class TestPullStatistics(CustomClusterTestSuite):
catalogd_args="--pull_incremental_statistics=true")
def test_pull_stats(self, vector, unique_database):
self.run_test_case('QueryTest/compute-stats-incremental', vector, unique_database)
-
- @pytest.mark.execute_serially
- @CustomClusterTestSuite.with_args(impalad_args="--pull_incremental_statistics=true",
- catalogd_args="--pull_incremental_statistics=true")
- def test_pull_stats_profile(self, vector, unique_database):
- """Checks that the frontend profile includes metrics when computing
- incremental statistics.
- """
- try:
- client = self.cluster.impalads[0].service.create_beeswax_client()
- create = "create table test like functional.alltypes"
- load = "insert into test partition(year, month) select * from functional.alltypes"
- insert = """insert into test partition(year=2009, month=1) values
- (29349999, true, 4, 4, 4, 40,4.400000095367432,40.4,
- "10/21/09","4","2009-10-21 03:24:09.600000000")"""
- stats_all = "compute incremental stats test"
- stats_part = "compute incremental stats test partition (year=2009,month=1)"
-
- # Checks that profile does not have metrics for incremental stats when
- # the operation is not 'compute incremental stats'.
- self.execute_query_expect_success(client, "use %s" % unique_database)
- profile = self.execute_query_expect_success(client, create).runtime_profile
- assert profile.count("StatsFetch") == 0
- # Checks that incremental stats metrics are present when 'compute incremental
- # stats' is run. Since the table has no stats, expect that no bytes are fetched.
- self.execute_query_expect_success(client, load)
- profile = self.execute_query_expect_success(client, stats_all).runtime_profile
- assert profile.count("StatsFetch") > 1
- assert profile.count("StatsFetch.CompressedBytes: 0") == 1
- # Checks that bytes fetched is non-zero since incremental stats are present now
- # and should have been fetched.
- self.execute_query_expect_success(client, insert)
- profile = self.execute_query_expect_success(client, stats_part).runtime_profile
- assert profile.count("StatsFetch") > 1
- assert profile.count("StatsFetch.CompressedBytes") == 1
- assert profile.count("StatsFetch.CompressedBytes: 0") == 0
- # Adds a partition, computes stats, and checks that the metrics in the profile
- # reflect the operation.
- alter = "alter table test add partition(year=2011, month=1)"
- insert_new_partition = """
- insert into test partition(year=2011, month=1) values
- (29349999, true, 4, 4, 4, 40,4.400000095367432,40.4,
- "10/21/09","4","2009-10-21 03:24:09.600000000")
- """
- self.execute_query_expect_success(client, alter)
- self.execute_query_expect_success(client, insert_new_partition)
- profile = self.execute_query_expect_success(client, stats_all).runtime_profile
- assert profile.count("StatsFetch.TotalPartitions: 25") == 1
- assert profile.count("StatsFetch.NumPartitionsWithStats: 24") == 1
- finally:
- client.close()