You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by tm...@apache.org on 2018/07/20 21:01:34 UTC
[1/2] impala git commit: IMPALA-6677: [DOCS] Document the next_day
function
Repository: impala
Updated Branches:
refs/heads/master 649397e37 -> 79eef4924
IMPALA-6677: [DOCS] Document the next_day function
Change-Id: I2dacc86ff69a1016b1863d9db66dd29fd832b715
Reviewed-on: http://gerrit.cloudera.org:8080/10893
Reviewed-by: Vuk Ercegovac <ve...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/eaea7d28
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/eaea7d28
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/eaea7d28
Branch: refs/heads/master
Commit: eaea7d289aa8d9bd4944d9ae59639e965f19d76b
Parents: 649397e
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Mon Jul 9 11:48:04 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Fri Jul 20 02:19:34 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_datetime_functions.xml | 73 ++++++++++++++++++++------
1 file changed, 58 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/eaea7d28/docs/topics/impala_datetime_functions.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_datetime_functions.xml b/docs/topics/impala_datetime_functions.xml
index cb2f03f..7c8d5e1 100644
--- a/docs/topics/impala_datetime_functions.xml
+++ b/docs/topics/impala_datetime_functions.xml
@@ -1913,6 +1913,42 @@ select now() as right_now, nanoseconds_sub(now(), 1e9) as 1_second_earlier;
</dd>
</dlentry>
+ <dlentry id="next_day">
+ <dt>
+ <codeph>next_day(timestamp date, string weekday)</codeph>
+ </dt>
+ <dd>
+ <b>Purpose:</b> Returns the date of the <varname>weekday</varname>
+ that follows the specified <varname>date</varname>. <p>
+ <b>Return type:</b>
+ <codeph>timestamp</codeph>
+ </p>
+ <p conref="../shared/impala_common.xml#common/usage_notes_blurb"/>
+ <p> The <varname>weekday</varname> parameter is case-insensitive. The
+ following values are accepted for <varname>weekday</varname>:
+ <codeph>"Sunday"</codeph>/<codeph>"Sun"</codeph>,
+ <codeph>"Monday"</codeph>/<codeph>"Mon"</codeph>,
+ <codeph>"Tuesday"</codeph>/<codeph>"Tue"</codeph>,
+ <codeph>"Wednesday"</codeph>/<codeph>"Wed"</codeph>,
+ <codeph>"Thursday"</codeph>/<codeph>"Thu"</codeph>,
+ <codeph>"Friday"</codeph>/<codeph>"Fri"</codeph>,
+ <codeph>"Saturday"</codeph>/<codeph>"Sat"</codeph>
+ </p>
+ <p> Calling the function with the current date and weekday returns the
+ date that is one week later.</p><p
+ conref="../shared/impala_common.xml#common/example_blurb"/>
+ <p>
+ <codeblock>select next_day('2013-12-25','Saturday');
+-- Returns '2013-12-28 00:00:00' the first Saturday after December 25, 2013.
+select next_day(to_timestamp('08-1987-21', 'mm-yyyy-dd'), 'Friday');
+-- Returns '1987-08-28 00:00:00' the first Friday after August 28, 1987.
+
+select next_day(now(), 'Thu');
+-- Executed on 2018-07-12, the function returns '2018-07-13 00:00:00', one week
+-- after the current date.</codeblock>
+ </p>
+ </dd>
+ </dlentry>
<dlentry id="now">
@@ -1922,34 +1958,41 @@ select now() as right_now, nanoseconds_sub(now(), 1e9) as 1_second_earlier;
<dd>
<indexterm audience="hidden">now() function</indexterm>
- <b>Purpose:</b> Returns the current date and time (in the local time zone) as a
- <codeph>TIMESTAMP</codeph> value.
+ <b>Purpose:</b> Returns the current date and time (in the local time
+ zone) as a <codeph>TIMESTAMP</codeph> value.
+
<p>
<b>Return type:</b> <codeph>timestamp</codeph>
</p>
<p conref="../shared/impala_common.xml#common/usage_notes_blurb"/>
+
<p>
- To find a date/time value in the future or the past relative to the current date
- and time, add or subtract an <codeph>INTERVAL</codeph> expression to the return value of
- <codeph>now()</codeph>. See <xref href="impala_timestamp.xml#timestamp"/> for examples.
+ To find a date/time value in the future or the past relative to the
+ current date and time, add or subtract an <codeph>INTERVAL</codeph>
+ expression to the return value of <codeph>now()</codeph>. See <xref
+ href="impala_timestamp.xml#timestamp"/> for examples.
</p>
<p>
- To produce a <codeph>TIMESTAMP</codeph> representing the current date and time that can be
- shared or stored without interoperability problems due to time zone differences, use the
- <codeph>to_utc_timestamp()</codeph> function and specify the time zone of the server.
- When <codeph>TIMESTAMP</codeph> data is stored in UTC form, any application that queries
- those values can convert them to the appropriate local time zone by calling the inverse
+ To produce a <codeph>TIMESTAMP</codeph> representing the current
+ date and time that can be shared or stored without interoperability
+ problems due to time zone differences, use the
+ <codeph>to_utc_timestamp()</codeph> function and specify the time
+ zone of the server. When <codeph>TIMESTAMP</codeph> data is stored
+ in UTC form, any application that queries those values can convert
+ them to the appropriate local time zone by calling the inverse
function, <codeph>from_utc_timestamp()</codeph>.
</p>
+
<p conref="../shared/impala_common.xml#common/current_timezone_tip"/>
- <p>
- Any references to the <codeph>now()</codeph> function are evaluated at the start of a query.
- All calls to <codeph>now()</codeph> within the same query return the same value,
- and the value does not depend on how long the query takes.
+
+ <p> Any references to the <codeph>now()</codeph> function are
+ evaluated at the start of a query. All calls to
+ <codeph>now()</codeph> within the same query return the same
+ value, and the value does not depend on how long the query takes.
</p>
<p conref="../shared/impala_common.xml#common/example_blurb"/>
-<codeblock>
+ <codeblock>
select now() as 'Current time in California USA',
to_utc_timestamp(now(), 'PDT') as 'Current time in Greenwich UK';
+--------------------------------+-------------------------------+
[2/2] impala git commit: IMPALA-7256: Aggregator mem usage isn't
reflected in summary
Posted by tm...@apache.org.
IMPALA-7256: Aggregator mem usage isn't reflected in summary
This patch fixes a bug where memory used by an Aggregator wasn't being
reflected in the exec summary for the corresponding aggregation node
by ensuring that the Aggregator's MemTracker is a child of the node's
MemTracker.
Testing:
- Manually ran a query and checked that all memory used by the
Aggregator is accounted for in the exec summary.
Change-Id: Iba6ef207bed47810fc742aec3481db5f313cf97f
Reviewed-on: http://gerrit.cloudera.org:8080/10989
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/79eef492
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/79eef492
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/79eef492
Branch: refs/heads/master
Commit: 79eef49248ecc763d54b39cf7b04f1ac5ab46936
Parents: eaea7d2
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Thu Jul 19 02:14:15 2018 +0000
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Fri Jul 20 21:00:00 2018 +0000
----------------------------------------------------------------------
be/src/exec/aggregator.cc | 3 ++-
be/src/exec/aggregator.h | 1 +
be/src/exec/exec-node.cc | 3 ++-
be/src/exec/grouping-aggregator.cc | 11 ++++++++---
be/src/exec/grouping-aggregator.h | 1 +
be/src/runtime/reservation-manager.cc | 9 +++++----
be/src/runtime/reservation-manager.h | 5 ++++-
7 files changed, 23 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/79eef492/be/src/exec/aggregator.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/aggregator.cc b/be/src/exec/aggregator.cc
index 87abc52..9e79bf4 100644
--- a/be/src/exec/aggregator.cc
+++ b/be/src/exec/aggregator.cc
@@ -46,6 +46,7 @@ const char* Aggregator::LLVM_CLASS_NAME = "class.impala::Aggregator";
Aggregator::Aggregator(ExecNode* exec_node, ObjectPool* pool, const TPlanNode& tnode,
const DescriptorTbl& descs, const std::string& name)
: id_(exec_node->id()),
+ exec_node_(exec_node),
pool_(pool),
intermediate_tuple_id_(tnode.agg_node.intermediate_tuple_id),
intermediate_tuple_desc_(descs.GetTupleDescriptor(intermediate_tuple_id_)),
@@ -82,7 +83,7 @@ Status Aggregator::Init(const TPlanNode& tnode, RuntimeState* state) {
Status Aggregator::Prepare(RuntimeState* state) {
mem_tracker_.reset(new MemTracker(
- runtime_profile_, -1, runtime_profile_->name(), state->instance_mem_tracker()));
+ runtime_profile_, -1, runtime_profile_->name(), exec_node_->mem_tracker()));
expr_mem_tracker_.reset(new MemTracker(-1, "Exprs", mem_tracker_.get(), false));
expr_perm_pool_.reset(new MemPool(expr_mem_tracker_.get()));
expr_results_pool_.reset(new MemPool(expr_mem_tracker_.get()));
http://git-wip-us.apache.org/repos/asf/impala/blob/79eef492/be/src/exec/aggregator.h
----------------------------------------------------------------------
diff --git a/be/src/exec/aggregator.h b/be/src/exec/aggregator.h
index f415606..1ea3c47 100644
--- a/be/src/exec/aggregator.h
+++ b/be/src/exec/aggregator.h
@@ -93,6 +93,7 @@ class Aggregator {
protected:
/// The id of the ExecNode this Aggregator corresponds to.
int id_;
+ ExecNode* exec_node_;
ObjectPool* pool_;
/// Account for peak memory used by this aggregator.
http://git-wip-us.apache.org/repos/asf/impala/blob/79eef492/be/src/exec/exec-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc
index 766f421..bbb9a4b 100644
--- a/be/src/exec/exec-node.cc
+++ b/be/src/exec/exec-node.cc
@@ -126,7 +126,8 @@ Status ExecNode::Prepare(RuntimeState* state) {
}
reservation_manager_.Init(
Substitute("$0 id=$1 ptr=$2", PrintThriftEnum(type_), id_, this), runtime_profile_,
- mem_tracker_.get(), resource_profile_, debug_options_);
+ state->instance_buffer_reservation(), mem_tracker_.get(), resource_profile_,
+ debug_options_);
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/79eef492/be/src/exec/grouping-aggregator.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/grouping-aggregator.cc b/be/src/exec/grouping-aggregator.cc
index 4f3e5cf..092ecfd 100644
--- a/be/src/exec/grouping-aggregator.cc
+++ b/be/src/exec/grouping-aggregator.cc
@@ -175,9 +175,13 @@ Status GroupingAggregator::Prepare(RuntimeState* state) {
MAX_PARTITION_DEPTH, 1, expr_perm_pool_.get(), expr_results_pool_.get(),
expr_results_pool_.get(), &ht_ctx_));
- reservation_manager_.Init(
- Substitute("GroupingAggregator id=$0 ptr=$1", id_, this), runtime_profile_,
- mem_tracker_.get(), resource_profile_, debug_options_);
+ reservation_tracker_.reset(new ReservationTracker);
+ reservation_tracker_->InitChildTracker(runtime_profile_,
+ state->instance_buffer_reservation(), exec_node_->mem_tracker(),
+ numeric_limits<int64_t>::max());
+ reservation_manager_.Init(Substitute("GroupingAggregator id=$0 ptr=$1", id_, this),
+ runtime_profile_, reservation_tracker_.get(), mem_tracker_.get(),
+ resource_profile_, debug_options_);
return Status::OK();
}
@@ -400,6 +404,7 @@ void GroupingAggregator::Close(RuntimeState* state) {
ScalarExpr::Close(build_exprs_);
reservation_manager_.Close(state);
+ if (reservation_tracker_ != nullptr) reservation_tracker_->Close();
// Must be called after tuple_pool_ is freed, so that mem_tracker_ can be closed.
Aggregator::Close(state);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/79eef492/be/src/exec/grouping-aggregator.h
----------------------------------------------------------------------
diff --git a/be/src/exec/grouping-aggregator.h b/be/src/exec/grouping-aggregator.h
index b766a1e..3d64711 100644
--- a/be/src/exec/grouping-aggregator.h
+++ b/be/src/exec/grouping-aggregator.h
@@ -216,6 +216,7 @@ class GroupingAggregator : public Aggregator {
/// Resource information sent from the frontend.
const TBackendResourceProfile resource_profile_;
+ std::unique_ptr<ReservationTracker> reservation_tracker_;
ReservationManager reservation_manager_;
BufferPool::ClientHandle* buffer_pool_client();
http://git-wip-us.apache.org/repos/asf/impala/blob/79eef492/be/src/runtime/reservation-manager.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/reservation-manager.cc b/be/src/runtime/reservation-manager.cc
index f16b48f..1712e09 100644
--- a/be/src/runtime/reservation-manager.cc
+++ b/be/src/runtime/reservation-manager.cc
@@ -29,10 +29,11 @@ using strings::Substitute;
namespace impala {
void ReservationManager::Init(string name, RuntimeProfile* runtime_profile,
- MemTracker* mem_tracker, const TBackendResourceProfile& resource_profile,
- const TDebugOptions& debug_options) {
+ ReservationTracker* parent_reservation, MemTracker* mem_tracker,
+ const TBackendResourceProfile& resource_profile, const TDebugOptions& debug_options) {
name_ = name;
runtime_profile_ = runtime_profile;
+ parent_reservation_ = parent_reservation;
mem_tracker_ = mem_tracker;
resource_profile_ = resource_profile;
debug_options_ = debug_options;
@@ -60,8 +61,8 @@ Status ReservationManager::ClaimBufferReservation(RuntimeState* state) {
}
RETURN_IF_ERROR(buffer_pool->RegisterClient(name_, state->query_state()->file_group(),
- state->instance_buffer_reservation(), mem_tracker_,
- resource_profile_.max_reservation, runtime_profile_, &buffer_pool_client_));
+ parent_reservation_, mem_tracker_, resource_profile_.max_reservation,
+ runtime_profile_, &buffer_pool_client_));
VLOG_FILE << name_ << " claiming reservation " << resource_profile_.min_reservation;
state->query_state()->initial_reservations()->Claim(
&buffer_pool_client_, resource_profile_.min_reservation);
http://git-wip-us.apache.org/repos/asf/impala/blob/79eef492/be/src/runtime/reservation-manager.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/reservation-manager.h b/be/src/runtime/reservation-manager.h
index 0545548..10563df 100644
--- a/be/src/runtime/reservation-manager.h
+++ b/be/src/runtime/reservation-manager.h
@@ -40,7 +40,8 @@ class ReservationManager {
/// and error messages, 'debug_options' is used for the SET_DENY_RESERVATION_PROBABILITY
/// action. Does not take ownership of 'runtime_profile' and 'mem_tracker'. Must be
/// called before ClaimBufferReservation().
- void Init(std::string name, RuntimeProfile* runtime_profile, MemTracker* mem_tracker,
+ void Init(std::string name, RuntimeProfile* runtime_profile,
+ ReservationTracker* parent_reservation, MemTracker* mem_tracker,
const TBackendResourceProfile& resource_profile,
const TDebugOptions& debug_options);
void Close(RuntimeState* state);
@@ -72,6 +73,8 @@ class ReservationManager {
MemTracker* mem_tracker_;
+ ReservationTracker* parent_reservation_;
+
/// Buffer pool client for this node. Initialized with the node's minimum reservation
/// in ClaimBufferReservation(). After initialization, the client must hold onto at
/// least the minimum reservation so that it can be returned to the initial