You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by yj...@apache.org on 2016/04/05 04:03:28 UTC
incubator-hawq git commit: HAWQ-623. Resource quota request does not
follow latest resource quota calculating logic
Repository: incubator-hawq
Updated Branches:
refs/heads/master 3828d9147 -> cf2744dc5
HAWQ-623. Resource quota request does not follow latest resource quota calculating logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cf2744dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cf2744dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cf2744dc
Branch: refs/heads/master
Commit: cf2744dc5fdefb47a4fbcd6f8fa807861822ebbd
Parents: 3828d91
Author: YI JIN <yj...@pivotal.io>
Authored: Tue Apr 5 12:03:19 2016 +1000
Committer: YI JIN <yj...@pivotal.io>
Committed: Tue Apr 5 12:03:19 2016 +1000
----------------------------------------------------------------------
src/backend/resourcemanager/requesthandler.c | 73 +++++++++++++++++++---
src/backend/resourcemanager/resqueuemanager.c | 40 +-----------
2 files changed, 69 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf2744dc/src/backend/resourcemanager/requesthandler.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c
index 4e24848..b89d23a 100644
--- a/src/backend/resourcemanager/requesthandler.c
+++ b/src/backend/resourcemanager/requesthandler.c
@@ -793,18 +793,52 @@ bool handleRMRequestAcquireResourceQuota(void **arg)
int res = FUNC_RETURN_OK;
ConnectionTrack conntrack = (ConnectionTrack)(*arg);
bool exist = false;
+ uint64_t reqtime = gettime_microsec();
+ /* If we run in YARN mode, we expect that we should try to get at least one
+ * available segment, and this requires at least once global resource manager
+ * cluster report returned.
+ */
+ if ( reqtime - DRMGlobalInstance->ResourceManagerStartTime <=
+ rm_nocluster_timeout * 1000000LL &&
+ PRESPOOL->RBClusterReportCounter == 0 )
+ {
+ elog(DEBUG3, "Resource manager defers the resource request.");
+ return false;
+ }
+
+ /*
+ * If resource queue has no concrete capacity set yet, no need to handle
+ * the request.
+ */
+ if ( PQUEMGR->RootTrack->QueueInfo->ClusterMemoryMB <= 0 )
+ {
+ elog(DEBUG3, "Resource manager defers the resource request because the "
+ "resource queues have no valid resource capacities yet.");
+ return false;
+ }
+
+ Assert(PRESPOOL->SlavesHostCount > 0);
+ int rejectlimit = ceil(PRESPOOL->SlavesHostCount * rm_rejectrequest_nseg_limit);
+ int unavailcount = PRESPOOL->SlavesHostCount - PRESPOOL->AvailNodeCount;
+ if ( unavailcount > rejectlimit )
+ {
+ snprintf(errorbuf, sizeof(errorbuf),
+ "%d of %d segments %s unavailable, exceeds %.1f%% defined in "
+ "GUC hawq_rm_rejectrequest_nseg_limit. The resource quota "
+ "request is rejected.",
+ unavailcount,
+ PRESPOOL->SlavesHostCount,
+ unavailcount == 1 ? "is" : "are",
+ rm_rejectrequest_nseg_limit*100.0);
+ elog(WARNING, "ConnID %d. %s", conntrack->ConnID, errorbuf);
+ res = RESOURCEPOOL_TOO_MANY_UAVAILABLE_HOST;
+ goto errorexit;
+ }
RPCRequestHeadAcquireResourceQuotaFromRMByOID request =
SMBUFF_HEAD(RPCRequestHeadAcquireResourceQuotaFromRMByOID,
&(conntrack->MessageBuff));
- elog(LOG, "ConnID %d. User "INT64_FORMAT" acquires query resource quota "
- "with expected %d vseg (MIN %d).",
- conntrack->ConnID,
- request->UseridOid,
- request->MaxSegCountFix,
- request->MinSegCountFix);
-
/* Get user name from oid. */
UserInfo reguser = getUserByUserOID(request->UseridOid, &exist);
if ( !exist )
@@ -826,6 +860,31 @@ bool handleRMRequestAcquireResourceQuota(void **arg)
conntrack->MinSegCountFixed = request->MinSegCountFix;
conntrack->VSegLimitPerSeg = request->VSegLimitPerSeg;
conntrack->VSegLimit = request->VSegLimit;
+ conntrack->StatVSegMemoryMB = request->StatVSegMemoryMB;
+ conntrack->StatNVSeg = request->StatNVSeg;
+
+ elog(RMLOG, "ConnID %d. User "INT64_FORMAT" acquires query resource quota. "
+ "Expect %d vseg (MIN %d). "
+ "Each segment has maximum %d vseg. "
+ "Query has maximum %d vseg. "
+ "Statement quota %d MB x %d vseg",
+ conntrack->ConnID,
+ request->UseridOid,
+ request->MaxSegCountFix,
+ request->MinSegCountFix,
+ request->VSegLimitPerSeg,
+ request->VSegLimit,
+ request->StatVSegMemoryMB,
+ request->StatNVSeg);
+
+ if ( conntrack->StatNVSeg > 0 )
+ {
+ elog(LOG, "ConnID %d. Statement level resource quota is active. "
+ "Expect resource ( %d MB ) x %d.",
+ conntrack->ConnID,
+ conntrack->StatVSegMemoryMB,
+ conntrack->StatNVSeg);
+ }
res = acquireResourceQuotaFromResQueMgr(conntrack, errorbuf, sizeof(errorbuf));
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf2744dc/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index eec2fb4..5d43f92 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -1958,21 +1958,10 @@ int acquireResourceQuotaFromResQueMgr(ConnectionTrack conntrack,
userinfo = getUserByUserName(conntrack->UserID,
strlen(conntrack->UserID),
&exist);
- if ( exist )
- {
- /* Get the queue, and check if the parallel limit is achieved. */
- queuetrack = getQueueTrackByQueueOID(userinfo->QueueOID, &exist);
- }
- else
- {
- elog(LOG, "ConnID %d. No user %s defined for registering connection. "
- "Assign to pg_default queue.",
- conntrack->ConnID,
- conntrack->UserID);
- queuetrack = PQUEMGR->DefaultTrack;
- userinfo = NULL;
- }
+ Assert(exist && userinfo != NULL);
+ /* Get the queue, and check if the parallel limit is achieved. */
+ queuetrack = getQueueTrackByQueueOID(userinfo->QueueOID, &exist);
Assert( queuetrack != NULL );
conntrack->QueueTrack = queuetrack;
@@ -1986,29 +1975,6 @@ int acquireResourceQuotaFromResQueMgr(ConnectionTrack conntrack,
if ( conntrack->StatNVSeg == 0 )
{
/*------------------------------------------------------------------
- * The following logic consider the actual resource requirement from
- * dispatcher based on table size, workload, etc. The requirement is
- * described by (MinSegCountFixed, MaxSegCountFixed). The requirement
- * can be satisfied only when there is a non-empty intersect between
- * (MinSegCountFixed, MaxSegCountFixed) and (SegNumMin, SegNum).
- *------------------------------------------------------------------
- */
- conntrack->SegNumMin = conntrack->MaxSegCountFixed < conntrack->SegNumMin ?
- conntrack->MinSegCountFixed :
- max(conntrack->SegNumMin, conntrack->MinSegCountFixed);
-
- conntrack->SegNum = min(conntrack->SegNum, conntrack->MaxSegCountFixed);
-
- Assert( conntrack->SegNumMin <= conntrack->SegNum );
- elog(LOG, "ConnID %d. Query resource quota expects (%d MB, %lf CORE) x %d "
- "( MIN %d ) resource after adjusting based on query characters.",
- conntrack->ConnID,
- conntrack->SegMemoryMB,
- conntrack->SegCore,
- conntrack->SegNum,
- conntrack->SegNumMin);
-
- /*------------------------------------------------------------------
* Adjust the number of virtual segments again based on
* NVSEG_*_LIMITs and NVSEG_*_LIMIT_PERSEGs. This adjustment must
* succeed.