You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by yj...@apache.org on 2016/04/09 04:36:29 UTC
incubator-hawq git commit: HAWQ-645. shrink LOG level log output from
resource manager
Repository: incubator-hawq
Updated Branches:
refs/heads/master 143c0e3f1 -> 37839128b
HAWQ-645. shrink LOG level log output from resource manager
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/37839128
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/37839128
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/37839128
Branch: refs/heads/master
Commit: 37839128bfe8b063918300a929825485c1d0b3af
Parents: 143c0e3
Author: YI JIN <yj...@pivotal.io>
Authored: Sat Apr 9 12:36:12 2016 +1000
Committer: YI JIN <yj...@pivotal.io>
Committed: Sat Apr 9 12:36:12 2016 +1000
----------------------------------------------------------------------
.../communication/rmcomm_Message.c | 10 +-
.../communication/rmcomm_QD2RM.c | 20 +--
src/backend/resourcemanager/requesthandler.c | 4 +-
.../resourcebroker/resourcebroker_LIBYARN.c | 67 ++++++---
.../resourcebroker_LIBYARN_proc.c | 149 ++++++++++---------
src/backend/resourcemanager/resourcemanager.c | 22 +--
src/backend/resourcemanager/resourcepool.c | 66 ++++----
src/backend/resourcemanager/resqueuemanager.c | 62 ++++----
src/backend/tcop/pquery.c | 2 -
9 files changed, 224 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/communication/rmcomm_Message.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_Message.c b/src/backend/resourcemanager/communication/rmcomm_Message.c
index 364c8a2..b12a734 100644
--- a/src/backend/resourcemanager/communication/rmcomm_Message.c
+++ b/src/backend/resourcemanager/communication/rmcomm_Message.c
@@ -96,13 +96,9 @@ void ReadPostHandler_Message(AsyncCommBuffer buffer)
if ( DRM_MSGFRAME_TAILTAG_MATCHED(p) )
{
/* Skip heart-beat message log. */
- elog((header->MessageID == REQUEST_QD_REFRESH_RESOURCE ||
- header->MessageID == REQUEST_RM_IMALIVE ||
- header->MessageID == RESPONSE_QD_REFRESH_RESOURCE ||
- header->MessageID == RESPONSE_RM_IMALIVE) ? DEBUG3 : LOG,
- "AsyncComm framework receives message %d from FD %d",
- header->MessageID,
- buffer->FD);
+ elog(RMLOG, "AsyncComm framework receives message %d from FD %d",
+ header->MessageID,
+ buffer->FD);
/* Get complete message and call the handler. */
if ( context->MessageRecvedHandler != NULL )
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
index 4808a8b..f5cd379 100644
--- a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
+++ b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
@@ -698,16 +698,16 @@ int acquireResourceFromRM(int index,
0 :
preferred_nodes_size;
- elog(LOG, "ConnID: %d. Acquire resource request for index %d. "
- "Max vseg size %d Min vseg size %d Estimated slice size %d "
- "estimated IO bytes size " INT64_FORMAT " Preferred node count %d.",
- curcontext->QD_Conn_ID,
- index,
- max_seg_count_fix,
- min_seg_count_fix,
- slice_size,
- iobytes,
- nodecount);
+ elog(RMLOG, "ConnID: %d. Acquire resource request for index %d. "
+ "Max vseg size %d Min vseg size %d Estimated slice size %d "
+ "estimated IO bytes size " INT64_FORMAT " Preferred node count %d.",
+ curcontext->QD_Conn_ID,
+ index,
+ max_seg_count_fix,
+ min_seg_count_fix,
+ slice_size,
+ iobytes,
+ nodecount);
/* Build request. */
resetSelfMaintainBuffer(sendbuffer);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/requesthandler.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c
index daef075..eb9fc1d 100644
--- a/src/backend/resourcemanager/requesthandler.c
+++ b/src/backend/resourcemanager/requesthandler.c
@@ -178,8 +178,8 @@ bool handleRMRequestConnectionRegByOID(void **arg)
if ( res == FUNC_RETURN_OK )
{
trackConnectionTrack(conntrack);
- elog(LOG, "ConnID %d. Resource manager tracked connection.",
- conntrack->ConnID);
+ elog(RMLOG, "ConnID %d. Resource manager tracked connection.",
+ conntrack->ConnID);
response.Result = FUNC_RETURN_OK;
response.ConnID = conntrack->ConnID;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index b8d205e..37e3bca 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -454,8 +454,8 @@ int RB_LIBYARN_getContainerReport(List **ctnstatl)
res = RESBROK_PIPE_ERROR;
}
destroySelfMaintainBuffer(&sendBuffer);
- elog(LOG, "YARN mode resource broker wrote get container status request to "
- "resource broker process.");
+ elog(RMLOG, "YARN mode resource broker wrote get container status request to "
+ "resource broker process.");
PipeReceivePending = res == FUNC_RETURN_OK;
return res;
}
@@ -509,7 +509,7 @@ int RB_LIBYARN_handleNotification(void)
}
PipeReceivePending = false;
- elog(LOG, "Finish processing message %d", messageid);
+ elog(RMLOG, "Finish processing message %d", messageid);
if ( res != FUNC_RETURN_OK )
{
@@ -568,9 +568,11 @@ int handleRB2RM_ClusterReport(void)
}
elog(LOG, "YARN mode resource broker got cluster report having %d host(s), "
- "maximum queue capacity is %lf.",
+ "queue capacity %lf (MAX %lf) current capacity %lf.",
response.MachineCount,
- response.QueueMaxCapacity);
+ response.QueueCapacity,
+ response.QueueMaxCapacity,
+ response.QueueCurCapacity);
/* Read machines. */
for( int i = 0 ; i < response.MachineCount ; ++i )
@@ -659,12 +661,20 @@ int handleRB2RM_ClusterReport(void)
setAllSegResourceGRMUnhandled();
+
+ ResourceBundleData oldGRMTotalAll;
+ ResourceBundleData oldGRMTotal;
+ resetResourceBundleDataByBundle(&oldGRMTotalAll,
+ &(PRESPOOL->GRMTotalHavingNoHAWQNode));
+ resetResourceBundleDataByBundle(&oldGRMTotal,
+ &(PRESPOOL->GRMTotal));
/*
* Start to update resource pool content. The YARN cluster total size is
* also counted the same time.
*/
-
resetResourceBundleData(&(PRESPOOL->GRMTotalHavingNoHAWQNode), 0, 0.0, 0);
+ int updatedcnt = 0;
+ int skipedcnt = 0;
MEMORY_CONTEXT_SWITCH_TO(PCONTEXT)
while( list_length(segstats) > 0 )
@@ -681,20 +691,28 @@ int handleRB2RM_ClusterReport(void)
SelfMaintainBufferData buffer;
initializeSelfMaintainBuffer(&buffer, PCONTEXT);
generateSegStatReport(segstat, &buffer);
- elog(LOG, "YARN mode resource broker updated segment configuration "
- "in resource pool. %s", buffer.Buffer);
+ elog(RMLOG, "YARN mode resource broker updated segment configuration "
+ "in resource pool. %s", buffer.Buffer);
destroySelfMaintainBuffer(&buffer);
+ updatedcnt++;
}
else
{
- elog(LOG, "YARN mode resource broker skipped segment configuration "
- "from host %s",
- GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)));
+ elog(WARNING, "YARN mode resource broker skipped segment configuration "
+ "from host %s",
+ GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)));
+ skipedcnt++;
}
rm_pfree(PCONTEXT, segstat);
segstats = list_delete_first(segstats);
}
+ elog(LOG, "YARN mode resource broker updated %d %s, skiped %d %s",
+ updatedcnt,
+ updatedcnt <= 1 ? "segment" : "segments",
+ skipedcnt,
+ skipedcnt <= 1 ? "segment" : "segments");
+
/*
* iterate all segments without GRM report,
* and update its status.
@@ -754,13 +772,26 @@ int handleRB2RM_ClusterReport(void)
MEMORY_CONTEXT_SWITCH_BACK
- elog(LOG, "Resource manager YARN resource broker counted HAWQ cluster now "
- "having (%d MB, %lf CORE) in a YARN cluster of total resource "
- "(%d MB, %lf CORE).",
- PRESPOOL->GRMTotal.MemoryMB,
- PRESPOOL->GRMTotal.Core,
- PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB,
- PRESPOOL->GRMTotalHavingNoHAWQNode.Core);
+ if ( oldGRMTotalAll.MemoryMB != PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB ||
+ oldGRMTotalAll.Core != PRESPOOL->GRMTotalHavingNoHAWQNode.Core )
+ {
+ elog(LOG, "Resource manager YARN resource broker counted YARN cluster "
+ "having total resource (%d MB, %lf CORE).",
+ PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB,
+ PRESPOOL->GRMTotalHavingNoHAWQNode.Core);
+ }
+
+ if ( oldGRMTotal.MemoryMB != PRESPOOL->GRMTotal.MemoryMB ||
+ oldGRMTotal.Core != PRESPOOL->GRMTotal.Core )
+ {
+ elog(LOG, "Resource manager YARN resource broker counted HAWQ cluster now "
+ "having (%d MB, %lf CORE) in a YARN cluster of total resource "
+ "(%d MB, %lf CORE).",
+ PRESPOOL->GRMTotal.MemoryMB,
+ PRESPOOL->GRMTotal.Core,
+ PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB,
+ PRESPOOL->GRMTotalHavingNoHAWQNode.Core);
+ }
/*
* If the segment is GRM unavailable or FTS unavailable,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
index ad49aae..5a6e6a2 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
@@ -189,8 +189,8 @@ int ResBrokerMainInternal(void)
* run anymore. The process goes to the exit phase directly.
*/
if ( getppid() != ResBrokerParentPID ) {
- elog(LOG, "Parent process of YARN mode resource broker quited. "
- "Resource broker process will actively close.");
+ elog(WARNING, "Parent process of YARN mode resource broker quit. "
+ "Resource broker process will actively close.");
ResBrokerKeepRun = false;
continue;
}
@@ -270,10 +270,10 @@ int ResBrokerMainInternal(void)
}
if ( res != FUNC_RETURN_OK ) {
- elog(LOG, "YARN mode resource broker failed to process request. "
- "Message id = %d, result = %d.",
- messageid,
- res);
+ elog(WARNING, "YARN mode resource broker failed to process request. "
+ "Message id = %d, result = %d.",
+ messageid,
+ res);
/* If this is a pipe error between RM and RB or YARN remove error.
* Exit RB and let RM restart RB process. */
if ( res == RESBROK_ERROR_GRM )
@@ -1035,9 +1035,9 @@ int handleRM2RB_ReturnResource(void)
for ( int i = 0 ; i < request.ContainerCount ; ++i )
{
- elog(LOG, "YARN mode resource broker tries to return container of id "
- INT64_FORMAT,
- containerids[i]);
+ elog(RMLOG, "YARN mode resource broker tries to return container of id "
+ INT64_FORMAT,
+ containerids[i]);
}
if ( YARNJobID == NULL )
@@ -1169,9 +1169,9 @@ int handleRM2RB_GetContainerReport(void)
for( int i = 0 ; i < size ; ++i )
{
- elog(LOG, "Container report ID:"INT64_FORMAT", isActive:%d",
- ctnstats[i].ContainerID,
- ctnstats[i].isActive);
+ elog(RMLOG, "Container report ID:"INT64_FORMAT", isActive:%d",
+ ctnstats[i].ContainerID,
+ ctnstats[i].isActive);
}
/* Build response message. */
@@ -1396,9 +1396,9 @@ int RB2YARN_getClusterReport(DQueue hosts)
&nodeReportArraySize);
if ( yarnres != FUNCTION_SUCCEEDED )
{
- elog(LOG, "YARN mode resource broker failed to get cluster information "
- "from YARN. %s",
- getErrorMessage());
+ elog(WARNING, "YARN mode resource broker failed to get cluster "
+ "information from YARN. %s",
+ getErrorMessage());
}
else
{
@@ -1525,37 +1525,37 @@ int RB2YARN_getClusterReport(DQueue hosts)
insertDQueueTailNode(hosts, segstat);
- elog(LOG, "YARN mode resource broker got YARN cluster host \n"
- "\thost:%s\n"
- "\tport:%d\n"
- "\thttpAddress:%s\n"
- "\trackName:%s\n"
- "\tmemoryUsed:%d\n"
- "\tvcoresUsed:%d\n"
- "\tmemoryCapability:%d\n"
- "\tvcoresCapability:%d\n"
- "\tnumContainers:%d\n"
- "\tnodeState:%d\n"
- "\thealthReport:%s\n"
- "\tlastHealthReportTime:"INT64_FORMAT"\n"
- "\tmachineidsize:%d",
- pnodereport->host,
- pnodereport->port,
- pnodereport->httpAddress,
- pnodereport->rackName,
- pnodereport->memoryUsed,
- pnodereport->vcoresUsed,
- pnodereport->memoryCapability,
- pnodereport->vcoresCapability,
- pnodereport->numContainers,
- pnodereport->nodeState,
- pnodereport->healthReport,
- pnodereport->lastHealthReportTime,
- segstat->Info.Size);
-
- elog(LOG, "YARN mode reosurce broker built cluster segment %s at rack %s",
- GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)),
- GET_SEGINFO_GRMRACKNAME(&(segstat->Info)));
+ elog(RMLOG, "YARN mode resource broker got YARN cluster host \n"
+ "\thost:%s\n"
+ "\tport:%d\n"
+ "\thttpAddress:%s\n"
+ "\trackName:%s\n"
+ "\tmemoryUsed:%d\n"
+ "\tvcoresUsed:%d\n"
+ "\tmemoryCapability:%d\n"
+ "\tvcoresCapability:%d\n"
+ "\tnumContainers:%d\n"
+ "\tnodeState:%d\n"
+ "\thealthReport:%s\n"
+ "\tlastHealthReportTime:"INT64_FORMAT"\n"
+ "\tmachineidsize:%d",
+ pnodereport->host,
+ pnodereport->port,
+ pnodereport->httpAddress,
+ pnodereport->rackName,
+ pnodereport->memoryUsed,
+ pnodereport->vcoresUsed,
+ pnodereport->memoryCapability,
+ pnodereport->vcoresCapability,
+ pnodereport->numContainers,
+ pnodereport->nodeState,
+ pnodereport->healthReport,
+ pnodereport->lastHealthReportTime,
+ segstat->Info.Size);
+
+ elog(RMLOG, "YARN mode reosurce broker built cluster segment %s at rack %s",
+ GET_SEGINFO_GRMHOSTNAME(&(segstat->Info)),
+ GET_SEGINFO_GRMRACKNAME(&(segstat->Info)));
}
freeMemNodeReportArray(nodeReportArray, nodeReportArraySize);
}
@@ -1592,9 +1592,9 @@ int RB2YARN_acquireResource(uint32_t memorymb,
&allocatedResourcesArray,
&allocatedResourcesArraySize);
if( yarnres != FUNCTION_SUCCEEDED ) {
- elog(LOG, "YARN mode resource broker failed to allocate "
- "containers from YARN. %s",
- getErrorMessage());
+ elog(WARNING, "YARN mode resource broker failed to allocate "
+ "containers from YARN. %s",
+ getErrorMessage());
return yarnres;
}
else if ( allocatedResourcesArraySize == 0 ) {
@@ -1640,17 +1640,18 @@ int RB2YARN_acquireResource(uint32_t memorymb,
&activeFailIds,
&activeFailSize);
if( yarnres != FUNCTION_SUCCEEDED ) {
- elog(LOG, "YARN mode resource broker failed to get active-fail "
- "containers. %s",
- getErrorMessage());
+ elog(WARNING, "YARN mode resource broker failed to get active-fail "
+ "containers. %s",
+ getErrorMessage());
goto exit;
}
/* Build temporary failed container ids in hash table for fast retrieving.*/
if ( activeFailSize > 0 ) {
for (int i = 0 ; i < activeFailSize ; ++i) {
- elog(LOG, "YARN mode resource broker failed to activate container "INT64_FORMAT,
- activeFailIds[i]);
+ elog(WARNING, "YARN mode resource broker failed to activate "
+ "container "INT64_FORMAT,
+ activeFailIds[i]);
SimpArray key;
setSimpleArrayRef(&key, (void *)&(activeFailIds[i]), sizeof(int64_t));
@@ -1662,7 +1663,7 @@ int RB2YARN_acquireResource(uint32_t memorymb,
activeFailIds,
activeFailSize);
if ( yarnres != FUNCTION_SUCCEEDED ) {
- elog(LOG, "YARN mode resource broker failed to return active-fail "
+ elog(WARNING, "YARN mode resource broker failed to return active-fail "
"containers. %s",
getErrorMessage());
}
@@ -1681,12 +1682,12 @@ int RB2YARN_acquireResource(uint32_t memorymb,
strcpy(hostnamestr, allocatedResourcesArray[i].host);
insertDQueueTailNode(containerhosts, hostnamestr);
- elog(LOG, "YARN mode resource broker allocated and activated container. "
- "ID : "INT64_FORMAT"(%d MB, %d CORE) at %s.",
- allocatedResourcesArray[i].containerId,
- allocatedResourcesArray[i].memory,
- allocatedResourcesArray[i].vCores,
- allocatedResourcesArray[i].host);
+ elog(RMLOG, "YARN mode resource broker allocated and activated container. "
+ "ID : "INT64_FORMAT"(%d MB, %d CORE) at %s.",
+ allocatedResourcesArray[i].containerId,
+ allocatedResourcesArray[i].memory,
+ allocatedResourcesArray[i].vCores,
+ allocatedResourcesArray[i].host);
}
exit:
@@ -1716,8 +1717,8 @@ int RB2YARN_returnResource(int64_t *contids, int contcount)
contids,
contcount);
if ( yarnres != FUNCTION_SUCCEEDED ) {
- elog(LOG, "YARN mode resource broker failed to return containers. %s",
- getErrorMessage());
+ elog(WARNING, "YARN mode resource broker failed to return containers. %s",
+ getErrorMessage());
}
for ( int i = 0 ; i < contcount ; ++i ) {
@@ -1744,8 +1745,9 @@ int RB2YARN_getContainerReport(RB_GRMContainerStat *ctnstats, int *size)
yarnres = getContainerReports(LIBYARNClient, YARNJobID, &ctnrparr, &arrsize);
if ( yarnres != FUNCTION_SUCCEEDED )
{
- elog(LOG, "YARN mode resource broker failed to get container report. %s",
- getErrorMessage());
+ elog(WARNING, "YARN mode resource broker failed to get container "
+ "report. %s",
+ getErrorMessage());
}
else if ( arrsize > 0 )
{
@@ -1769,8 +1771,9 @@ int RB2YARN_getContainerReport(RB_GRMContainerStat *ctnstats, int *size)
&ctnstatsize);
if ( yarnres != FUNCTION_SUCCEEDED )
{
- elog(LOG, "YARN mode resource broker failed to get container status. %s",
- getErrorMessage());
+ elog(WARNING, "YARN mode resource broker failed to get container "
+ "status. %s",
+ getErrorMessage());
}
rm_pfree(PCONTEXT, ctnidarr);
@@ -1834,10 +1837,10 @@ int RB2YARN_getQueueReport(char *queuename,
Assert( queuename != NULL );
yarnres = getQueueInfo(LIBYARNClient, queuename, true, true, true, &queueInfo);
if ( yarnres != FUNCTION_SUCCEEDED ) {
- elog(LOG, "YARN mode resource broker failed to get YARN queue report of"
- " queue %s. %s",
- queuename,
- RB2YARN_getErrorMessage());
+ elog(WARNING, "YARN mode resource broker failed to get YARN queue report "
+ "of queue %s. %s",
+ queuename,
+ RB2YARN_getErrorMessage());
return yarnres;
}
@@ -1857,8 +1860,8 @@ int RB2YARN_finishYARNApplication(void)
int yarnres = FUNCTION_SUCCEEDED;
yarnres = finishJob(LIBYARNClient, YARNJobID, APPLICATION_SUCCEEDED);
if (yarnres != FUNCTION_SUCCEEDED) {
- elog(LOG, "YARN mode resource broker failed to finish job in YARN. %s",
- getErrorMessage());
+ elog(WARNING, "YARN mode resource broker failed to finish job in YARN. %s",
+ getErrorMessage());
}
else {
elog(LOG, "YARN mode resource broker finished job in YARN.");
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index f460ba5..ccbf460 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -1259,10 +1259,10 @@ int loadUserPropertiesFromCatalog(List **users)
foreach(cell, *users)
{
KVProperty property = lfirst(cell);
- elog(LOG, "Resource manager loaded role specifications from pg_authid : "
- "[%s]=[%s]",
- property->Key.Str,
- property->Val.Str);
+ elog(RMLOG, "Resource manager loaded role specifications from pg_authid : "
+ "[%s]=[%s]",
+ property->Key.Str,
+ property->Val.Str);
}
elog(LOG, "Resource manager successfully loaded role specifications.");
@@ -1569,10 +1569,10 @@ int loadQueuePropertiesFromCatalog(List **queues)
foreach(cell, *queues)
{
KVProperty property = lfirst(cell);
- elog(LOG, "Resource manger loaded resource queue specifications from "
- "pg_resqueue : [%s]=[%s]",
- property->Key.Str,
- property->Val.Str);
+ elog(RMLOG, "Resource manger loaded resource queue specifications from "
+ "pg_resqueue : [%s]=[%s]",
+ property->Key.Str,
+ property->Val.Str);
}
elog(LOG, "Resource manger successfully loaded resource queue specifications");
@@ -2947,9 +2947,9 @@ bool cleanedAllGRMContainers(void)
if ( PRESPOOL->AddPendingContainerCount > 0 ||
PRESPOOL->RetPendingContainerCount > 0 )
{
- elog(DEBUG3, "Pending GRM container count inc %d, dec %d.",
- PRESPOOL->AddPendingContainerCount,
- PRESPOOL->RetPendingContainerCount);
+ elog(RMLOG, "Pending GRM container count inc %d, dec %d.",
+ PRESPOOL->AddPendingContainerCount,
+ PRESPOOL->RetPendingContainerCount);
}
/* Condition 2. No on-the-fly GRM containers for increasing and decreasing.*/
return PRESPOOL->AddPendingContainerCount == 0 &&
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index 9941226..1e744f4 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -987,7 +987,11 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
Assert(false);
}
- elog(LOG, "Resource manager tracked segment %d of host %s.", segid, hostname);
+ SelfMaintainBufferData logcontent;
+ initializeSelfMaintainBuffer(&logcontent, PCONTEXT);
+ appendSelfMaintainBuffer(&logcontent, "host ", sizeof("host ")-1);
+ appendSelfMaintainBuffer(&logcontent, hostname, strlen(hostname));
+ appendSelfMaintainBuffer(&logcontent, " ip address ", sizeof(" ip address"));
/* Index all HAWQ node's ip addresses.
*
@@ -1012,11 +1016,23 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
TYPCONVERT(void *, segid),
false /* There should be no old value. */);
+ appendSelfMaintainBuffer(&logcontent, ",", sizeof(",")-1);
+
+ appendSelfMaintainBuffer(&logcontent,
+ hostaddrkey.Array,
+ hostaddrkey.Len);
+
elog(LOG, "Resource manager tracked ip address '%.*s' for host '%s'",
hostaddrkey.Len, hostaddrkey.Array,
hostname);
}
}
+ appendSMBStr(&logcontent, "");
+
+ elog(LOG, "Resource manager tracked segment %d of %s.",
+ segid,
+ SMBUFF_CONTENT(&logcontent));
+ destroySelfMaintainBuffer(&logcontent);
/*
* If in GRM mode, the new registered segment is marked
@@ -1541,11 +1557,11 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
segres->Stat = newSegStat;
- elog(LOG, "Resource manager update segment info, hostname:%s, "
- "with GRM hostname:%s, GRM rackname:%s",
- GET_SEGINFO_HOSTNAME(&(newSegStat->Info)),
- GET_SEGINFO_GRMHOSTNAME(&(newSegStat->Info)),
- GET_SEGINFO_GRMRACKNAME(&(newSegStat->Info)));
+ elog(RMLOG, "Resource manager update segment info, hostname:%s, "
+ "with GRM hostname:%s, GRM rackname:%s",
+ GET_SEGINFO_HOSTNAME(&(newSegStat->Info)),
+ GET_SEGINFO_GRMHOSTNAME(&(newSegStat->Info)),
+ GET_SEGINFO_GRMRACKNAME(&(newSegStat->Info)));
elog(RMLOG, "After resource manager "
"updates segment info's GRM host name and rack name, "
@@ -1999,8 +2015,8 @@ int addGRMContainerToToBeAccepted(GRMContainer ctn)
appendGRMContainerSetContainer(ctns, ctn);
PRESPOOL->AddPendingContainerCount++;
- elog(LOG, "AddPendingContainerCount added 1, current value %d.",
- PRESPOOL->AddPendingContainerCount);
+ elog(RMLOG, "AddPendingContainerCount added 1, current value %d.",
+ PRESPOOL->AddPendingContainerCount);
return FUNC_RETURN_OK;
}
@@ -2073,15 +2089,15 @@ void addGRMContainerToResPool(GRMContainer container)
reorderSegResourceAllocIndex(segresource, ratio);
reorderSegResourceCombinedWorkloadIndex(segresource);
- elog(LOG, "Resource manager added resource container into resource pool "
- "(%d MB, %d CORE) at %s (%d:%.*s), still pending %d MB",
- container->MemoryMB,
- container->Core,
- container->HostName,
- segresource->Stat->ID,
- segresource->Stat->Info.HostNameLen,
- GET_SEGRESOURCE_HOSTNAME(segresource),
- segresource->IncPending.MemoryMB);
+ elog(RMLOG, "Resource manager added resource container into resource pool "
+ "(%d MB, %d CORE) at %s (%d:%.*s), still pending %d MB",
+ container->MemoryMB,
+ container->Core,
+ container->HostName,
+ segresource->Stat->ID,
+ segresource->Stat->Info.HostNameLen,
+ GET_SEGRESOURCE_HOSTNAME(segresource),
+ segresource->IncPending.MemoryMB);
}
void addGRMContainerToToBeKicked(GRMContainer ctn)
@@ -3059,10 +3075,10 @@ void addSegResourceAllocIndex(SegResource segres)
Assert(false);
}
- elog(LOG, "Resource manager tracked host %s in allocated resource "
- "ordered index for mem/core ratio %d MBPCORE.",
- GET_SEGRESOURCE_HOSTNAME(segres),
- ratio);
+ elog(RMLOG, "Resource manager tracked host %s in allocated resource "
+ "ordered index for mem/core ratio %d MBPCORE.",
+ GET_SEGRESOURCE_HOSTNAME(segres),
+ ratio);
}
}
@@ -3077,8 +3093,8 @@ void addSegResourceCombinedWorkloadIndex(SegResource segres)
Assert(false);
}
- elog(LOG, "Resource manager tracked host %s in io bytes workload.",
- GET_SEGRESOURCE_HOSTNAME(segres));
+ elog(RMLOG, "Resource manager tracked host %s in io bytes workload.",
+ GET_SEGRESOURCE_HOSTNAME(segres));
}
int reorderSegResourceAvailIndex(SegResource segres, uint32_t ratio)
@@ -3490,8 +3506,8 @@ void moveAllAcceptedGRMContainersToResPool(void)
addGRMContainerToResPool(ctn);
PRESPOOL->AddPendingContainerCount--;
- elog(LOG, "AddPendingContainerCount minused 1, current value %d",
- PRESPOOL->AddPendingContainerCount);
+ elog(RMLOG, "AddPendingContainerCount minus 1, current value %d",
+ PRESPOOL->AddPendingContainerCount);
addNewResourceToResourceManager(ctn->MemoryMB, ctn->Core);
removePendingResourceRequestInRootQueue(ctn->MemoryMB, ctn->Core, true);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index 630a38a..e20ef1b 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -1897,9 +1897,9 @@ int acquireResourceFromResQueMgr(ConnectionTrack conntrack,
int res = FUNC_RETURN_OK;
DynResourceQueueTrack queuetrack = conntrack->QueueTrack;
- elog(LOG, "ConnID %d. Expect query resource for session "INT64_FORMAT,
- conntrack->ConnID,
- conntrack->SessionID);
+ elog(RMLOG, "ConnID %d. Expect query resource for session "INT64_FORMAT,
+ conntrack->ConnID,
+ conntrack->SessionID);
/* Call quota logic to make decision of resource for current query. */
res = computeQueryQuota(conntrack, errorbuf, errorbufsize);
@@ -3397,13 +3397,13 @@ int computeQueryQuota(ConnectionTrack conn, char *errorbuf, int errorbufsize)
return res;
}
- elog(LOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d "
- "(MIN %d) after checking queue capacity.",
- conn->ConnID,
- conn->SegMemoryMB,
- conn->SegCore,
- conn->SegNum,
- conn->SegNumMin);
+ elog(RMLOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d "
+ "(MIN %d) after checking queue capacity.",
+ conn->ConnID,
+ conn->SegMemoryMB,
+ conn->SegCore,
+ conn->SegNum,
+ conn->SegNumMin);
/*------------------------------------------------------------------
* The following logic consider the actual resource requirement from
@@ -3427,15 +3427,15 @@ int computeQueryQuota(ConnectionTrack conn, char *errorbuf, int errorbufsize)
conn->SegNum = conn->MaxSegCountFixed;
}
- elog(LOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d "
- "(MIN %d) after checking query expectation %d (MIN %d).",
- conn->ConnID,
- conn->SegMemoryMB,
- conn->SegCore,
- conn->SegNum,
- conn->SegNumMin,
- conn->MaxSegCountFixed,
- conn->MinSegCountFixed);
+ elog(RMLOG, "ConnID %d. Expect query resource (%d MB, %lf CORE) x %d "
+ "(MIN %d) after checking query expectation %d (MIN %d).",
+ conn->ConnID,
+ conn->SegMemoryMB,
+ conn->SegCore,
+ conn->SegNum,
+ conn->SegNumMin,
+ conn->MaxSegCountFixed,
+ conn->MinSegCountFixed);
}
@@ -5003,9 +5003,9 @@ int rebuildAllResourceQueueTrackDynamicStatusInShadow(List *quehavingshadow,
}
else
{
- elog(LOG, "Resource manager passed rebuilding resource queue %s "
- "dynamic status in its shadow.",
- quetrack->QueueInfo->Name);
+ elog(RMLOG, "Resource manager passed rebuilding resource queue %s "
+ "dynamic status in its shadow.",
+ quetrack->QueueInfo->Name);
}
res = detectAndDealWithDeadLockInShadow(quetrack, queuechanged);
@@ -5020,13 +5020,14 @@ int rebuildAllResourceQueueTrackDynamicStatusInShadow(List *quehavingshadow,
}
else
{
- elog(LOG, "Resource manager passed detecting deadlock issues in the "
- "shadow of resource queue %s",
- quetrack->QueueInfo->Name);
+ elog(RMLOG, "Resource manager passed detecting deadlock issues in the "
+ "shadow of resource queue %s",
+ quetrack->QueueInfo->Name);
}
}
- elog(LOG, "Resource manager finished rebuilding resource queues' dynamic status");
+ elog(RMLOG, "Resource manager finished rebuilding resource queues' dynamic "
+ "status");
return FUNC_RETURN_OK;
}
@@ -5037,8 +5038,8 @@ int rebuildResourceQueueTrackDynamicStatusInShadow(DynResourceQueueTrack quetra
{
int res = FUNC_RETURN_OK;
- elog(LOG, "Rebuild resource queue %s dynamic status in its shadow.",
- quetrack->QueueInfo->Name);
+ elog(RMLOG, "Rebuild resource queue %s dynamic status in its shadow.",
+ quetrack->QueueInfo->Name);
DynResourceQueueTrack shadowtrack = quetrack->ShadowQueueTrack;
/* Get deadlock detector ready in the shadow instance. */
@@ -5139,8 +5140,9 @@ int rebuildResourceQueueTrackDynamicStatusInShadow(DynResourceQueueTrack quetra
shadowtrack->DLDetector.InUseTotal.MemoryMB,
shadowtrack->DLDetector.LockedTotal.MemoryMB);
- elog(LOG, "Finished rebuilding resource queue %s dynamic status in its shadow.",
- quetrack->QueueInfo->Name);
+ elog(RMLOG, "Finished rebuilding resource queue %s dynamic status in its "
+ "shadow.",
+ quetrack->QueueInfo->Name);
return FUNC_RETURN_OK;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37839128/src/backend/tcop/pquery.c
----------------------------------------------------------------------
diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c
index 8ef445a..efa5bce 100644
--- a/src/backend/tcop/pquery.c
+++ b/src/backend/tcop/pquery.c
@@ -1123,8 +1123,6 @@ CleanupGlobalQueryResources(void)
int ret;
char errorbuf[1024];
- elog(LOG, "In CleanupGlobalQueryResources().");
-
/* Force using new socket connection to return and free. */
bool oldval = rm_enable_connpool;
rm_enable_connpool = false;