You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by se...@apache.org on 2017/02/14 21:05:40 UTC
[2/3] incubator-trafodion git commit: [TRAFODION-2478] Reduce the
number of memory monitoring threads in Trafodion SQL processes
[TRAFODION-2478] Reduce the number of memory monitoring threads in Trafodion SQL processes
Rework as per the review comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/8ba597f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/8ba597f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/8ba597f3
Branch: refs/heads/master
Commit: 8ba597f3d081737e9e62bfdbc861d2bba7bd0a36
Parents: 3d7df2d
Author: selvaganesang <se...@esgyn.com>
Authored: Fri Feb 10 23:56:56 2017 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Tue Feb 14 00:06:28 2017 +0000
----------------------------------------------------------------------
core/sql/bin/ex_esp_main.cpp | 5 +++--
core/sql/cli/memorymonitor.cpp | 29 ++++++++++++++---------------
core/sql/cli/memorymonitor.h | 17 +++++++----------
3 files changed, 24 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8ba597f3/core/sql/bin/ex_esp_main.cpp
----------------------------------------------------------------------
diff --git a/core/sql/bin/ex_esp_main.cpp b/core/sql/bin/ex_esp_main.cpp
index d8bb1eb..b377c63 100644
--- a/core/sql/bin/ex_esp_main.cpp
+++ b/core/sql/bin/ex_esp_main.cpp
@@ -364,10 +364,11 @@ Int32 runESP(Int32 argc, char** argv, GuaReceiveFastStart *guaReceiveFastStart)
// Start the memory monitor for dynamic memory management
Lng32 memMonitorWindowSize = 10;
Lng32 memMonitorSampleInterval = 10;
- MemoryMonitor memMonitor(memMonitorWindowSize,
+ MemoryMonitor *memMonitor = new (espExecutorHeap)
+ MemoryMonitor(memMonitorWindowSize,
memMonitorSampleInterval,
espExecutorHeap);
- cliGlobals->setMemoryMonitor(&memMonitor);
+ cliGlobals->setMemoryMonitor(memMonitor);
}
// After CLI globals are initialized but before we begin ESP message
// processing, have the CLI context set its user identity based on
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8ba597f3/core/sql/cli/memorymonitor.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/memorymonitor.cpp b/core/sql/cli/memorymonitor.cpp
index b53b3d6..33abc00 100644
--- a/core/sql/cli/memorymonitor.cpp
+++ b/core/sql/cli/memorymonitor.cpp
@@ -59,6 +59,7 @@ extern "C" Lng32 __stdcall
PdhOpenQuery (LPCSTR szD, DWORD dw, HQUERY *phQ );
#endif
+#define FREAD_BUFFER_SIZE 2048
//SQ_LINUX #ifdef NA_WINNT
@@ -66,7 +67,7 @@ NABoolean MemoryMonitor::threadIsCreated_ = 0;
HANDLE MemoryMonitor::updateThread_ = (HANDLE) 0;
ULng32 MemoryMonitor::threadId_ = 0;
-DWORD WINAPI memMonitorUpdateThread(void * param) {
+DWORD WINAPI MemoryMonitor::memMonitorUpdateThread(void * param) {
// params points to the memory monitor object
MemoryMonitor *memMonitor = (MemoryMonitor*) param;
Lng32 sleepTime = memMonitor->getSampleInterval();
@@ -102,18 +103,17 @@ MemoryMonitor::MemoryMonitor(Lng32 windowSize,
{
// if the windowSize is 0, we do not need memory monitor.
assert(windowSize);
- char buffer[2048];
+ char buffer[FREAD_BUFFER_SIZE+1];
char *currPtr;
- size_t bytesRead;
+ size_t bytesRead = 0;
fd_meminfo_ = fopen("/proc/meminfo", "r");
if (fd_meminfo_) {
- bytesRead = fread(buffer, 1, 2048, fd_meminfo_);
+ bytesRead = fread(buffer, 1, FREAD_BUFFER_SIZE, fd_meminfo_);
if (ferror(fd_meminfo_))
assert(false);
if (feof(fd_meminfo_))
clearerr(fd_meminfo_);
- else
- buffer[bytesRead] = '\0';
+ buffer[bytesRead] = '\0';
currPtr = strstr(buffer, "MemTotal");
if (currPtr) {
sscanf(currPtr, "%*s " PF64 " kB", &memTotal_);
@@ -152,7 +152,7 @@ MemoryMonitor::MemoryMonitor(Lng32 windowSize,
{
// and finally start the update thread
updateThread_ = CreateNewThread(
- &memMonitorUpdateThread, // Thread func
+ &MemoryMonitor::memMonitorUpdateThread, // Thread func
this ); // Argument for thread
threadIsCreated_ = TRUE;
}
@@ -222,19 +222,18 @@ void MemoryMonitor::update(float &scale) {
Int32 success = fseek(fd_meminfo_, 0, SEEK_SET);
if (success != 0)
return;
- char buffer[4096];
+ char buffer[FREAD_BUFFER_SIZE+1];
Int64 memFree = -1, memCommitAS = 0;
Int64 pgpgout = -1, pgpgin = -1;
- size_t bytesRead;
+ size_t bytesRead = 0;
char * currPtr;
- bytesRead = fread(buffer, 1, 2048, fd_meminfo_);
+ bytesRead = fread(buffer, 1, FREAD_BUFFER_SIZE, fd_meminfo_);
// Make sure there wasn't an error (next fseek will clear eof)
if (ferror(fd_meminfo_))
assert(false);
if (feof(fd_meminfo_))
clearerr(fd_meminfo_);
- else
- buffer[bytesRead] = '\0';
+ buffer[bytesRead] = '\0';
currPtr = strstr(buffer, "MemFree");
if (currPtr) sscanf(currPtr, "%*s " PF64 " kB", &memFree);
currPtr = strstr(buffer, "Committed_AS");
@@ -268,13 +267,13 @@ void MemoryMonitor::update(float &scale) {
pressure_ = 0;
return;
}
- bytesRead = fread(buffer, 1, 2048, fd_vmstat_);
+ bytesRead = 0;
+ bytesRead = fread(buffer, 1, FREAD_BUFFER_SIZE, fd_vmstat_);
if (ferror(fd_vmstat_))
assert(false);
if (feof(fd_vmstat_))
clearerr(fd_vmstat_);
- else
- buffer[bytesRead] = '\0';
+ buffer[bytesRead] = '\0';
currPtr = strstr(buffer, "pgpgin");
if (currPtr) sscanf(currPtr, "%*s " PF64 " kB", &pgpgin);
currPtr = strstr(buffer, "pgpgout");
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8ba597f3/core/sql/cli/memorymonitor.h
----------------------------------------------------------------------
diff --git a/core/sql/cli/memorymonitor.h b/core/sql/cli/memorymonitor.h
index 9ddefc0..fe109e1 100644
--- a/core/sql/cli/memorymonitor.h
+++ b/core/sql/cli/memorymonitor.h
@@ -50,7 +50,8 @@
// NT performance data helper DLL.
// The memory monitor itself. There is one instance of MemoryMonitor per
-// process (executor, ESP, or external sort process)
+// node. The MXSSCP process on the node creates a thread to constantly update
+// the MemoryMonitor object values
class MemoryMonitor : public NABasicObject {
public:
// windowSize is the number of entries in the slinding window we keep
@@ -59,19 +60,11 @@ public:
MemoryMonitor(Lng32 windowSize, Lng32 sampleInterval, CollHeap *heap);
~MemoryMonitor();
-//SQ_LINUX #ifdef NA_WINNT
-
// calculate the memory pressure indicator. This indicator has a value between 0
// (no pressure) and 99 (system is in deep, deep trouble). It is up to the
// caller to decide on the action to take based on this value
- Lng32 memoryPressure();
-
- // explicitly update the performance counters in the MemoryMonitor.
- // Whenever an operator changed its memory consumption significantly, the
- // counters should be updated to reflect the change
- void update(float &scale);
+ Lng32 memoryPressure();
- void updatePageFaultRate(Int64 pageFaultValue);
inline float getPageFaultRate() { return pageFaultRate_; }
inline Lng32 getSampleInterval() const { return sampleInterval_; };
@@ -83,6 +76,10 @@ public:
inline void setEnable(NABoolean b) { enable_ = b; }
Int64 availablePhyMemKb() { return memFree_; }
+ static DWORD WINAPI memMonitorUpdateThread(void * param);
+private:
+ void update(float &scale);
+ void updatePageFaultRate(Int64 pageFaultValue);
private: