You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by se...@apache.org on 2017/09/18 17:01:36 UTC
[1/3] incubator-trafodion git commit: [TRAFODION-2738] Rowset buffer
size during insert/upsert should be limited
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 574452e97 -> e2789f97c
[TRAFODION-2738] Rowset buffer size during insert/upsert should be limited
Rowset insert includes UnpackRows operator in its query tree. UnPackRows
operator was allocating a larger memory based on rowset size and row size
to flow the data to its parent operator.
Introduced a CQD EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB to limit the amount
of memory allocated in this operator. The default value is 100MB.
In addition, streamlined the NAMemory infrastruture to return NULL or
dump core when more than 2 GB of memory is allocated in one chunk.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/3a15d7c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/3a15d7c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/3a15d7c8
Branch: refs/heads/master
Commit: 3a15d7c8932fa26b123d4d9c85e41e8d7bf697b0
Parents: 574452e
Author: selvaganesang <se...@esgyn.com>
Authored: Fri Sep 15 14:37:07 2017 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Fri Sep 15 14:37:07 2017 +0000
----------------------------------------------------------------------
core/sql/common/ComSpace.cpp | 11 +++++++++--
core/sql/common/ComSpace.h | 2 +-
core/sql/common/NAMemory.cpp | 23 ++++++++++++++++++++---
core/sql/common/NAMemory.h | 4 ++++
core/sql/generator/GenRelPackedRows.cpp | 14 ++++++++++++--
core/sql/sqlcomp/DefaultConstants.h | 2 ++
core/sql/sqlcomp/nadefaults.cpp | 2 ++
7 files changed, 50 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3a15d7c8/core/sql/common/ComSpace.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/ComSpace.cpp b/core/sql/common/ComSpace.cpp
index 12f2e87..33bbdd2 100644
--- a/core/sql/common/ComSpace.cpp
+++ b/core/sql/common/ComSpace.cpp
@@ -381,6 +381,8 @@ char *Space::privateAllocateSpace(ULng32 size, NABoolean failureIsFatal) {
}
void * Space::allocateSpaceMemory(size_t size, NABoolean failureIsFatal) {
+ if (! checkSize(size, failureIsFatal))
+ return NULL;
void * rc = allocateAlignedSpace(size, failureIsFatal);
HEAPLOG_ADD_ENTRY(rc, size, heapID_.heapNum, getName())
if (rc) return rc;
@@ -398,9 +400,11 @@ void * Space::allocateSpaceMemory(size_t size, NABoolean failureIsFatal) {
}
-char *Space::allocateAlignedSpace(ULng32 size, NABoolean failureIsFatal) {
+char *Space::allocateAlignedSpace(size_t size, NABoolean failureIsFatal) {
if (size <= 0)
- return 0;
+ return NULL;
+ if (! checkSize(size, failureIsFatal))
+ return NULL;
// return aligned space on an 8 byte boundary
return privateAllocateSpace((ULng32) roundUp8((Lng32)size), failureIsFatal);
@@ -426,6 +430,9 @@ char *Space::allocateAndCopyToAlignedSpace(const char *dp,
alen = countPrefixSize + dlen;
}
+ if (! checkSize(alen, failureIsFatal))
+ return NULL;
+
char* rp = allocateAlignedSpace(alen, failureIsFatal);
switch (countPrefixSize) {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3a15d7c8/core/sql/common/ComSpace.h
----------------------------------------------------------------------
diff --git a/core/sql/common/ComSpace.h b/core/sql/common/ComSpace.h
index 4965c70..50c3b2f 100644
--- a/core/sql/common/ComSpace.h
+++ b/core/sql/common/ComSpace.h
@@ -129,7 +129,7 @@ public:
void setParent(CollHeap * parent);
- char * allocateAlignedSpace(ULng32 size, NABoolean failureIsFatal = TRUE);
+ char * allocateAlignedSpace(size_t size, NABoolean failureIsFatal = TRUE);
char * allocateAndCopyToAlignedSpace(const char* dp,
size_t dlen,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3a15d7c8/core/sql/common/NAMemory.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/NAMemory.cpp b/core/sql/common/NAMemory.cpp
index ce78db9..05e42be 100644
--- a/core/sql/common/NAMemory.cpp
+++ b/core/sql/common/NAMemory.cpp
@@ -2489,6 +2489,16 @@ NABoolean NAMemory::getUsage(size_t * lastBlockSize, size_t * freeSize, size_t *
return crowded;
}
+NABoolean NAMemory::checkSize(size_t size, NABoolean failureIsFatal)
+{
+ if (size > MAX_MEMORY_SIZE_IN_AN_ALLOC) {
+ if (failureIsFatal)
+ abort();
+ else
+ return FALSE;
+ }
+ return TRUE;
+}
// ---------------------------------------------------------------------------
// NASegGlobals methods
@@ -2931,7 +2941,10 @@ void * NAHeap::allocateHeapMemory(size_t userSize, NABoolean failureIsFatal)
// allocate 0 bytes. But this would waste memory to maintain a
// heap fragment of size 0.
if (userSize == 0)
- return NULL;
+ return NULL;
+
+ if (! checkSize(userSize, failureIsFatal))
+ return NULL;
// getSharedMemory() check alone is enough since it will return for both
// global and process stats heap. Leaving the rest of the condition here
@@ -3163,7 +3176,7 @@ void * NAHeap::allocateHeapMemory(size_t userSize, NABoolean failureIsFatal)
// If we return from this call it means that the caller wanted
// a memory allocation failure to be fatal yet did not set the
// the jump buffer. This is not good.
- assert(0);
+ abort();
}
// Caller will handle the error so just return null.
@@ -4090,6 +4103,9 @@ void DefaultIpcHeap::destroy()
void * DefaultIpcHeap::allocateIpcHeapMemory(size_t size, NABoolean failureIsFatal)
{
+ if (! checkSize(size, failureIsFatal))
+ return NULL;
+
void * rc = ::operator new(size);
#pragma nowarn(1506) // warning elimination
HEAPLOG_ADD_ENTRY(rc, size, heapID_.heapNum, getName())
@@ -4099,7 +4115,7 @@ void * DefaultIpcHeap::allocateIpcHeapMemory(size_t size, NABoolean failureIsFat
{
// Might never return...
handleExhaustedMemory();
- assert(0);
+ abort();
}
return rc;
}
@@ -4142,3 +4158,4 @@ SEG_ID getStatsSegmentId()
}
return gStatsSegmentId_;
}
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3a15d7c8/core/sql/common/NAMemory.h
----------------------------------------------------------------------
diff --git a/core/sql/common/NAMemory.h b/core/sql/common/NAMemory.h
index a6f04c5..f6ff77d 100644
--- a/core/sql/common/NAMemory.h
+++ b/core/sql/common/NAMemory.h
@@ -86,6 +86,8 @@ class NATreeFragment;
class MemBinsem;
class SegmentStatus;
+#define MAX_MEMORY_SIZE_IN_AN_ALLOC INT_MAX
+
// MemoryStats is used for dynamically allocated statistics (when MEMDEBUG=1 or higher)
// and if NAHeap::dump() is called.
@@ -403,6 +405,8 @@ public:
void setSharedMemory() { sharedMemory_ = TRUE; }
NABoolean isComSpace(void) { return derivedClass_ == COMSPACE_CLASS; } ;
+
+ NABoolean checkSize(size_t size, NABoolean failureIsFatal);
protected:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3a15d7c8/core/sql/generator/GenRelPackedRows.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelPackedRows.cpp b/core/sql/generator/GenRelPackedRows.cpp
index 5a81488..1f34b72 100644
--- a/core/sql/generator/GenRelPackedRows.cpp
+++ b/core/sql/generator/GenRelPackedRows.cpp
@@ -414,8 +414,18 @@ PhysUnPackRows::codeGen(Generator *generator)
// queue size values represent initial (and final) queue
// sizes (not max queue sizes).
//
- queue_index upQueueSize =
- (queue_index)getGroupAttr()->getOutputLogPropList()[0]->getResultCardinality().value();
+ ULng32 rowsetSize =
+ getGroupAttr()->getOutputLogPropList()[0]->getResultCardinality().value();
+ double memoryLimitPerInstance =
+ ActiveSchemaDB()->getDefaults().getAsLong(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB) * 1024 * 1024;
+ double estimatedMemory = rowsetSize * unPackColsTupleLen;
+
+ if (estimatedMemory > memoryLimitPerInstance)
+ {
+ estimatedMemory = memoryLimitPerInstance;
+ rowsetSize = estimatedMemory / unPackColsTupleLen;
+ }
+ queue_index upQueueSize = rowsetSize;
// Make sure it is at least 1024.
upQueueSize = (upQueueSize < 1024 ? 1024 : upQueueSize);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3a15d7c8/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h
index ac4eb33..3852e70 100644
--- a/core/sql/sqlcomp/DefaultConstants.h
+++ b/core/sql/sqlcomp/DefaultConstants.h
@@ -3887,6 +3887,8 @@ enum DefaultConstants
// equally across all BMO operators
BMO_MEMORY_EQUAL_QUOTA_SHARE_RATIO,
+ EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB,
+
// This enum constant must be the LAST one in the list; it's a count,
// not an Attribute (it's not IN DefaultDefaults; it's the SIZE of it)!
__NUM_DEFAULT_ATTRIBUTES
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3a15d7c8/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index 42548bf..382d430 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -1336,6 +1336,8 @@ SDDkwd__(EXE_DIAGNOSTIC_EVENTS, "OFF"),
SDDui___(EXE_MEMORY_FOR_PROBE_CACHE_IN_MB, "100"),
+ SDDui___(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB, "100"),
+
// lower-bound memory limit for BMOs/nbmos (in MB)
DDui___(EXE_MEMORY_LIMIT_LOWER_BOUND_EXCHANGE, "10"),
DDui___(EXE_MEMORY_LIMIT_LOWER_BOUND_MERGEJOIN, "10"),
[3/3] incubator-trafodion git commit: Merge PR 1232 [TRAFODION-2738]
Rowset buffer size during insert/upsert should be limited
Posted by se...@apache.org.
Merge PR 1232 [TRAFODION-2738] Rowset buffer size during insert/upsert should be limited
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/e2789f97
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/e2789f97
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/e2789f97
Branch: refs/heads/master
Commit: e2789f97c0a0705fbbf352ee5e88d9c0e5a7a703
Parents: 574452e 4534b2b
Author: selvaganesang <se...@apache.org>
Authored: Mon Sep 18 17:00:29 2017 +0000
Committer: selvaganesang <se...@apache.org>
Committed: Mon Sep 18 17:00:29 2017 +0000
----------------------------------------------------------------------
core/sql/common/ComSpace.cpp | 11 +++++++++--
core/sql/common/ComSpace.h | 2 +-
core/sql/common/NAMemory.cpp | 23 ++++++++++++++++++++---
core/sql/common/NAMemory.h | 4 ++++
core/sql/executor/ex_queue.cpp | 26 +++++++++++++++-----------
core/sql/executor/ex_queue.h | 3 ++-
core/sql/generator/GenRelPackedRows.cpp | 25 +++++++++++++++++--------
core/sql/sqlcomp/DefaultConstants.h | 2 ++
core/sql/sqlcomp/nadefaults.cpp | 2 ++
9 files changed, 72 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
[2/3] incubator-trafodion git commit: [TRAFODION-2738] Rowset buffer
size during insert/upsert should be limited
Posted by se...@apache.org.
[TRAFODION-2738] Rowset buffer size during insert/upsert should be limited
Ensured that up queue size is set to proper value in ExUnPackRowsTcb
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/4534b2b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/4534b2b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/4534b2b2
Branch: refs/heads/master
Commit: 4534b2b26af1800902531d612d4a2fdca3453208
Parents: 3a15d7c
Author: selvaganesang <se...@esgyn.com>
Authored: Fri Sep 15 22:49:27 2017 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Fri Sep 15 22:49:27 2017 +0000
----------------------------------------------------------------------
core/sql/executor/ex_queue.cpp | 26 +++++++++++++++-----------
core/sql/executor/ex_queue.h | 3 ++-
core/sql/generator/GenRelPackedRows.cpp | 11 +++++------
3 files changed, 22 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4534b2b2/core/sql/executor/ex_queue.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ex_queue.cpp b/core/sql/executor/ex_queue.cpp
index 460aa60..3bd10d3 100644
--- a/core/sql/executor/ex_queue.cpp
+++ b/core/sql/executor/ex_queue.cpp
@@ -66,17 +66,7 @@ ex_queue::ex_queue(const queue_type type,
// make size a power of 2 and greater than 1.
//
- ULng32 count = 1;
- queue_index s = size_ - 1;
- while (s > 1) {
- count++;
- s = s >> 1;
- };
-
- if (count > (sizeof(queue_index) * 8))
- size_ = maxQueueSize;
- else
- size_ = (1 << count);
+ size_ = roundUp2Power(size_);
ex_assert(size_ > 1, "invalid queue size");
@@ -950,7 +940,21 @@ void ex_queue::logRemoveHead()
#endif
}
+queue_index ex_queue::roundUp2Power(queue_index i)
+{
+ ULng32 count = 1;
+ queue_index s = i - 1;
+ while (s > 1) {
+ count++;
+ s = s >> 1;
+ };
+ if (count > (sizeof(queue_index) * 8))
+ s = maxQueueSize;
+ else
+ s = (1 << count);
+ return s;
+}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4534b2b2/core/sql/executor/ex_queue.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ex_queue.h b/core/sql/executor/ex_queue.h
index 6e4d82a..2b51c52 100644
--- a/core/sql/executor/ex_queue.h
+++ b/core/sql/executor/ex_queue.h
@@ -324,6 +324,7 @@ inline
{ ExQueueAssert(upDown_ == DOWN_QUEUE,""); nextSubtask_ = nextSubtask; }
//?johannes??
+ static queue_index roundUp2Power(queue_index i);
private:
@@ -444,7 +445,7 @@ private:
}
void logRemoveHead();
-
+
}; // ex_queue
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4534b2b2/core/sql/generator/GenRelPackedRows.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelPackedRows.cpp b/core/sql/generator/GenRelPackedRows.cpp
index 1f34b72..4088035 100644
--- a/core/sql/generator/GenRelPackedRows.cpp
+++ b/core/sql/generator/GenRelPackedRows.cpp
@@ -27,6 +27,7 @@
#include "GenExpGenerator.h"
#include "ExpCriDesc.h"
#include "ComTdbUnPackRows.h"
+#include "ex_queue.h"
// PhysUnPackRows::preCodeGen() -------------------------------------------
// Perform local query rewrites such as for the creation and
@@ -418,20 +419,18 @@ PhysUnPackRows::codeGen(Generator *generator)
getGroupAttr()->getOutputLogPropList()[0]->getResultCardinality().value();
double memoryLimitPerInstance =
ActiveSchemaDB()->getDefaults().getAsLong(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB) * 1024 * 1024;
+
+ rowsetSize = (rowsetSize < 1024 ? 1024 : rowsetSize);
double estimatedMemory = rowsetSize * unPackColsTupleLen;
if (estimatedMemory > memoryLimitPerInstance)
{
estimatedMemory = memoryLimitPerInstance;
rowsetSize = estimatedMemory / unPackColsTupleLen;
+ rowsetSize = MAXOF(rowsetSize, 1);
}
- queue_index upQueueSize = rowsetSize;
-
- // Make sure it is at least 1024.
- upQueueSize = (upQueueSize < 1024 ? 1024 : upQueueSize);
- // Make sure that it is not more the 64K.
- upQueueSize = (upQueueSize > 65536 ? 65536 : upQueueSize);
+ queue_index upQueueSize = ex_queue::roundUp2Power((queue_index)rowsetSize);
unPackTdb =
new (space) ComTdbUnPackRows(childTdb,