You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sa...@apache.org on 2018/04/05 21:24:11 UTC
[1/2] trafodion git commit: Performance changes for LOB and
increasing limit for larger varchars/bobs to use with batch input.
Repository: trafodion
Updated Branches:
refs/heads/master 81640f930 -> 052f7faa4
Performance changes for LOB and increasing limit for larger varchars/bobs to use with batch input.
Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/528e9c60
Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/528e9c60
Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/528e9c60
Branch: refs/heads/master
Commit: 528e9c60345234e554d32db637c82728b63c945b
Parents: efaebe1
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Thu Mar 29 05:56:15 2018 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Thu Mar 29 05:56:15 2018 +0000
----------------------------------------------------------------------
core/sql/common/CharType.cpp | 2 +-
core/sql/common/CharType.h | 27 ++++++-----
core/sql/exp/ExpLOBaccess.cpp | 27 ++++++-----
core/sql/exp/exp_attrs.cpp | 12 ++---
core/sql/exp/exp_attrs.h | 6 ++-
core/sql/generator/GenExpGenerator.cpp | 7 +--
core/sql/generator/GenRelExeUtil.cpp | 5 +-
core/sql/generator/GenRelJoin.cpp | 8 ++--
core/sql/generator/GenRelPackedRows.cpp | 7 ++-
core/sql/generator/Generator.h | 1 +
core/sql/optimizer/BindItemExpr.cpp | 12 +++--
core/sql/optimizer/ItemFunc.h | 34 ++++++++------
core/sql/optimizer/ObjectNames.cpp | 2 +-
core/sql/optimizer/SynthType.cpp | 8 ++--
core/sql/parser/sqlparser.y | 4 +-
core/sql/qmscommon/QRLogger.cpp | 2 +-
core/sql/sqlcomp/CmpMain.cpp | 8 +++-
core/sql/sqlcomp/nadefaults.cpp | 68 ++++++++++++++++------------
18 files changed, 139 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/common/CharType.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/CharType.cpp b/core/sql/common/CharType.cpp
index 9b7566b..cc6c18a 100644
--- a/core/sql/common/CharType.cpp
+++ b/core/sql/common/CharType.cpp
@@ -1477,7 +1477,7 @@ SQLlob::SQLlob(NAMemory *h,
)
: NAType(h, (ev == NA_LOB_TYPE ? LiteralLOB : LiteralLOB)
, ev
- , (externalFormat ? extFormatLen : 512)
+ , extFormatLen
, allowSQLnull
, allowSQLnull ? SQL_NULL_HDR_SIZE : 0
, TRUE
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/common/CharType.h
----------------------------------------------------------------------
diff --git a/core/sql/common/CharType.h b/core/sql/common/CharType.h
index c92a7db..61eee56 100644
--- a/core/sql/common/CharType.h
+++ b/core/sql/common/CharType.h
@@ -735,16 +735,21 @@ public:
SQLlob(NAMemory *h,
NABuiltInTypeEnum ev,
- Int64 lobLength,
- LobsStorage lobStorage,
+ Int64 lobLength=1024,
+ LobsStorage lobStorage=Lob_Invalid_Storage,
NABoolean allowSQLnull = TRUE,
NABoolean inlineIfPossible = FALSE,
NABoolean externalFormat = FALSE,
- Lng32 extFormatLen = 100);
+ Lng32 extFormatLen=1024 );
SQLlob(const SQLlob & aLob,NAMemory * heap)
- :NAType(aLob,heap),
- charSet_(CharInfo::UnknownCharSet)
- {}
+ :NAType(aLob,heap)
+ {
+ lobLength_ = aLob.lobLength_;
+ lobStorage_ = aLob.lobStorage_;
+ externalFormat_ = aLob.externalFormat_;
+ extFormatLen_ = aLob.extFormatLen_;
+ setCharSet(CharInfo::ISO88591);//lobhandle can only be in ISO format
+ }
//is this type a blob/clob
virtual NABoolean isLob() const {return TRUE;};
@@ -798,14 +803,14 @@ class SQLBlob : public SQLlob
public:
SQLBlob(NAMemory *h,
- Int64 blobLength,
+ Int64 blobLength=1024,
LobsStorage lobStorage = Lob_Invalid_Storage,
NABoolean allowSQLnull = TRUE,
NABoolean inlineIfPossible = FALSE,
NABoolean externalFormat = FALSE,
Lng32 extFormatLen = 1024);
SQLBlob(const SQLBlob & aBlob,NAMemory * heap)
- :SQLlob(aBlob,heap)
+ :SQLlob(aBlob,heap)
{}
virtual short getFSDatatype() const
{return REC_BLOB;}
@@ -838,14 +843,14 @@ class SQLClob : public SQLlob
public:
SQLClob(NAMemory *h,
- Int64 blobLength,
+ Int64 blobLength=1024,
LobsStorage lobStorage = Lob_Invalid_Storage,
NABoolean allowSQLnull = TRUE,
NABoolean inlineIfPossible = FALSE,
NABoolean externalFormat = FALSE,
Lng32 extFormatLen = 1024);
SQLClob(const SQLClob & aClob,NAMemory * heap)
- :SQLlob(aClob,heap)
+ :SQLlob(aClob,heap)
{}
virtual short getFSDatatype() const
{return REC_CLOB;}
@@ -864,6 +869,6 @@ public:
private:
}; // class SQLClob
-// sss #endif
+
#endif /* CHARTYPE_H */
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index b5a427b..93b340f 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -304,15 +304,14 @@ Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operL
if (fInfo == NULL) {
return LOB_DATA_FILE_NOT_FOUND_ERROR;
}
+ if (fdData_)
+ {
+ hdfsCloseFile(fs_, fdData_);
+ fdData_=NULL;
+ }
+ openFlags_ = O_WRONLY | O_APPEND;
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, 0, 0, 0);
}
- hdfsCloseFile(fs_, fdData_);
- fdData_=NULL;
- openFlags_ = O_WRONLY | O_APPEND;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, 0, 0, 0);
- if (!fdData_) {
- openFlags_ = -1;
- return LOB_DATA_FILE_OPEN_ERROR;
- }
if ((operLen = hdfsWrite(fs_, fdData_, data, size)) == -1) {
return LOB_DATA_WRITE_ERROR;
@@ -946,6 +945,10 @@ Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOpe
LobInputOutputFileType srcFileType = fileType(source);
if (srcFileType != HDFS_FILE)
return LOB_SOURCE_FILE_READ_ERROR;
+ //Check if external file exists
+ Int64 sourceEOD = 0;
+ if (statSourceFile(source, sourceEOD) != LOB_OPER_OK)
+ return LOB_SOURCE_FILE_READ_ERROR;
}
// Calculate sourceLen for each subOper.
if ((subOper == Lob_File))
@@ -1256,6 +1259,8 @@ Ex_Lob_Error ExLob::insertSelect(ExLob *srcLobPtr,
// we have received the external data file name from the descriptor table
// replace the contents of the lobDataFile with this name
str_cpy_all(extFileName, blackBox, blackBoxLen);
+ extFileName[blackBoxLen]= '\0';
+
extFileNameLen = blackBoxLen;
// Now insert this into the target lob descriptor
@@ -1822,8 +1827,8 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset,
hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.data());
if (fInfo)
dataOffset = fInfo->mSize;
-
- if ((lobGCLimit != 0) && (dataOffset > lobGCLimit)) // 5 GB default
+ // if -1, don't do GC or if reached the limit do GC
+ if ((lobGCLimit != -1) && (dataOffset > lobGCLimit))
{
str_sprintf(logBuf,"Starting GC. Current Offset : %ld",dataOffset);
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
@@ -1876,7 +1881,7 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
Ex_Lob_Error rc = LOB_OPER_OK;
char logBuf[4096];
lobDebugInfo("In ExLob::compactLobDataFile",0,__LINE__,lobTrace_);
- Int64 maxMemChunk = 1024*1024*1024; //1GB limit for intermediate buffer for transfering data
+ Int64 maxMemChunk = 100*1024*1024; //100 MB limit for intermediate buffer for transfering data
// make some temporary file names
size_t len = lobDataFile_.length();
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/exp/exp_attrs.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/exp_attrs.cpp b/core/sql/exp/exp_attrs.cpp
index 8ded216..7215168 100644
--- a/core/sql/exp/exp_attrs.cpp
+++ b/core/sql/exp/exp_attrs.cpp
@@ -316,15 +316,13 @@ void Attributes::displayContents(Space * space, Int32 operandNum,
if ((getDatatype() == REC_BLOB) ||
(getDatatype() == REC_CLOB))
{
- Int16 precision = getPrecision();
- UInt16 scale = getScaleAsUI();
+
- Lng32 lobLen = (precision << 16);
- lobLen += scale;
-
- Int64 ll = (Int64)lobLen;
+ Int64 ll = getLength();
+ if (isLengthInKB())
+ ll = ll*1024;
// Int64 ll = (Int64)getPrecision() * 1000 + (Int64)getScale();
- str_sprintf(buf, " LobLength = %ld Mb", ll);
+ str_sprintf(buf, " LobLength = %ld bytes", ll);
space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short));
}
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/exp/exp_attrs.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/exp_attrs.h b/core/sql/exp/exp_attrs.h
index 32a818d..62fe617 100644
--- a/core/sql/exp/exp_attrs.h
+++ b/core/sql/exp/exp_attrs.h
@@ -378,6 +378,9 @@ public:
NABoolean isForceFixed() { return (flags_ & FORCE_FIXED_) != 0; }
void setForceFixed() { flags_ |= FORCE_FIXED_; }
+ NABoolean isLengthInKB() { return (flags_ & LENGTH_IN_KB_) != 0; }
+ void setLengthInKB() { flags_ |= LENGTH_IN_KB_; }
+
// Bulk move flags
void setBulkMoveable( NABoolean flag = TRUE ) { (flag ? flags_ |= BULK_MOVE_ : flags_ &= ~BULK_MOVE_); }
NABoolean isBulkMoveable() { return (flags_ & BULK_MOVE_) != 0; }
@@ -584,9 +587,10 @@ private:
CASEINSENSITIVE_ = 0x0400, // caseinsensitive char/varchar datatype
- FORCE_FIXED_ = 0x0800 // Force this attribute to be treated as fixed
+ FORCE_FIXED_ = 0x0800, // Force this attribute to be treated as fixed
// in an aligned row. Used by HashGroupby for
// varchar aggregates
+ LENGTH_IN_KB_ =0x1000 // Indicates length is in KB
};
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/GenExpGenerator.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenExpGenerator.cpp b/core/sql/generator/GenExpGenerator.cpp
index cde3481..92b2aa4 100644
--- a/core/sql/generator/GenExpGenerator.cpp
+++ b/core/sql/generator/GenExpGenerator.cpp
@@ -359,13 +359,8 @@ Attributes * ExpGenerator::convertNATypeToAttributes
else if (naType->getTypeQualifier() == NA_LOB_TYPE)
{
+
SQLlob *lobType = (SQLlob *) naType;
-
- Lng32 lobLen = (Lng32)lobType->getLobLength();
- Int16 precision = (lobLen & 0xFFFF0000) >> 16;
- Int16 scale = lobLen & 0xFFFF;
- attr->setPrecision(precision);
- attr->setScale(scale);
attr->setCharSet(lobType->getCharSet());
}
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/GenRelExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp
index c667f99..5df0849 100644
--- a/core/sql/generator/GenRelExeUtil.cpp
+++ b/core/sql/generator/GenRelExeUtil.cpp
@@ -4410,7 +4410,10 @@ short ExeUtilLobUpdate::codeGen(Generator * generator)
exe_util_lobupdate_tdb->setTotalBufSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024);
exe_util_lobupdate_tdb->setLobMaxSize( CmpCommon::getDefaultNumeric(LOB_MAX_SIZE) * 1024 * 1024);
exe_util_lobupdate_tdb->setLobMaxChunkSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024);
- exe_util_lobupdate_tdb->setLobGCLimit(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)*1024*1024);
+ if ((CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE) >=0))
+ exe_util_lobupdate_tdb->setLobGCLimit(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)*1024*1024);
+ else
+ exe_util_lobupdate_tdb->setLobGCLimit(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE));
generator->setCriDesc(givenDesc, Generator::DOWN);
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/GenRelJoin.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelJoin.cpp b/core/sql/generator/GenRelJoin.cpp
index cc97b1d..f54a48a 100644
--- a/core/sql/generator/GenRelJoin.cpp
+++ b/core/sql/generator/GenRelJoin.cpp
@@ -3482,9 +3482,11 @@ short NestedJoin::codeGen(Generator * generator)
// Make sure that the LHS up queue can grow as large as the RHS down
// queue.
- if(tdb1->getMaxQueueSizeUp() < tdb2->getInitialQueueSizeDown()) {
- tdb1->setMaxQueueSizeUp(tdb2->getInitialQueueSizeDown());
- }
+ //We can't let upqueue of unpack to grow beyond what was calculated and set earlier in codeGen of the unpack operator
+ if (tdb1->getClassID() != ComTdb::ex_UNPACKROWS)
+ if(tdb1->getMaxQueueSizeUp() < tdb2->getInitialQueueSizeDown()) {
+ tdb1->setMaxQueueSizeUp(tdb2->getInitialQueueSizeDown());
+ }
// If this NestedJoin itself is not on the RHS of a Flow/NestedJoin,
// Then reset the largeQueueSize to 0.
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/GenRelPackedRows.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelPackedRows.cpp b/core/sql/generator/GenRelPackedRows.cpp
index adfc388..cc3e6fc 100644
--- a/core/sql/generator/GenRelPackedRows.cpp
+++ b/core/sql/generator/GenRelPackedRows.cpp
@@ -426,10 +426,11 @@ PhysUnPackRows::codeGen(Generator *generator)
ULng32 rowsetSize =
getGroupAttr()->getOutputLogPropList()[0]->getResultCardinality().value();
double memoryLimitPerInstance =
- ActiveSchemaDB()->getDefaults().getAsLong(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB) * 1024 * 1024;
+ (ActiveSchemaDB()->getDefaults().getAsLong(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB) * 1024 * 1024)/2; // At runtime (ExUnpackRowsTcb::ExUnpackRowsTcb) we allocate twice the size of the queue. So ensure it fits in to half the specified limit.
rowsetSize = (rowsetSize < 1024 ? 1024 : rowsetSize);
- double estimatedMemory = rowsetSize * unPackColsTupleLen;
+
+ double estimatedMemory = (Int64)rowsetSize * (Int64)unPackColsTupleLen;
if (estimatedMemory > memoryLimitPerInstance)
{
@@ -466,6 +467,8 @@ PhysUnPackRows::codeGen(Generator *generator)
//
generator->removeAll(localMapTable);
+ //Turn off override of queue sizes for unpack. Once set they should remain.
+ generator->setMakeOnljLeftQueuesBig(FALSE);
// Add the explain Information for this node to the EXPLAIN
// Fragment. Set the explainTuple pointer in the generator so
// the parent of this node can get a handle on this explainTuple.
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/generator/Generator.h
----------------------------------------------------------------------
diff --git a/core/sql/generator/Generator.h b/core/sql/generator/Generator.h
index e4524a8..8f2f50d 100644
--- a/core/sql/generator/Generator.h
+++ b/core/sql/generator/Generator.h
@@ -1580,6 +1580,7 @@ public:
// Helpers for the special ONLJ queue sizing defaults.
NABoolean const getMakeOnljLeftQueuesBig()
{ return makeOnljLeftQueuesBig_; }
+ void setMakeOnljLeftQueuesBig(NABoolean x) {makeOnljLeftQueuesBig_ = x;}
ULng32 const getOnljLeftUpQueue() { return onljLeftUpQueue_; }
ULng32 const getOnljLeftDownQueue() {return onljLeftDownQueue_; }
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/optimizer/BindItemExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindItemExpr.cpp b/core/sql/optimizer/BindItemExpr.cpp
index 3b71061..282c852 100644
--- a/core/sql/optimizer/BindItemExpr.cpp
+++ b/core/sql/optimizer/BindItemExpr.cpp
@@ -6103,20 +6103,22 @@ ItemExpr *Assign::bindNode(BindWA *bindWA)
NAType * newType = NULL;
- double lob_input_limit_for_batch = CmpCommon::getDefaultNumeric(LOB_INPUT_LIMIT_FOR_BATCH);
+ double lob_input_limit_for_batch = CmpCommon::getDefaultNumeric(LOB_INPUT_LIMIT_FOR_BATCH)*1024;
double lob_size = lobType.getLobLength();
if (fs_datatype == REC_CLOB) {
- newType = new (bindWA->wHeap()) SQLClob(bindWA->wHeap(), (CmpCommon::getDefaultNumeric(LOB_MAX_SIZE) * 1024 * 1024),
+ newType = new (bindWA->wHeap()) SQLClob(bindWA->wHeap(), lob_input_limit_for_batch < lob_size ? lob_input_limit_for_batch : lob_size,
lobType.getLobStorage(),
- TRUE, FALSE, TRUE,
+ TRUE, FALSE, FALSE,
lob_input_limit_for_batch < lob_size ? lob_input_limit_for_batch : lob_size);
}
else {
- newType = new (bindWA->wHeap()) SQLBlob(bindWA->wHeap(), (CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024),
+ newType = new (bindWA->wHeap()) SQLBlob(bindWA->wHeap(),lob_input_limit_for_batch < lob_size ? lob_input_limit_for_batch : lob_size ,
lobType.getLobStorage(),
- TRUE, FALSE, TRUE,
+ TRUE, FALSE, FALSE,
lob_input_limit_for_batch < lob_size ? lob_input_limit_for_batch : lob_size);
}
+
+ CMPASSERT(lob_input_limit_for_batch < INT_MAX);
vid1.coerceType(*newType, NA_LOB_TYPE);
if (bindWA->getCurrentScope()->context()->inUpdate())
{
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/optimizer/ItemFunc.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h
index 0435521..ca16965 100644
--- a/core/sql/optimizer/ItemFunc.h
+++ b/core/sql/optimizer/ItemFunc.h
@@ -2869,9 +2869,9 @@ public:
obj_(obj),
lobNum_(-1),
lobStorageType_(Lob_Invalid_Storage),
- lobMaxSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE) * 1024 * 1024),
- lobMaxChunkMemSize_(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)),
- lobGCLimit_(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)),
+ lobMaxSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024),
+ lobMaxChunkMemSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024),
+ lobGCLimit_((Int64) CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)*1024*1024),
hdfsPort_((Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT)),
hdfsServer_( CmpCommon::getDefaultString(LOB_HDFS_SERVER))
{
@@ -2911,8 +2911,14 @@ public:
LobsStorage &lobStorageType() { return lobStorageType_; }
NAString &lobStorageLocation() { return lobStorageLocation_; }
Int64 getLobMaxSize() {return lobMaxSize_; }
- Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_*1024*1024;}
- Int64 getLobGCLimit() { return lobGCLimit_*1024*1024;}
+ Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_;}
+ Int64 getLobGCLimit()
+ {
+ if (lobGCLimit_>0)
+ return (Int64)lobGCLimit_;
+ else
+ return -1;
+ }
Int32 getLobHdfsPort() { return hdfsPort_;}
NAString &getLobHdfsServer(){return hdfsServer_;}
protected:
@@ -2922,8 +2928,8 @@ public:
LobsStorage lobStorageType_;
NAString lobStorageLocation_;
Int64 lobMaxSize_; // In byte units
- Int32 lobMaxChunkMemSize_; //In MB Units
- Int32 lobGCLimit_ ;//In MB Units
+ Int64 lobMaxChunkMemSize_; //In MB Units
+ Int64 lobGCLimit_ ;//In MB Units
Int32 hdfsPort_;
NAString hdfsServer_;
@@ -2943,7 +2949,7 @@ class LOBinsert : public LOBoper
objectUID_(-1),
append_(isAppend),
lobAsVarchar_(treatLobAsVarchar),
- lobSize_(0),
+ lobSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024),
fsType_(REC_BLOB)
{};
@@ -2968,7 +2974,7 @@ class LOBinsert : public LOBoper
// Lng32 & lobNum() { return lobNum_; }
- Lng32 & lobSize() { return lobSize_; }
+ Int64 & lobSize() { return lobSize_; }
Lng32 & lobFsType() { return fsType_; }
NABoolean lobAsVarchar() const { return lobAsVarchar_;}
@@ -2984,7 +2990,7 @@ class LOBinsert : public LOBoper
// column this blob is being inserted into.
// Lng32 lobNum_;
- Lng32 lobSize_;
+ Int64 lobSize_;
Lng32 fsType_;
@@ -3048,7 +3054,7 @@ class LOBupdate : public LOBoper
NABoolean treatLobAsVarchar =FALSE)
: LOBoper(ITM_LOBUPDATE, val1Ptr, val2Ptr,val3Ptr,fromObj),
objectUID_(-1),
- lobSize_(0),
+ lobSize_((Int64)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024),
append_(isAppend),
lobAsVarchar_(treatLobAsVarchar)
{};
@@ -3070,7 +3076,7 @@ class LOBupdate : public LOBoper
Int64 & updatedTableObjectUID() { return objectUID_; }
NAString &updatedTableSchemaName() { return schName_; }
- Lng32 & lobSize() { return lobSize_; }
+ Int64 & lobSize() { return lobSize_; }
NABoolean lobAsVarchar() const { return lobAsVarchar_;}
private:
// ---------------------------------------------------------------//
@@ -3083,7 +3089,7 @@ class LOBupdate : public LOBoper
NABoolean append_;
- Lng32 lobSize_;
+ Int64 lobSize_;
NABoolean lobAsVarchar_;//This means this lob insert will get it's input data
// in varchar format
}; // class LOBupdate
@@ -3092,7 +3098,7 @@ class LOBconvert : public LOBoper
{
public:
- LOBconvert(ItemExpr *val1Ptr, ObjectType toObj, Lng32 tgtSize = 32000)
+ LOBconvert(ItemExpr *val1Ptr, ObjectType toObj, Lng32 tgtSize= CmpCommon::getDefaultNumeric(LOB_OUTPUT_SIZE) )
: LOBoper(ITM_LOBCONVERT, val1Ptr, NULL,NULL,toObj),
tgtSize_(tgtSize)
{};
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/optimizer/ObjectNames.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ObjectNames.cpp b/core/sql/optimizer/ObjectNames.cpp
index fb4f9b1..6547a98 100644
--- a/core/sql/optimizer/ObjectNames.cpp
+++ b/core/sql/optimizer/ObjectNames.cpp
@@ -710,7 +710,7 @@ NABoolean QualifiedName::isHbaseCellOrRow() const
NABoolean QualifiedName::isLOBDesc() const
{
- if (getObjectName().index(LOB_DESC_HANDLE_PREFIX) == 0)
+ if ((getObjectName().index(LOB_DESC_HANDLE_PREFIX) == 0) || (getObjectName().index(LOB_DESC_CHUNK_PREFIX) ==0))
return TRUE;
else
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/optimizer/SynthType.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/SynthType.cpp b/core/sql/optimizer/SynthType.cpp
index 5494472..72d15e6 100644
--- a/core/sql/optimizer/SynthType.cpp
+++ b/core/sql/optimizer/SynthType.cpp
@@ -6680,7 +6680,7 @@ const NAType *LOBoper::synthesizeType()
{
// Return blob or clob type
- NAType *result = new HEAP SQLBlob(HEAP, 1000);
+ NAType *result = new HEAP SQLBlob(HEAP, ((Int64) CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024));
if (child(0))
{
@@ -6689,12 +6689,12 @@ const NAType *LOBoper::synthesizeType()
if (typ1.getFSDatatype() == REC_BLOB)
{
- result = new HEAP SQLBlob(HEAP, 1000, Lob_Local_File,
+ result = new HEAP SQLBlob(HEAP, ((Int64) CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), Lob_Invalid_Storage,
typ1.supportsSQLnull());
}
else if (typ1.getFSDatatype() == REC_CLOB)
{
- result = new HEAP SQLClob(HEAP, 1000, Lob_Invalid_Storage,
+ result = new HEAP SQLClob(HEAP, ((Int64) CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), Lob_Invalid_Storage,
typ1.supportsSQLnull());
}
}
@@ -6923,7 +6923,7 @@ const NAType *LOBconvertHandle::synthesizeType()
return NULL;
}
- result = new HEAP SQLBlob(HEAP, 1000, Lob_Invalid_Storage, typ1.supportsSQLnull(), FALSE,
+ result = new HEAP SQLBlob(HEAP, ((Int64) CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024), Lob_Invalid_Storage, typ1.supportsSQLnull(), FALSE,
FALSE);
return result;
}
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/parser/sqlparser.y
----------------------------------------------------------------------
diff --git a/core/sql/parser/sqlparser.y b/core/sql/parser/sqlparser.y
index f4f6da6..edbca46 100755
--- a/core/sql/parser/sqlparser.y
+++ b/core/sql/parser/sqlparser.y
@@ -11647,7 +11647,7 @@ blob_optional_left_len_right: '(' NUMERIC_LITERAL_EXACT_NO_SCALE optional_lob_un
if (CmpCommon::getDefault(TRAF_BLOB_AS_VARCHAR) == DF_ON)
{
- $$ = (Int64)100000;
+ $$ = (Int64)CmpCommon::getDefault(TRAF_MAX_CHARACTER_COL_LENGTH );
}
else
{
@@ -11683,7 +11683,7 @@ clob_optional_left_len_right: '(' NUMERIC_LITERAL_EXACT_NO_SCALE optional_lob_un
if (CmpCommon::getDefault(TRAF_CLOB_AS_VARCHAR) == DF_ON)
{
- $$ = (Int64)100000;
+ $$ = (Int64)CmpCommon::getDefault(TRAF_MAX_CHARACTER_COL_LENGTH );
}
else
{
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/qmscommon/QRLogger.cpp
----------------------------------------------------------------------
diff --git a/core/sql/qmscommon/QRLogger.cpp b/core/sql/qmscommon/QRLogger.cpp
index f4fb75d..b71f190 100644
--- a/core/sql/qmscommon/QRLogger.cpp
+++ b/core/sql/qmscommon/QRLogger.cpp
@@ -266,7 +266,7 @@ const char* QRLogger::getMyProcessInfo()
XPROCESSHANDLE_DECOMPOSE_(&myphandle, &mycpu, &mypin, &mynodenumber,NULL,100, NULL, myprocname,100, &myproclength);
myprocname[myproclength] = '\0';
- snprintf(procInfo, 300, "Node Number: %d, CPU: %d, PIN: %d, Process Name: %s", mynodenumber,mycpu, mypin, myprocname);
+ snprintf(procInfo, 300, "Node Number: %d, CPU: %d, PIN: %d, Process Name: %s", mycpu,mycpu, mypin, myprocname);
processInfo_ = procInfo;
return processInfo_.data();
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/sqlcomp/CmpMain.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpMain.cpp b/core/sql/sqlcomp/CmpMain.cpp
index b7b8851..650d181 100644
--- a/core/sql/sqlcomp/CmpMain.cpp
+++ b/core/sql/sqlcomp/CmpMain.cpp
@@ -725,9 +725,13 @@ CmpMain::ReturnStatus CmpMain::sqlcomp(QueryText& input, //IN
//if using special tables e.g. using index as base table
//select * from table (index_table T018ibc);
- //then refresh metadata cache
+ //then refresh metadata cache. Make an exception for internal exeutil
+ // statements that use this parserflag. It causes too many long compiles
+ // and affects performance - for eg LOB access which uses ghost tables and
+ // has the SPECIALTABLETYPE flag set..
if(Get_SqlParser_Flags(ALLOW_SPECIALTABLETYPE) &&
- CmpCommon::context()->schemaDB_->getNATableDB()->cachingMetaData())
+ CmpCommon::context()->schemaDB_->getNATableDB()->cachingMetaData() &&
+ !Get_SqlParser_Flags(INTERNAL_QUERY_FROM_EXEUTIL) )
CmpCommon::context()->schemaDB_->getNATableDB()->refreshCacheInThisStatement();
MonitorMemoryUsage_Enter("Parser");
http://git-wip-us.apache.org/repos/asf/trafodion/blob/528e9c60/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index a75c20b..f8f8e75 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -1154,7 +1154,7 @@ SDDkwd__(EXE_DIAGNOSTIC_EVENTS, "OFF"),
SDDui___(EXE_MEMORY_FOR_PROBE_CACHE_IN_MB, "100"),
- SDDui___(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB, "100"),
+ SDDui___(EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB, "1024"),
// lower-bound memory limit for BMOs/nbmos (in MB)
DDui___(EXE_MEMORY_LIMIT_LOWER_BOUND_EXCHANGE, "10"),
@@ -1741,21 +1741,21 @@ SDDkwd__(ISO_MAPPING, (char *)SQLCHARSETSTRING_ISO88591),
// precision but degraded performance.
SDDkwd__(LIMIT_MAX_NUMERIC_PRECISION, "SYSTEM"),
- // Size in bytes used to perform garbage collection to lob data file
- // default size is 5GB . Change to adjust disk usage. If 0 it means
- // don't do GC
- DDint__(LOB_GC_LIMIT_SIZE, "5000"),
+ // Size in MB used to perform garbage collection to lob data file
+ // Recommended size is 20GB . Change to adjust disk usage. If -1 it means
+ // don't do GC during insert/update operations.
+ DDint__(LOB_GC_LIMIT_SIZE, "-1"),
DDint__(LOB_HDFS_PORT, "0"),
DD_____(LOB_HDFS_SERVER, "default"),
-
- DDint__(LOB_INPUT_LIMIT_FOR_BATCH, "4096"),
+ // For JDBC/ODBC batch operations, LOB size limited to 4K bytes
+ DDint__(LOB_INPUT_LIMIT_FOR_BATCH, "16384"),
// Size of memoryin Megabytes used to perform I/O to lob data file
- // default size is 512MB . Change to adjust memory usage.
- DDint__(LOB_MAX_CHUNK_MEM_SIZE, "512"),
- // default size is 10 G (10000 M)
- DDint__(LOB_MAX_SIZE, "10000"),
- // (unused)default size is 32000. Change this to extract more data into memory.
+ // default size is 128MB . Change to adjust memory usage.
+ DDint__(LOB_MAX_CHUNK_MEM_SIZE, "128"),
+ // default size is 5 G (5120 M)
+ DDint__(LOB_MAX_SIZE, "5120"),
+ // default size is 32000bytes. Change this to extract more data into memory.
DDui___(LOB_OUTPUT_SIZE, "32000"),
DD_____(LOB_STORAGE_FILE_DIR, "/user/trafodion/lobs"),
@@ -1915,7 +1915,7 @@ SDDkwd__(ISO_MAPPING, (char *)SQLCHARSETSTRING_ISO88591),
DDui___(MEMORY_LIMIT_QCACHE_UPPER_KB, "0"),
// Checked at compile time. Set to -1 to disable check.
// Value should be >= EXE_MEMORY_FOR_UNPACK_ROWS_IN_MB
- DDint__(MEMORY_LIMIT_ROWSET_IN_MB, "500"),
+ DDint__(MEMORY_LIMIT_ROWSET_IN_MB, "1024"),
// SQL/MX Compiler/Optimzer Memory Monitor.
DDkwd__(MEMORY_MONITOR, "OFF"),
@@ -5427,7 +5427,32 @@ enum DefaultConstants NADefaults::validateAndInsert(const char *attrName,
}
}
break;
+ case LOB_GC_LIMIT_SIZE:
+ if ((atoi(value.data()) < -1) || (((atoi(value.data()) >0) && (atoi(value.data())) <= 512)))
+ {
+ *CmpCommon::diags() << DgSqlCode(-2055)
+ << DgString0(value)
+ << DgString1(lookupAttrName(attrEnum));
+ }
+ break;
+ case LOB_MAX_CHUNK_MEM_SIZE:
+ if (atoi(value.data()) < 0 || atoi(value.data()) > 5120)
+ {
+ *CmpCommon::diags() << DgSqlCode(-2055)
+ << DgString0(value)
+ << DgString1(lookupAttrName(attrEnum));
+ }
+ break;
+
+ case LOB_INPUT_LIMIT_FOR_BATCH:
+ if (atoi(value.data()) <0 && (atoi(value.data())*1024) > INT_MAX )
+ {
+ *CmpCommon::diags() << DgSqlCode(-2055)
+ << DgString0(value)
+ << DgString1(lookupAttrName(attrEnum));
+ }
+ break;
default:
break;
}
@@ -6466,20 +6491,6 @@ DefaultToken NADefaults::token(Int32 attrEnum,
isValid = TRUE;
break;
- case LOB_OUTPUT_SIZE:
- if (tok >=0 && tok <= 512000)
- isValid = TRUE;
- break;
-
- case LOB_MAX_CHUNK_MEM_SIZE:
- if (tok >=0 && tok <= 512000)
- isValid = TRUE;
- break;
-
- case LOB_GC_LIMIT_SIZE:
- if (tok >= 0 )
- isValid=TRUE;
-
case TRAF_TRANS_TYPE:
if (tok == DF_MVCC || tok == DF_SSCC)
isValid = TRUE;
@@ -6782,8 +6793,7 @@ void NADefaults::setSchemaAsLdapUser(const NAString val)
insert(SCHEMA, schName);
}
else
- {
- *CmpCommon::diags() << DgSqlCode(-2055)
+ { *CmpCommon::diags() << DgSqlCode(-2055)
<< DgString0(schName)
<< DgString1("SCHEMA");
}
[2/2] trafodion git commit: Merge remote branch 'origin/pr/1503/head'
into merge_1503
Posted by sa...@apache.org.
Merge remote branch 'origin/pr/1503/head' into merge_1503
Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/052f7faa
Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/052f7faa
Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/052f7faa
Branch: refs/heads/master
Commit: 052f7faa4174560729ae1b263efb403127bfc6d7
Parents: 81640f9 528e9c6
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Thu Apr 5 21:23:50 2018 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Thu Apr 5 21:23:50 2018 +0000
----------------------------------------------------------------------
core/sql/common/CharType.cpp | 2 +-
core/sql/common/CharType.h | 27 ++++++-----
core/sql/exp/ExpLOBaccess.cpp | 27 ++++++-----
core/sql/exp/exp_attrs.cpp | 12 ++---
core/sql/exp/exp_attrs.h | 6 ++-
core/sql/generator/GenExpGenerator.cpp | 7 +--
core/sql/generator/GenRelExeUtil.cpp | 5 +-
core/sql/generator/GenRelJoin.cpp | 8 ++--
core/sql/generator/GenRelPackedRows.cpp | 7 ++-
core/sql/generator/Generator.h | 1 +
core/sql/optimizer/BindItemExpr.cpp | 12 +++--
core/sql/optimizer/ItemFunc.h | 34 ++++++++------
core/sql/optimizer/ObjectNames.cpp | 2 +-
core/sql/optimizer/SynthType.cpp | 8 ++--
core/sql/parser/sqlparser.y | 4 +-
core/sql/qmscommon/QRLogger.cpp | 2 +-
core/sql/sqlcomp/CmpMain.cpp | 8 +++-
core/sql/sqlcomp/nadefaults.cpp | 68 ++++++++++++++++------------
18 files changed, 139 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/052f7faa/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/052f7faa/core/sql/generator/GenExpGenerator.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/052f7faa/core/sql/optimizer/BindItemExpr.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/052f7faa/core/sql/optimizer/ItemFunc.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/052f7faa/core/sql/optimizer/ObjectNames.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/052f7faa/core/sql/optimizer/SynthType.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/052f7faa/core/sql/parser/sqlparser.y
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/052f7faa/core/sql/qmscommon/QRLogger.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/052f7faa/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------