You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sa...@apache.org on 2017/04/18 17:35:40 UTC
[3/5] incubator-trafodion git commit: Changed return lobtype iafter
prepare during describe,
to be FS type 160.Changed a few limits for lob chunksize to handle large files
faster. Changed sqlci to handle param input as LOB.
Changed return lobtype iafter prepare during describe, to be FS type 160.Changed a few limits for lob chunksize to handle large files faster. Changed sqlci to handle param input as LOB.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/738c15e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/738c15e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/738c15e9
Branch: refs/heads/master
Commit: 738c15e95005602e6563b9a381756ce1377ca6a2
Parents: ac9b227
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Wed Apr 5 08:07:36 2017 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Wed Apr 5 08:07:36 2017 +0000
----------------------------------------------------------------------
core/sql/bin/clitest.cpp | 6 ++--
core/sql/comexe/ComTdbExeUtil.cpp | 2 +-
core/sql/comexe/ComTdbExeUtil.h | 2 +-
core/sql/exp/exp_clause_derived.h | 2 +-
core/sql/exp/exp_conv.cpp | 16 +++++++----
core/sql/exp/exp_fixup.cpp | 4 +--
core/sql/generator/GenRelExeUtil.cpp | 4 +--
core/sql/optimizer/BindItemExpr.cpp | 9 ++++++
core/sql/optimizer/SynthType.cpp | 47 ++++++++++---------------------
core/sql/sqlci/Param.cpp | 32 ++++++++++++---------
core/sql/sqlcomp/nadefaults.cpp | 4 +--
11 files changed, 66 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/bin/clitest.cpp
----------------------------------------------------------------------
diff --git a/core/sql/bin/clitest.cpp b/core/sql/bin/clitest.cpp
index f95fa3a..137943a 100644
--- a/core/sql/bin/clitest.cpp
+++ b/core/sql/bin/clitest.cpp
@@ -366,7 +366,7 @@ case 9:
cin.getline(columnname,40);
cout << "Column Name : " << columnname << endl;
cout << "Input source file name :" << endl;
- cin.getline(filename,256);
+ cin.getline(filename,512);
cout << "Source name : " << filename << endl;
//extract lob handle
@@ -392,11 +392,11 @@ case 9:
}
Int64 sourceEOF = statbuf.st_size;
- char *fileInputData= new char[32*1024]; //32 KB chunks
+ char *fileInputData= new char[32*1024*1024]; //32 KB chunks
//char *fileInputData = new char[10000];
Int64 lengthRead=0;
Int64 offset=0;
- Int64 chunkLen = 32*1024;
+ Int64 chunkLen = 32*1024*1024;
//Int64 chunkLen = 10000;
while ((offset < sourceEOF) )
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/comexe/ComTdbExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.cpp b/core/sql/comexe/ComTdbExeUtil.cpp
index 2d0e477..67dbc59 100644
--- a/core/sql/comexe/ComTdbExeUtil.cpp
+++ b/core/sql/comexe/ComTdbExeUtil.cpp
@@ -2426,7 +2426,7 @@ ComTdbExeUtilLobExtract::ComTdbExeUtilLobExtract
if (toType_ == ExtractToType::TO_FILE_)
{
// extractSize_ is irrelevant since the whole lob will be read into the output file
- // bufAddr_ is not passed in by user. It is a CQD value LOB_OUTPUT_SIZE
+ // bufAddr_ is not passed in by user. It is a CQD value LOB_MAX_CHUNK_MEM_SIZE
extractSizeIOAddr_ = 0;
bufAddr_ = 0;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/comexe/ComTdbExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h
index fe63422..58c032f 100644
--- a/core/sql/comexe/ComTdbExeUtil.h
+++ b/core/sql/comexe/ComTdbExeUtil.h
@@ -3003,7 +3003,7 @@ public:
void setLobMaxSize(Int64 lms) {lobMaxSize_ = lms;};
Int64 getLobMaxSize() { return lobMaxSize_;};
void setLobMaxChunkSize(Int64 lmcs) {lobMaxChunkSize_ = lmcs;};
- Int64 getLobMaxChunkSize() { return lobMaxChunkSize_*1024*1024;};
+ Int64 getLobMaxChunkSize() { return lobMaxChunkSize_;};
void setLobGCLimit(Int64 gcl ) {lobGCLimit_ = gcl;};
Int64 getLobGCLimit() { return lobGCLimit_;};
private:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/exp/exp_clause_derived.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/exp_clause_derived.h b/core/sql/exp/exp_clause_derived.h
index e2c7646..9932c82 100644
--- a/core/sql/exp/exp_clause_derived.h
+++ b/core/sql/exp/exp_clause_derived.h
@@ -1696,7 +1696,7 @@ enum ConvInstruction {
CONV_BIN8S_BIN8U =285,
CONV_BIN8U_BIN8S =286,
-
+ CONV_ASCII_F_BLOB =287
};
class SQLEXP_LIB_FUNC ex_conv_clause : public ex_clause {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/exp/exp_conv.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/exp_conv.cpp b/core/sql/exp/exp_conv.cpp
index 8c93b05..c1c9c38 100644
--- a/core/sql/exp/exp_conv.cpp
+++ b/core/sql/exp/exp_conv.cpp
@@ -11458,15 +11458,21 @@ convDoIt(char * source,
*(Int64*)target = (year - 1900)* 10000 + month * 100 + day;
}
break;
-
+
case CONV_BLOB_BLOB:
{
- str_cpy_all(target, source, sourceLen);
-
- setVCLength(varCharLen, varCharLenSize, sourceLen);
+ str_cpy_all(target, source, sourceLen);
+ setVCLength(varCharLen, varCharLenSize, sourceLen);
+ }
+ break;
+ case CONV_ASCII_F_BLOB:
+ {
+ //convert fixed length input to a varchar format LOB buffer input
+ //targetType must have enough room to hold source and a 2 byte length
+ str_cpy_all(target,source,sourceLen);
+ // setVCLength(varCharLen,sizeof(Int16),sourceLen);
}
break;
-
case CONV_BLOB_ASCII_F:
{
// conversion from internal format blob handle to external format
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/exp/exp_fixup.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/exp_fixup.cpp b/core/sql/exp/exp_fixup.cpp
index 801e109..29d4f58 100644
--- a/core/sql/exp/exp_fixup.cpp
+++ b/core/sql/exp/exp_fixup.cpp
@@ -1604,7 +1604,7 @@ const ex_conv_clause::ConvInstrStruct ex_conv_clause::convInstrInfo[] = {
{REC_BYTE_F_ASCII, REC_NCHAR_V_ANSI_UNICODE, instrAndText(CONV_ASCII_TO_ANSI_V_UNICODE)},
{REC_BYTE_F_ASCII, REC_BOOLEAN, instrAndText(CONV_ASCII_BOOL)},
-
+ {REC_BYTE_F_ASCII, REC_BLOB, instrAndText(CONV_ASCII_F_BLOB)},
{REC_NCHAR_F_UNICODE, REC_NCHAR_F_UNICODE, instrAndText(CONV_UNICODE_F_F)},
{REC_NCHAR_F_UNICODE, REC_NCHAR_V_UNICODE, instrAndText(CONV_UNICODE_F_V)},
{REC_NCHAR_F_UNICODE, REC_BYTE_F_ASCII, instrAndText(CONV_UNICODE_F_ASCII_F)},
@@ -1719,7 +1719,7 @@ const ex_conv_clause::ConvInstrStruct ex_conv_clause::convInstrInfo[] = {
{REC_NCHAR_V_ANSI_UNICODE, REC_NCHAR_F_UNICODE, instrAndText(CONV_ANSI_V_UNICODE_TO_UNICODE_F)},
{REC_NCHAR_V_ANSI_UNICODE, REC_NCHAR_V_UNICODE, instrAndText(CONV_ANSI_V_UNICODE_TO_UNICODE_V)},
- {REC_BLOB, REC_BLOB, instrAndText(CONV_BLOB_BLOB)},
+ {REC_BLOB, REC_BLOB, instrAndText(CONV_BLOB_BLOB)},
{REC_BLOB, REC_BYTE_F_ASCII, instrAndText(CONV_BLOB_ASCII_F)},
{REC_CLOB, REC_CLOB, instrAndText(CONV_BLOB_BLOB)},
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/generator/GenRelExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp
index 8c1f64d..76c6bc8 100644
--- a/core/sql/generator/GenRelExeUtil.cpp
+++ b/core/sql/generator/GenRelExeUtil.cpp
@@ -4430,9 +4430,9 @@ short ExeUtilLobUpdate::codeGen(Generator * generator)
addExplainInfo(exe_util_lobupdate_tdb, childExplainTuple, 0, generator));
}
- exe_util_lobupdate_tdb->setTotalBufSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE));
+ exe_util_lobupdate_tdb->setTotalBufSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024);
exe_util_lobupdate_tdb->setLobMaxSize( CmpCommon::getDefaultNumeric(LOB_MAX_SIZE) * 1024 * 1024);
- exe_util_lobupdate_tdb->setLobMaxChunkSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE));
+ exe_util_lobupdate_tdb->setLobMaxChunkSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024);
exe_util_lobupdate_tdb->setLobGCLimit(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)*1024*1024);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/optimizer/BindItemExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindItemExpr.cpp b/core/sql/optimizer/BindItemExpr.cpp
index d3f018a..432afba 100644
--- a/core/sql/optimizer/BindItemExpr.cpp
+++ b/core/sql/optimizer/BindItemExpr.cpp
@@ -6105,6 +6105,15 @@ ItemExpr *Assign::bindNode(BindWA *bindWA)
ValueId vid1 = child(1)->castToItemExpr()->getValueId();
// Add a stringToLob node
ItemExpr *newChild;
+ const NAType &desiredType = child(0)->getValueId().getType();
+ SQLBlob &lobType = (SQLBlob&)desiredType;
+
+ NAType * newType = new SQLBlob((CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024),
+ lobType.getLobStorage(),
+ TRUE, FALSE, TRUE,
+ CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024);
+ // vid1.coerceType(desiredType, NA_LOB_TYPE);
+ vid1.coerceType(*newType, NA_LOB_TYPE);
if (bindWA->getCurrentScope()->context()->inUpdate())
{
newChild = new (bindWA->wHeap()) LOBupdate( vid1.getItemExpr(), child(0), NULL,LOBoper::STRING_, FALSE,TRUE);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/optimizer/SynthType.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/SynthType.cpp b/core/sql/optimizer/SynthType.cpp
index 370dd21..f7b1e34 100644
--- a/core/sql/optimizer/SynthType.cpp
+++ b/core/sql/optimizer/SynthType.cpp
@@ -6692,8 +6692,7 @@ const NAType *LOBoper::synthesizeType()
result = new HEAP SQLClob(1000, Lob_Invalid_Storage,
typ1.supportsSQLnull());
}
- }
-
+ }
return result;
}
@@ -6713,25 +6712,16 @@ const NAType *LOBinsert::synthesizeType()
(obj_ == EXTERNAL_) ||
(obj_ == LOAD_))
{
- if ((obj_ == STRING_)&& (typ1 && typ1->getTypeQualifier() == NA_UNKNOWN_TYPE))
- {
- //Check if it's a special case of untyped param.
- //If so ,assume it's a varchar format.
+
+ if (typ1 && typ1->getTypeQualifier() != NA_CHARACTER_TYPE)
+ {
if (!lobAsVarchar())
{
// 4221 The operand of a $0~String0 function must be character.
*CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBINSERT")
- << DgString1("CHARACTER");
+ << DgString1("CHARACTER");
return NULL;
}
-
- }
- else if (typ1 && typ1->getTypeQualifier() != NA_CHARACTER_TYPE)
- {
- // 4221 The operand of a $0~String0 function must be character.
- *CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBINSERT")
- << DgString1("CHARACTER");
- return NULL;
}
}
else if (obj_ == LOB_)
@@ -6807,35 +6797,28 @@ const NAType *LOBupdate::synthesizeType()
(obj_ == FILE_) ||
(obj_ == EXTERNAL_))
{
- if ((obj_ == STRING_)&& (typ1 && typ1->getTypeQualifier() == NA_UNKNOWN_TYPE))
- {
- //Check if it's a special case of untyped param.
- //If so ,assume it's a varchar format.
- if (!lobAsVarchar())
+
+ if (typ1->getTypeQualifier() != NA_CHARACTER_TYPE)
+ {
+ if(!lobAsVarchar())
{
// 4221 The operand of a $0~String0 function must be character.
*CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBUPDATE")
- << DgString1("CHARACTER");
+ << DgString1("CHARACTER");
return NULL;
}
-
- }
- else if (typ1->getTypeQualifier() != NA_CHARACTER_TYPE)
- {
- // 4221 The operand of a $0~String0 function must be character.
- *CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBUPDATE")
- << DgString1("CHARACTER");
- return NULL;
}
}
else if (obj_ == LOB_)
{
if (typ1->getTypeQualifier() != NA_LOB_TYPE)
{
- // 4043 The operand of a $0~String0 function must be blob
- *CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBUPDATE")
+
+ // 4043 The operand of a $0~String0 function must be blob
+ *CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBUPDATE")
<< DgString1("LOB");
- return NULL;
+ return NULL;
+
}
}
else if (obj_ == BUFFER_)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/sqlci/Param.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlci/Param.cpp b/core/sql/sqlci/Param.cpp
index 3f81a83..d84821b 100644
--- a/core/sql/sqlci/Param.cpp
+++ b/core/sql/sqlci/Param.cpp
@@ -346,19 +346,22 @@ short Param::convertValue(SqlciEnv * sqlci_env, short targetType,
case REC_BYTE_V_ASCII_LONG:
case REC_NCHAR_F_UNICODE:
case REC_NCHAR_V_UNICODE:
+ case REC_BLOB:
{
char *VCLen = NULL;
short VCLenSize = 0;
-
-// 5/27/98: added VARNCHAR cases
+ if (targetType == REC_BLOB)
+ targetLen = sourceLen;
+
+ // 5/27/98: added VARCHAR cases
if ((targetType == REC_BYTE_V_ASCII) ||
(targetType == REC_BYTE_V_ASCII_LONG) ||
- (targetType == REC_NCHAR_V_UNICODE))
- {
- // add bytes for variable length field
- VCLenSize = vcIndLen; //sizeof(short);
- VCLen = converted_value = new char[targetLen + VCLenSize];
- } else
+ (targetType == REC_NCHAR_V_UNICODE))
+ {
+ // add bytes for variable length field
+ VCLenSize = vcIndLen; //sizeof(short);
+ VCLen = converted_value = new char[targetLen + VCLenSize];
+ } else
converted_value = new char[targetLen];
@@ -369,10 +372,13 @@ short Param::convertValue(SqlciEnv * sqlci_env, short targetType,
NAString* tempstr;
if (
- DFS2REC::isAnyCharacter(sourceType) && DFS2REC::isAnyCharacter(targetType) &&
- !(getUTF16StrLit() != NULL && sourceType == REC_NCHAR_F_UNICODE && targetScale == CharInfo::UCS2) &&
- /*source*/cs != targetScale/*i.e., targetCharSet*/
- )
+ (
+ DFS2REC::isAnyCharacter(sourceType) && DFS2REC::isAnyCharacter(targetType) &&
+ !(getUTF16StrLit() != NULL && sourceType == REC_NCHAR_F_UNICODE && targetScale == CharInfo::UCS2) &&
+ /*source*/cs != targetScale/*i.e., targetCharSet*/
+ )
+ )
+
{
charBuf cbuf((unsigned char*)pParamValue, sourceLen);
NAWcharBuf* wcbuf = 0;
@@ -393,7 +399,7 @@ short Param::convertValue(SqlciEnv * sqlci_env, short targetType,
if ( sourceLen > targetLen )
return SQL_Error;
}
-
+
ok = convDoIt(pParamValue,
sourceLen,
sourceType,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/738c15e9/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index 0814a15..299c11d 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -2085,12 +2085,12 @@ SDDkwd__(ISO_MAPPING, (char *)SQLCHARSETSTRING_ISO88591),
DDint__(LOB_HDFS_PORT, "0"),
DD_____(LOB_HDFS_SERVER, "default"),
- // Size of memoryin bytes used to perform I/O to lob data file
+ // Size of memoryin Megabytes used to perform I/O to lob data file
// default size is 512MB . Change to adjust memory usage.
DDint__(LOB_MAX_CHUNK_MEM_SIZE, "512"),
// default size is 10 G (10000 M)
DDint__(LOB_MAX_SIZE, "10000"),
- // default size is 32000. Change this to extract more data into memory.
+ // (unused)default size is 32000. Change this to extract more data into memory.
DDui___(LOB_OUTPUT_SIZE, "32000"),
DD_____(LOB_STORAGE_FILE_DIR, "/lobs"),