You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sa...@apache.org on 2017/04/18 17:35:39 UTC
[2/5] incubator-trafodion git commit: Rework the fix for
[TRAFODION-2516] to returntype for param as LOB instead of coercing the type
by default to char type since it confuses JDBC and Catalog APIs. The expected
input param still is expected as a varchar
Rework the fix for [TRAFODION-2516] to returntype for param as LOB instead of coercing the type by default to char type since it confuses JDBC and Catalog APIs. The expected input param still is expected as a varchar format. Also added testcases both for the regression as well as the clitestdriver program.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/ac9b2273
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/ac9b2273
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/ac9b2273
Branch: refs/heads/master
Commit: ac9b22737ee98d782d5ecb9986d36ba8c4fa28e5
Parents: 45bbb16
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Sun Apr 2 04:46:59 2017 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Sun Apr 2 04:46:59 2017 +0000
----------------------------------------------------------------------
core/sql/bin/clitest.cpp | 63 +++++++++++++++++-
core/sql/cli/Cli.cpp | 2 +-
core/sql/clitest/blobtest.cpp | 5 +-
core/sql/clitest/blobtest.h | 5 +-
core/sql/comexe/ComTdbExeUtil.h | 2 +-
core/sql/exp/ExpLOBaccess.cpp | 8 ++-
core/sql/generator/GenRelExeUtil.cpp | 2 +-
core/sql/optimizer/BindItemExpr.cpp | 57 +++++++++++-----
core/sql/optimizer/ItemFunc.h | 19 ++++--
core/sql/optimizer/SynthType.cpp | 43 +++++++++----
core/sql/regress/executor/EXPECTED130 | 100 +++++++++++++++++------------
core/sql/regress/executor/TEST130 | 5 ++
12 files changed, 224 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/bin/clitest.cpp
----------------------------------------------------------------------
diff --git a/core/sql/bin/clitest.cpp b/core/sql/bin/clitest.cpp
index bdb8a56..f95fa3a 100644
--- a/core/sql/bin/clitest.cpp
+++ b/core/sql/bin/clitest.cpp
@@ -85,6 +85,7 @@ int main(int argc, const char * argv[])
cout << "Blob test to update to lob lobhandle 6 "<< endl;
cout << "Blob test to append to lobhandle 7 "<< endl;
cout << "Blob test to truncate to lobhandle 8 "<< endl;
+ cout << "Blob test append lobhandle from a buffer in chunks from file 9 " << endl;
return 0;
}
Int32 retcode = 0;
@@ -316,7 +317,9 @@ case 7:
return retcode;
}
cout << "LOB handle for "<< columnname << ": " << lobHandle << endl;
- retcode = updateAppendBufferToLobHandle(cliGlob,lobHandle);
+ char *source= new char[300];
+ strcpy(source,"zzzzzzzzzzzzzzzzzzzz");
+ retcode = updateAppendBufferToLobHandle(cliGlob,lobHandle,20,(Int64)source);
return retcode;
}
break;
@@ -351,6 +354,64 @@ case 8:
}
break;
+case 9:
+ {
+
+ cout <<"*************************************" <<endl;
+ cout << "Blob test append lobhandle from a buffer from file " << endl;
+ cout << "Input lob table name :" << endl;
+ cin.getline(tablename,40);
+ cout << "Table name : " << tablename << endl;
+ cout << "Input lob column name to get handle from :" << endl;
+ cin.getline(columnname,40);
+ cout << "Column Name : " << columnname << endl;
+ cout << "Input source file name :" << endl;
+ cin.getline(filename,256);
+ cout << "Source name : " << filename << endl;
+ //extract lob handle
+
+ char *lobHandle = new char[1024];
+ str_cpy_all(lobHandle," ",1024);
+ cout << "Extracting lob handle for column " << columnname << "..." << endl;
+ retcode = extractLobHandle(cliGlob, lobHandle, (char *)columnname,tablename);
+ if (retcode)
+ {
+ cout << "extractLobHandle returned " << retcode <<endl;
+
+ delete lobHandle;
+ return retcode;
+ }
+ cout << "LOB handle for "<< columnname << ": " << lobHandle << endl;
+
+
+ //read from input source file in chunks and append
+ int fdSourceFile = open(filename,O_RDONLY);
+ struct stat statbuf;
+ if (stat(filename, &statbuf) != 0) {
+ return -1;
+ }
+
+ Int64 sourceEOF = statbuf.st_size;
+ char *fileInputData= new char[32*1024]; //32 KB chunks
+ //char *fileInputData = new char[10000];
+ Int64 lengthRead=0;
+ Int64 offset=0;
+ Int64 chunkLen = 32*1024;
+ //Int64 chunkLen = 10000;
+
+ while ((offset < sourceEOF) )
+ {
+ if (lengthRead=pread(fdSourceFile, fileInputData, chunkLen, offset) == -1) {
+ close(fdSourceFile);
+ return -1;
+ }
+ chunkLen = min(chunkLen,sourceEOF-offset);
+ retcode = updateAppendBufferToLobHandle(cliGlob,lobHandle,chunkLen,(Int64)fileInputData);
+ offset +=chunkLen;
+ }
+ return retcode;
+ }
+ break;
}//end switch
return 0;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index 27e6b1e..1a6eaf5 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -10384,7 +10384,7 @@ Lng32 SQLCLI_LOB_GC_Interface
ExLobInMemoryDescChunksEntry *dcInMemoryArray = new ExLobInMemoryDescChunksEntry[numEntries];
//Read the desc chunks table into memory
- str_sprintf(query, "select dataOffset, descPartnKey,descSyskey,chunkLen,chunkNum from table(ghost table %s) order by dataOffset for read committed access",
+ str_sprintf(query, "select dataOffset, descPartnKey,descSyskey,chunkLen,chunkNum from table(ghost table %s) order by dataOffset,chunkLen for read committed access",
lobDescChunksName);
lobDebugInfo(logBuf,0,__LINE__,lobTrace);
// set parserflags to allow ghost table
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/clitest/blobtest.cpp
----------------------------------------------------------------------
diff --git a/core/sql/clitest/blobtest.cpp b/core/sql/clitest/blobtest.cpp
index 71eeca3..843f05a 100644
--- a/core/sql/clitest/blobtest.cpp
+++ b/core/sql/clitest/blobtest.cpp
@@ -303,7 +303,7 @@ Int32 updateBufferToLobHandle(CliGlobals *cliglob,char *handle)
}
-Int32 updateAppendBufferToLobHandle(CliGlobals *cliglob,char *handle)
+Int32 updateAppendBufferToLobHandle(CliGlobals *cliglob,char *handle, Int64 lobUpdateLen, Int64 sourceAddress)
{
Int32 retcode = 0;
ExeCliInterface cliInterface((cliglob->currContext())->exHeap(), (Int32)SQLCHARSETCODE_UTF8, cliglob->currContext(),NULL);
@@ -313,9 +313,8 @@ Int32 updateAppendBufferToLobHandle(CliGlobals *cliglob,char *handle)
char statusBuf[200] = {'\0'};
Int32 statusBufLen = 0;
- Int64 lobUpdateLen = 20;
char *lobDataBuf = new char[lobUpdateLen];
- memcpy(lobDataBuf, "zzzzzzzzzzzzzzzzzzzz",20);
+ memcpy(lobDataBuf, (char *)sourceAddress,lobUpdateLen);
str_sprintf(query,"update lob (LOB '%s' , LOCATION %Ld, SIZE %Ld,append )", handle, (Int64)lobDataBuf, lobUpdateLen);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/clitest/blobtest.h
----------------------------------------------------------------------
diff --git a/core/sql/clitest/blobtest.h b/core/sql/clitest/blobtest.h
index 65b53c0..6df473a 100644
--- a/core/sql/clitest/blobtest.h
+++ b/core/sql/clitest/blobtest.h
@@ -62,6 +62,9 @@
#include "ExExeUtil.h"
#include "Globals.h"
#include "Context.h"
+#include <sys/stat.h>
+#include <stdlib.h>
+
Int32 extractLobHandle(CliGlobals *cliglob, char *& lobHandle,
char *lobColumnName, char *tableName);
@@ -76,4 +79,4 @@ Int32 updateBufferToLob(CliGlobals *cliglob, char *tableName, char *columnName);
Int32 updateAppendBufferToLob(CliGlobals *cliGlob, char *tableName, char *columnName);
Int32 updateBufferToLobHandle(CliGlobals *cliglob, char *handle);
Int32 updateTruncateLobHandle(CliGlobals *cliglob, char *handle);
-Int32 updateAppendBufferToLobHandle(CliGlobals *cliglob, char *handle);
+Int32 updateAppendBufferToLobHandle(CliGlobals *cliglob, char *handle,Int64 updateLen, Int64 sourceAddress);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/comexe/ComTdbExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h
index 58c032f..fe63422 100644
--- a/core/sql/comexe/ComTdbExeUtil.h
+++ b/core/sql/comexe/ComTdbExeUtil.h
@@ -3003,7 +3003,7 @@ public:
void setLobMaxSize(Int64 lms) {lobMaxSize_ = lms;};
Int64 getLobMaxSize() { return lobMaxSize_;};
void setLobMaxChunkSize(Int64 lmcs) {lobMaxChunkSize_ = lmcs;};
- Int64 getLobMaxChunkSize() { return lobMaxChunkSize_;};
+ Int64 getLobMaxChunkSize() { return lobMaxChunkSize_*1024*1024;};
void setLobGCLimit(Int64 gcl ) {lobGCLimit_ = gcl;};
Int64 getLobGCLimit() { return lobGCLimit_;};
private:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 3e06949..27d5011 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -2102,11 +2102,12 @@ Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 siz
//handle reading the multiple chunks like a cursor
err = readCursor(lobData,chunkSize, handleIn,
handleInLen, operLen, transId);
-
+ if ((operLen == 0) && (err == LOB_OPER_OK)) //this may be an empty lob section
+ continue;
if ((err != LOB_OPER_OK) || (operLen != chunkSize))
{
getLobGlobalHeap()->deallocateMemory(lobData);
- return err;
+ return LOB_DATA_READ_ERROR;
}
writeOperLen += pwrite(fdDestFile, lobData, chunkSize, tgtOffset) ;
@@ -2224,7 +2225,8 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s
//handle reading the multiple chunks like a cursor
err = readCursor(lobData,chunkSize, handleIn,
handleInLen, operLen, transId);
-
+ if ((operLen == 0) && (err == LOB_OPER_OK)) //this may be an empty lob section
+ continue;
if ((err != LOB_OPER_OK) || (operLen != chunkSize))
{
getLobGlobalHeap()->deallocateMemory(lobData);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/generator/GenRelExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp
index 2987075..8c1f64d 100644
--- a/core/sql/generator/GenRelExeUtil.cpp
+++ b/core/sql/generator/GenRelExeUtil.cpp
@@ -4433,7 +4433,7 @@ short ExeUtilLobUpdate::codeGen(Generator * generator)
exe_util_lobupdate_tdb->setTotalBufSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE));
exe_util_lobupdate_tdb->setLobMaxSize( CmpCommon::getDefaultNumeric(LOB_MAX_SIZE) * 1024 * 1024);
exe_util_lobupdate_tdb->setLobMaxChunkSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE));
- exe_util_lobupdate_tdb->setLobGCLimit(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE));
+ exe_util_lobupdate_tdb->setLobGCLimit(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)*1024*1024);
generator->setCriDesc(givenDesc, Generator::DOWN);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/optimizer/BindItemExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindItemExpr.cpp b/core/sql/optimizer/BindItemExpr.cpp
index 91e1325..d3f018a 100644
--- a/core/sql/optimizer/BindItemExpr.cpp
+++ b/core/sql/optimizer/BindItemExpr.cpp
@@ -6065,14 +6065,20 @@ ItemExpr *Assign::bindNode(BindWA *bindWA)
}
#endif
- ItemExpr *boundExpr ;
+ ItemExpr *boundExpr, *boundExpr_0, *boundExpr_1 ;
- boundExpr = child(0)->bindNode(bindWA);
+ boundExpr_0 = child(0)->bindNode(bindWA);
if (bindWA->errStatus())
return this;
- child(0) = boundExpr;
+ child(0) = boundExpr_0;
-
+ if (child(1))
+ {
+ boundExpr_1 = child(1)->bindNode(bindWA);
+ if (bindWA->errStatus())
+ return this;
+ child(1) = boundExpr_1;
+ }
if (CmpCommon::getDefault(JDBC_PROCESS) == DF_ON)
{
// if an untyped param is being assigned to a column, and
@@ -6085,6 +6091,35 @@ ItemExpr *Assign::bindNode(BindWA *bindWA)
return ie;
}
+
+ NABuiltInTypeEnum targetType = child(0)->castToItemExpr()->getValueId().getType().getTypeQualifier() ;
+ if (targetType == NA_LOB_TYPE)
+ {
+ if (child(1))
+ {
+ NABuiltInTypeEnum sourceType = child(1)->castToItemExpr()->getValueId().getType().getTypeQualifier() ;
+ //If it's a dynamic param with unknown type or if it is a
+ // character type, trasnform the insert.
+ if ((child(1)->getOperatorType() == ITM_DYN_PARAM && sourceType == NA_UNKNOWN_TYPE) || sourceType == NA_CHARACTER_TYPE)
+ {
+ ValueId vid1 = child(1)->castToItemExpr()->getValueId();
+ // Add a stringToLob node
+ ItemExpr *newChild;
+ if (bindWA->getCurrentScope()->context()->inUpdate())
+ {
+ newChild = new (bindWA->wHeap()) LOBupdate( vid1.getItemExpr(), child(0), NULL,LOBoper::STRING_, FALSE,TRUE);
+ }
+ else
+ {
+ newChild = new (bindWA->wHeap()) LOBinsert( vid1.getItemExpr(),NULL, LOBoper::STRING_, FALSE,TRUE);
+ }
+ newChild->bindNode(bindWA);
+ if (bindWA->errStatus())
+ return boundExpr;
+ setChild(1, newChild);
+ }
+ }
+ }
// Assign is a directly derived subclass of ItemExpr; safe to invoke this
boundExpr = ItemExpr::bindNode(bindWA);
if (bindWA->errStatus())
@@ -6173,18 +6208,8 @@ ItemExpr *Assign::bindNode(BindWA *bindWA)
} // QSTUFF
} // isUserSpecified
- NABuiltInTypeEnum sourceType = child(1)->castToItemExpr()->getValueId().getType().getTypeQualifier() ;
- NABuiltInTypeEnum targetType = child(0)->castToItemExpr()->getValueId().getType().getTypeQualifier() ;
- if ((sourceType == NA_CHARACTER_TYPE) && (targetType == NA_LOB_TYPE))
- {
- ValueId vid1 = child(1)->castToItemExpr()->getValueId();
- // Add a stringToLob node
- ItemExpr *newChild = new (bindWA->wHeap()) LOBinsert( vid1.getItemExpr(), NULL, LOBoper::STRING_, FALSE);
- newChild->bindNode(bindWA);
- if (bindWA->errStatus())
- return boundExpr;
- setChild(1, newChild);
- }
+
+ targetType = child(0)->castToItemExpr()->getValueId().getType().getTypeQualifier() ;
if ((NOT child(0)->getValueId().getType().
isCompatible(child(1)->getValueId().getType())) &&
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/optimizer/ItemFunc.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h
index eaf9a1c..388c81e 100644
--- a/core/sql/optimizer/ItemFunc.h
+++ b/core/sql/optimizer/ItemFunc.h
@@ -2937,10 +2937,12 @@ class LOBinsert : public LOBoper
ItemExpr *val2Ptr,
ObjectType fromObj,
NABoolean isAppend = FALSE,
+ NABoolean treatLobAsVarchar =FALSE,
OperatorTypeEnum otype = ITM_LOBINSERT)
: LOBoper(otype, val1Ptr, val2Ptr,NULL,fromObj),
objectUID_(-1),
append_(isAppend),
+ lobAsVarchar_(treatLobAsVarchar),
lobSize_(0),
fsType_(REC_BLOB)
{};
@@ -2969,7 +2971,7 @@ class LOBinsert : public LOBoper
Lng32 & lobSize() { return lobSize_; }
Lng32 & lobFsType() { return fsType_; }
-
+ NABoolean lobAsVarchar() const { return lobAsVarchar_;}
protected:
// ---------------------------------------------------------------//
// ObjectUID of the table this blob is being inserted into
@@ -2987,6 +2989,8 @@ class LOBinsert : public LOBoper
Lng32 fsType_;
NABoolean append_;
+ NABoolean lobAsVarchar_;//This means this lob insert will get it's input data
+ // in varchar format
}; // class LOBinsert
class LOBselect : public LOBoper
@@ -3040,11 +3044,13 @@ class LOBupdate : public LOBoper
ItemExpr *val2Ptr,
ItemExpr *val3Ptr,
ObjectType fromObj,
- NABoolean isAppend = FALSE)
+ NABoolean isAppend = FALSE,
+ NABoolean treatLobAsVarchar =FALSE)
: LOBoper(ITM_LOBUPDATE, val1Ptr, val2Ptr,val3Ptr,fromObj),
- objectUID_(-1),
- lobSize_(0),
- append_(isAppend)
+ objectUID_(-1),
+ lobSize_(0),
+ append_(isAppend),
+ lobAsVarchar_(treatLobAsVarchar)
{};
// copyTopNode method
@@ -3065,6 +3071,7 @@ class LOBupdate : public LOBoper
NAString &updatedTableSchemaName() { return schName_; }
Lng32 & lobSize() { return lobSize_; }
+ NABoolean lobAsVarchar() const { return lobAsVarchar_;}
private:
// ---------------------------------------------------------------//
// ObjectUID of the table this blob is being inserted into
@@ -3077,6 +3084,8 @@ class LOBupdate : public LOBoper
NABoolean append_;
Lng32 lobSize_;
+ NABoolean lobAsVarchar_;//This means this lob insert will get it's input data
+ // in varchar format
}; // class LOBupdate
class LOBconvert : public LOBoper
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ac9b2273/core/sql/optimizer/SynthType.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/SynthType.cpp b/core/sql/optimizer/SynthType.cpp
index 9993430..370dd21 100644
--- a/core/sql/optimizer/SynthType.cpp
+++ b/core/sql/optimizer/SynthType.cpp
@@ -1867,16 +1867,7 @@ const NAType *Assign::doSynthesizeType(ValueId & targetId, ValueId & sourceId)
NABoolean sourceIsUntypedParam =
(sourceId.getType().getTypeQualifier() == NA_UNKNOWN_TYPE);
- if ((sourceId.getItemExpr()->getOperatorType() == ITM_CONSTANT)
- && (((ConstValue*)sourceId.getItemExpr())->isNull()))
- isSourceNullConst = TRUE;
- if (sourceIsUntypedParam && (targetType.getTypeQualifier() == NA_LOB_TYPE)
- && !isSourceNullConst)
- {
- ValueId vid1 = child(1)->castToItemExpr()->getValueId();
- SQLVarChar c1(CmpCommon::getDefaultNumeric(MAX_LONG_VARCHAR_DEFAULT_SIZE));
- vid1.coerceType(c1, NA_CHARACTER_TYPE);
- }
+
// Charset inference.
const NAType& sourceType = sourceId.getType();
targetId.coerceType(sourceType);
@@ -6722,8 +6713,21 @@ const NAType *LOBinsert::synthesizeType()
(obj_ == EXTERNAL_) ||
(obj_ == LOAD_))
{
- if (typ1 && typ1->getTypeQualifier() != NA_CHARACTER_TYPE)
- {
+ if ((obj_ == STRING_)&& (typ1 && typ1->getTypeQualifier() == NA_UNKNOWN_TYPE))
+ {
+ //Check if it's a special case of untyped param.
+ //If so ,assume it's a varchar format.
+ if (!lobAsVarchar())
+ {
+ // 4221 The operand of a $0~String0 function must be character.
+ *CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBINSERT")
+ << DgString1("CHARACTER");
+ return NULL;
+ }
+
+ }
+ else if (typ1 && typ1->getTypeQualifier() != NA_CHARACTER_TYPE)
+ {
// 4221 The operand of a $0~String0 function must be character.
*CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBINSERT")
<< DgString1("CHARACTER");
@@ -6803,7 +6807,20 @@ const NAType *LOBupdate::synthesizeType()
(obj_ == FILE_) ||
(obj_ == EXTERNAL_))
{
- if (typ1->getTypeQualifier() != NA_CHARACTER_TYPE)
+ if ((obj_ == STRING_)&& (typ1 && typ1->getTypeQualifier() == NA_UNKNOWN_TYPE))
+ {
+ //Check if it's a special case of untyped param.
+ //If so ,assume it's a varchar format.
+ if (!lobAsVarchar())
+ {
+ // 4221 The operand of a $0~String0 function must be character.
+ *CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBUPDATE")
+ << DgString1("CHARACTER");
+ return NULL;
+ }
+
+ }
+ else if (typ1->getTypeQualifier() != NA_CHARACTER_TYPE)
{
// 4221 The operand of a $0~String0 function must be character.
*CmpCommon::diags() << DgSqlCode(-4221) << DgString0("LOBUPDATE")