You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2017/01/24 23:47:01 UTC
[1/6] incubator-trafodion git commit: [TRAFODION-2440] Replace fixed
length buffers with strings in LOB code
Repository: incubator-trafodion
Updated Branches:
refs/heads/master a89a6585a -> 660c80e33
[TRAFODION-2440] Replace fixed length buffers with strings in LOB code
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/49498dfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/49498dfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/49498dfd
Branch: refs/heads/master
Commit: 49498dfdf6b629bf349f9630fdb5ceb197e93bf9
Parents: 8c7ff9a
Author: Dave Birdsall <db...@apache.org>
Authored: Tue Jan 17 17:07:11 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Tue Jan 17 17:07:11 2017 +0000
----------------------------------------------------------------------
core/sql/exp/ExpLOBaccess.cpp | 219 ++++++++++++++++++++-----------------
core/sql/exp/ExpLOBaccess.h | 29 ++---
2 files changed, 137 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/49498dfd/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index aecf153..e5bd380 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -84,8 +84,7 @@ ExLob::ExLob() :
openFlags_(0),
lobTrace_(FALSE)
{
- memset(lobDataFile_,'\0',sizeof(lobDataFile_));
-
+ // nothing else to do
}
ExLob::~ExLob()
@@ -98,7 +97,7 @@ ExLob::~ExLob()
}
-Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
+Ex_Lob_Error ExLob::initialize(const char *lobFile, Ex_Lob_Mode mode,
char *lobStorageLocation,
LobsStorage storage,
char *hdfsServer, Int64 hdfsPort,
@@ -119,15 +118,16 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
}
if (lobFile)
- snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s/%s",
- lobStorageLocation_.c_str(),
- lobFile);
-
+ {
+ lobDataFile_ = lobStorageLocation;
+ lobDataFile_ += "/";
+ lobDataFile_ += lobFile;
+ }
}
else
{
if (lobFile)
- snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s", lobFile);
+ lobDataFile_ = lobFile;
}
@@ -167,14 +167,14 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
if (mode == EX_LOB_CREATE)
{
// check if file is already created
- hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
if (fInfo != NULL)
{
hdfsFreeFileInfo(fInfo, 1);
return LOB_DATA_FILE_CREATE_ERROR;
}
openFlags = O_WRONLY | O_CREAT;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags, bufferSize, replication, blockSize);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags, bufferSize, replication, blockSize);
if (!fdData_)
{
return LOB_DATA_FILE_CREATE_ERROR;
@@ -198,7 +198,7 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff
lobCursors_it it = lobCursors_.find(string(handleIn, handleLenIn));
char logBuf[4096];
lobDebugInfo("In ExLob::fetchCursor",0,__LINE__,lobTrace_);
- char *blackBox = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
+ char *blackBox = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; // TODO: do we ever go this way for Hive?
Int32 blackBoxLen = 0;
if (it == lobCursors_.end())
@@ -251,8 +251,10 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff
{
// we have received the external data file name from the descriptor table
// replace the contents of the lobDataFile with this name
- str_cpy_and_null(lobDataFile_, blackBox, blackBoxLen,'\0','0',TRUE);
-
+ lobDataFile_.assign(blackBox,0,blackBoxLen); // copy blackBox safely (up to blackBoxLen chars)
+ size_t found = lobDataFile_.find_last_not_of("0");
+ if (found != string::npos)
+ lobDataFile_.erase(found+1); // trim off any trailing '0'
}
outOffset = offset;
outSize = size;
@@ -304,7 +306,7 @@ Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operL
if (!fdData_ || (openFlags_ != (O_WRONLY | O_APPEND))) // file is not open for write
{
// get file info
- hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
if (fInfo == NULL) {
return LOB_DATA_FILE_NOT_FOUND_ERROR;
}
@@ -312,7 +314,7 @@ Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operL
hdfsCloseFile(fs_, fdData_);
fdData_=NULL;
openFlags_ = O_WRONLY | O_APPEND;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
if (!fdData_) {
openFlags_ = -1;
return LOB_DATA_FILE_OPEN_ERROR;
@@ -338,7 +340,7 @@ Ex_Lob_Error ExLob::writeDataSimple(char *data, Int64 size, LobsSubOper subOpera
if (!fdData_ || (openFlags_ != (O_WRONLY | O_APPEND))) // file is not open for write
{
// get file info
- hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
if (fInfo == NULL) {
return LOB_DATA_FILE_NOT_FOUND_ERROR;
} else {
@@ -351,7 +353,7 @@ Ex_Lob_Error ExLob::writeDataSimple(char *data, Int64 size, LobsSubOper subOpera
hdfsCloseFile(fs_, fdData_);
fdData_=NULL;
openFlags_ = O_WRONLY | O_APPEND ;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, bufferSize, replication, blockSize);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, bufferSize, replication, blockSize);
if (!fdData_) {
openFlags_ = -1;
return LOB_DATA_FILE_OPEN_ERROR;
@@ -976,8 +978,10 @@ Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size, Int64 &operLen,char *
// we have received the external data file name from the descriptor table
// replace the contents of the lobDataFile with this name
- str_cpy_and_null(lobDataFile_, blackBox, blackBoxLen,'\0','0',TRUE);
-
+ lobDataFile_.assign(blackBox,0,blackBoxLen); // copy blackBox safely (up to blackBoxLen chars)
+ size_t found = lobDataFile_.find_last_not_of("0");
+ if (found != string::npos)
+ lobDataFile_.erase(found+1); // trim off any trailing '0'
}
if (blackBoxLen == -1)
{
@@ -1048,12 +1052,14 @@ Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 tgtLength, Int64 &operLe
}
if (blackBox)
- {
- // we have received the external data file name from the descriptor table
- // replace the contents of the lobDataFile with this name
- str_cpy_and_null(lobDataFile_, blackBox, blackBoxLen,'\0','0',TRUE);
-
- }
+ {
+ // we have received the external data file name from the descriptor table
+ // replace the contents of the lobDataFile with this name
+ lobDataFile_.assign(blackBox,0,blackBoxLen); // copy blackBox safely (up to blackBoxLen chars)
+ size_t found = lobDataFile_.find_last_not_of("0");
+ if (found != string::npos)
+ lobDataFile_.erase(found+1); // trim off any trailing '0'
+ }
if (tgtType == HDFS_FILE)
{
err = readDataToHdfsFile(tgtFileName, srcOffset , tgtLength,operLen, lobMaxChunkMemLen, fileflags,handleIn,handleInLen,multipleChunks,transId);
@@ -1281,10 +1287,11 @@ Ex_Lob_Error ExLob::delDesc(char *handleIn, Int32 handleInLen, Int64 transId)
Ex_Lob_Error ExLob::purgeLob()
{
char logBuf[4096];
- if (hdfsDelete(fs_, lobDataFile_, 0) != 0)
+ if (hdfsDelete(fs_, lobDataFile_.c_str(), 0) != 0)
{
- str_sprintf(logBuf,"hdfsDelete of %s returned error",lobDataFile_);
- lobDebugInfo(lobDataFile_,0,__LINE__,lobTrace_);
+ string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit
+ str_sprintf(logBuf,"hdfsDelete of %s returned error",lobDataFileSubstr.c_str());
+ lobDebugInfo("In ExLob::purgeLob",0,__LINE__,lobTrace_);
return LOB_DATA_FILE_DELETE_ERROR;
}
@@ -1340,7 +1347,7 @@ Ex_Lob_Error ExLob::openCursor(char *handleIn, Int32 handleInLen,Int64 transId)
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type,
+Ex_Lob_Error ExLob::openDataCursor(const char *file, LobsCursorType type,
Int64 range, Int64 bufMaxSize,
Int64 maxBytes, Int64 waited,
ExLobGlobals *lobGlobals,
@@ -1383,7 +1390,7 @@ Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type,
cursor.prefetch_ = !waited;
cursor.bufferHits_ = 0;
cursor.bufferMisses_ = 0;
- strcpy(cursor.name_, file);
+ cursor.name_ = file;
cursor.currentRange_ = -1;
cursor.endRange_ = -1;
@@ -1405,7 +1412,7 @@ Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type,
hdfsCloseFile(fs_, fdData_);
fdData_ = NULL;
openFlags_ = O_RDONLY;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
if (!fdData_)
{
@@ -1501,31 +1508,28 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset,
if (size == 0) //we are trying to empty this lob.
{
//rename lob datafile
- char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
- str_sprintf(saveLobDataFile, "%s_save",lobDataFile_);
- Int32 rc2 = hdfsRename(fs_,lobDataFile_,saveLobDataFile);
+ string saveLobDataFile = lobDataFile_ + "_save";
+ Int32 rc2 = hdfsRename(fs_,lobDataFile_.c_str(),saveLobDataFile.c_str());
if (rc2 == -1)
{
lobDebugInfo("Problem renaming datafile to save data file",0,__LINE__,lobTrace_);
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
return LOB_DATA_FILE_WRITE_ERROR;
}
//create a new file of the same name.
- hdfsFile fdNew = hdfsOpenFile(fs_, lobDataFile_,O_WRONLY|O_CREAT,0,0,0);
+ hdfsFile fdNew = hdfsOpenFile(fs_, lobDataFile_.c_str(),O_WRONLY|O_CREAT,0,0,0);
if (!fdNew)
{
- str_sprintf(logBuf,"Could not create/open file:%s",lobDataFile_);
+ string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit
+ str_sprintf(logBuf,"Could not create/open file:%s",lobDataFileSubstr.c_str());
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
//restore previous version
- Int32 rc2 = hdfsRename(fs_,saveLobDataFile,lobDataFile_);
+ Int32 rc2 = hdfsRename(fs_,saveLobDataFile.c_str(),lobDataFile_.c_str());
if (rc2 == -1)
{
lobDebugInfo("Problem restoring datafile . Will need to retry the update",0,__LINE__,lobTrace_);
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
return LOB_DATA_FILE_WRITE_ERROR;
}
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
return LOB_DATA_FILE_OPEN_ERROR;
}
@@ -1533,17 +1537,16 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset,
{
//A new empty data file has been created.
// delete the saved data file
- Int32 rc2 = hdfsDelete(fs_,saveLobDataFile,FALSE);//ok to ignore error.nt32
+ Int32 rc2 = hdfsDelete(fs_,saveLobDataFile.c_str(),FALSE);//ok to ignore error.nt32
if (rc2 == -1)
{
lobDebugInfo("Problem deleting saved datafile . Will need to manually cleanup saved datafile",0,__LINE__,lobTrace_);
}
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
hdfsCloseFile(fs_,fdNew);
fdNew = NULL;
}
}
- hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
if (fInfo)
dataOffset = fInfo->mSize;
@@ -1571,17 +1574,19 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset,
if (GCDone) // recalculate the new offset
{
hdfsFreeFileInfo(fInfo, 1);
- fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
+ fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
}
if (fInfo)
dataOffset = fInfo->mSize;
+
+ string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 70)); // make small enough to fit below
if (GCDone)
str_sprintf(logBuf,"Done GC. Allocating new Offset %Ld in %s",
- dataOffset,lobDataFile_);
+ dataOffset,lobDataFileSubstr.c_str());
else
str_sprintf(logBuf,"Allocating new Offset %Ld in %s ",
- dataOffset,lobDataFile_);
+ dataOffset,lobDataFileSubstr.c_str());
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
//Find the last offset in the file
// dataOffset = hdfsTell(fs_,fdData_); //commenting out.hdfsTell always returns 0 !!
@@ -1594,12 +1599,14 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
char logBuf[4096];
lobDebugInfo("In ExLob::compactLobDataFile",0,__LINE__,lobTrace_);
Int64 maxMemChunk = 1024*1024*1024; //1GB limit for intermediate buffer for transfering data
- char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
- str_sprintf(saveLobDataFile, "%s_save",lobDataFile_);
- char * tmpLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+5];
- str_sprintf(tmpLobDataFile, "%s_tmp",lobDataFile_);
+ string saveLobDataFile = lobDataFile_ + "_save";
+ string tmpLobDataFile = lobDataFile_ + "_tmp";
- str_sprintf(logBuf,"DataFile %s, TempDataFile : %s, SaveDataFile : %s ",lobDataFile_,tmpLobDataFile, saveLobDataFile);
+ string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf)/3 - 20); // make small enough to fit
+ string tmpLobDataFileSubstr = tmpLobDataFile.substr(0,sizeof(logBuf)/3 - 20);
+ string saveLobDataFileSubstr = saveLobDataFile.substr(0,sizeof(logBuf)/3 - 20);
+ str_sprintf(logBuf,"DataFile %s, TempDataFile : %s, SaveDataFile : %s ",
+ lobDataFileSubstr.c_str(),tmpLobDataFileSubstr.c_str(), saveLobDataFileSubstr.c_str());
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_);
@@ -1607,11 +1614,12 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
return LOB_DATA_FILE_OPEN_ERROR;
- hdfsFile fdData = hdfsOpenFile(fs, lobDataFile_, O_RDONLY, 0, 0,0);
+ hdfsFile fdData = hdfsOpenFile(fs, lobDataFile_.c_str(), O_RDONLY, 0, 0,0);
if (!fdData)
{
- str_sprintf(logBuf,"Could not open file:%s",lobDataFile_);
+ string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit
+ str_sprintf(logBuf,"Could not open file:%s",lobDataFileSubstr.c_str());
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
hdfsCloseFile(fs,fdData);
fdData = NULL;
@@ -1619,10 +1627,11 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
}
- hdfsFile fdTemp = hdfsOpenFile(fs, tmpLobDataFile,O_WRONLY|O_CREAT,0,0,0);
+ hdfsFile fdTemp = hdfsOpenFile(fs, tmpLobDataFile.c_str(),O_WRONLY|O_CREAT,0,0,0);
if (!fdTemp)
{
- str_sprintf(logBuf,"Could not open file:%s",tmpLobDataFile);
+ string tmpLobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit
+ str_sprintf(logBuf,"Could not open file:%s",tmpLobDataFileSubstr.c_str());
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
hdfsCloseFile(fs,fdTemp);
fdTemp = NULL;
@@ -1691,24 +1700,18 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
//Now save the data file and rename the tempfile to the original datafile
- Int32 rc2 = hdfsRename(fs,lobDataFile_,saveLobDataFile);
+ Int32 rc2 = hdfsRename(fs,lobDataFile_.c_str(),saveLobDataFile.c_str());
if (rc2 == -1)
{
lobDebugInfo("Problem renaming datafile to save data file",0,__LINE__,lobTrace_);
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
- NADELETEBASIC(tmpLobDataFile,getLobGlobalHeap());
return LOB_DATA_FILE_WRITE_ERROR;
}
- rc2 = hdfsRename(fs,tmpLobDataFile, lobDataFile_);
+ rc2 = hdfsRename(fs,tmpLobDataFile.c_str(), lobDataFile_.c_str());
if (rc2 == -1)
{
lobDebugInfo("Problem renaming temp datafile to data file",0,__LINE__,lobTrace_);
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
- NADELETEBASIC(tmpLobDataFile,getLobGlobalHeap());
return LOB_DATA_FILE_WRITE_ERROR;
}
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
- NADELETEBASIC(tmpLobDataFile,getLobGlobalHeap());
return LOB_OPER_OK;
}
@@ -1720,18 +1723,15 @@ Ex_Lob_Error ExLob::restoreLobDataFile()
hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_);
if (fs == NULL)
return LOB_DATA_FILE_OPEN_ERROR;
- char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
- str_sprintf(saveLobDataFile, "%s_save",lobDataFile_);
- Int32 rc2 = hdfsDelete(fs,lobDataFile_,FALSE);//ok to ignore error.
- rc2 = hdfsRename(fs,saveLobDataFile, lobDataFile_);
- if (rc2)
+ string saveLobDataFile = lobDataFile_ + "_save";
+ Int32 rc2 = hdfsDelete(fs,lobDataFile_.c_str(),FALSE);//ok to ignore error.
+ rc2 = hdfsRename(fs,saveLobDataFile.c_str(), lobDataFile_.c_str());
+ if (rc2)
{
lobDebugInfo("Problem renaming savedatafile to data file",0,__LINE__,lobTrace_);
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
return LOB_OPER_ERROR;
}
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
- return rc;
+ return rc;
}
@@ -1742,12 +1742,10 @@ Ex_Lob_Error ExLob::purgeBackupLobDataFile()
hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_);
if (fs == NULL)
return LOB_DATA_FILE_OPEN_ERROR;
- char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
- str_sprintf(saveLobDataFile, "%s_save",lobDataFile_);
- Int32 rc2 = hdfsDelete(fs,saveLobDataFile,FALSE);//ok to ignore error.
+ string saveLobDataFile = lobDataFile_ + "_save";
+ Int32 rc2 = hdfsDelete(fs,saveLobDataFile.c_str(),FALSE);//ok to ignore error.
- NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
- return rc;
+ return rc;
}
///////////////////////////////////////////////////////////////////////////////
// ExLobDescHeader definitions
@@ -1832,7 +1830,7 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
hdfsCloseFile(fs_, fdData_);
fdData_=NULL;
openFlags_ = O_RDONLY;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
if (!fdData_)
{
@@ -1907,7 +1905,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
hdfsCloseFile(fs_, fdData_);
fdData_=NULL;
openFlags_ = O_RDONLY;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
if (!fdData_)
{
@@ -1919,7 +1917,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
}
else
{
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
if (!fdData_)
{
@@ -1940,7 +1938,10 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
return LOB_DATA_READ_ERROR;
}
- str_sprintf(logBuf,"After hdfsPread: File:%s, Offset:%Ld, Size:%Ld,Target Mem Addr:%Ld",lobDataFile_,offset,size,memAddr);
+
+ string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 100)); // make small enough to fit
+ str_sprintf(logBuf,"After hdfsPread: File:%s, Offset:%Ld, Size:%Ld,Target Mem Addr:%Ld",
+ lobDataFileSubstr.c_str(),offset,size,memAddr);
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
operLen = bytesRead;
return LOB_OPER_OK;
@@ -2285,7 +2286,7 @@ Ex_Lob_Error ExLobsOper (
{
Ex_Lob_Error err = LOB_OPER_OK;
ExLob *lobPtr = NULL;
- char fn[MAX_LOB_FILE_NAME_LEN];
+ string fn;
struct timespec startTime;
struct timespec endTime;
Int64 secs, nsecs, totalnsecs;
@@ -2300,7 +2301,7 @@ Ex_Lob_Error ExLobsOper (
clock_gettime(CLOCK_MONOTONIC, &startTime);
- char *fileName = lobName;
+ const char *fileName = lobName;
if (globPtr == NULL)
{
@@ -2405,7 +2406,8 @@ Ex_Lob_Error ExLobsOper (
case Lob_Read:
if (storage == Lob_External_HDFS_File)
//Allocate storage to read the lob external file name from the
- //descriptor tables to get the data from.
+ //descriptor tables to get the data from.
+ // TODO: do we ever take this code path for Hive files?
blackBox = new(lobGlobals->getHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
if (subOperation == Lob_Memory)
@@ -2440,12 +2442,21 @@ Ex_Lob_Error ExLobsOper (
case Lob_OpenDataCursorSimple:
if (openType == 1) { // preopen
- sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
- preOpenObj = new (lobGlobals->getHeap()) ExLobPreOpen(lobPtr, fn, descNumIn, sourceLen, cursorBytes, waited);
+ char temp1[30]; // big enough for :%Lx:
+ sprintf(temp1, ":%Lx:",(long long unsigned int)lobName);
+ fn = lobPtr->getDataFileName();
+ fn += temp1;
+ fn += cursorId;
+ preOpenObj = new (lobGlobals->getHeap()) ExLobPreOpen(lobPtr, fn.c_str(), descNumIn, sourceLen,
+ cursorBytes, waited, lobGlobals->getHeap());
lobGlobals->addToPreOpenList(preOpenObj);
} else if (openType == 2) { // must open
- sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
- fileName = fn;
+ char temp2[30]; // big enough for :%Lx:
+ sprintf(temp2, ":%Lx:",(long long unsigned int)lobName);
+ fn = lobPtr->getDataFileName();
+ fn += temp2;
+ fn += cursorId;
+ fileName = fn.c_str();
err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals, (Int32 *)blackBox);
} else
err = LOB_SUBOPER_ERROR;
@@ -2461,9 +2472,15 @@ Ex_Lob_Error ExLobsOper (
break;
case Lob_ReadDataCursorSimple:
- sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
- fileName = fn;
- err = lobPtr->readDataCursorSimple(fileName, source, sourceLen, retOperLen, lobGlobals);
+ {
+ char temp3[30]; // big enough for :%Lx:
+ sprintf(temp3, ":%Lx:",(long long unsigned int)lobName);
+ fn = lobPtr->getDataFileName();
+ fn += temp3;
+ fn += cursorId;
+ fileName = fn.c_str();
+ err = lobPtr->readDataCursorSimple(fileName, source, sourceLen, retOperLen, lobGlobals);
+ }
break;
case Lob_CloseFile:
@@ -2482,9 +2499,15 @@ Ex_Lob_Error ExLobsOper (
break;
case Lob_CloseDataCursorSimple:
- sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
- fileName = fn;
- err = lobPtr->closeDataCursorSimple(fileName, lobGlobals);
+ {
+ char temp4[30]; // big enough for :%Lx:
+ sprintf(temp4, ":%Lx:",(long long unsigned int)lobName);
+ fn = lobPtr->getDataFileName();
+ fn += temp4;
+ fn += cursorId;
+ fileName = fn.c_str();
+ err = lobPtr->closeDataCursorSimple(fileName, lobGlobals);
+ }
break;
case Lob_Append:
@@ -2684,7 +2707,7 @@ prefetchBufList. It continues doing this until end of range is reached or the
buffer limit (128MB) has been reached.
*/
-Ex_Lob_Error ExLob::readDataCursorSimple(char *file, char *tgt, Int64 tgtSize,
+Ex_Lob_Error ExLob::readDataCursorSimple(const char *file, char *tgt, Int64 tgtSize,
Int64 &operLen, ExLobGlobals *lobGlobals)
{
int dataOffset;
@@ -2775,7 +2798,7 @@ Ex_Lob_Error ExLob::readDataCursorSimple(char *file, char *tgt, Int64 tgtSize,
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::closeDataCursorSimple(char *fileName, ExLobGlobals *lobGlobals)
+Ex_Lob_Error ExLob::closeDataCursorSimple(const char *fileName, ExLobGlobals *lobGlobals)
{
cursor_t *cursor = NULL;
Int64 secs = 0;
@@ -2924,7 +2947,7 @@ Ex_Lob_Error ExLobGlobals::performRequest(ExLobHdfsRequest *request)
traceMessage("locking cursor",cursor,__LINE__);
cursor->lock_.unlock();
if (cursor->eol_) { // never reaches here ??
- lobPtr->deleteCursor(cursor->name_, this);
+ lobPtr->deleteCursor(cursor->name_.c_str(), this);
}
}
processPreOpens();
@@ -3018,7 +3041,7 @@ void ExLobCursor::emptyPrefetchList(ExLobGlobals *lobGlobals)
// Seems like this is currently unused.
// closeDataCusrorSimple takes care of destroying the cursor.But addign code
// similar to closeDataCursorSimple for correctness in case it is used in future
-Ex_Lob_Error ExLob::deleteCursor(char *cursorName, ExLobGlobals *lobGlobals)
+Ex_Lob_Error ExLob::deleteCursor(const char *cursorName, ExLobGlobals *lobGlobals)
{
cursor_t *cursor = NULL;
@@ -3389,7 +3412,7 @@ Ex_Lob_Error ExLobGlobals::processPreOpens()
{
ExLob *lobPtr = preOpenObj->lobPtr_;
- lobPtr->openDataCursor(preOpenObj->cursorName_, Lob_Cursor_Simple, preOpenObj->range_,
+ lobPtr->openDataCursor(preOpenObj->cursorName_.data(), Lob_Cursor_Simple, preOpenObj->range_,
preOpenObj->bufMaxSize_, preOpenObj->maxBytes_,
preOpenObj->waited_, this,0);
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/49498dfd/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index a8e0ea7..a793487 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -147,7 +147,7 @@ class ExLobRequest
Int64 transId_;
SB_Transid_Type transIdBig_;
SB_Transseq_Type transStartId_;
- char descFileName_[MAX_LOB_FILE_NAME_LEN];
+ char descFileName_[MAX_LOB_FILE_NAME_LEN]; // TODO: If we ever use this again, change to use string or NAString
char blackBox_[MAX_BLACK_BOX_LEN];
Int64 blackBoxLen_;
};
@@ -368,7 +368,7 @@ public:
Int64 bufMaxSize_; // max size of buffer
Int64 maxBytes_; // bytesLeft to prefetch
Int64 prefetch_; // prefetch or not to prefetch
- char name_[MAX_LOB_FILE_NAME_LEN];
+ string name_; // TODO: change to NAString when ExLobCursor is allocated off of an NAHeap
Lng32 currentRange_; // current index of file for multi cursor
Lng32 endRange_; // end index of file for multi cursor
@@ -406,13 +406,13 @@ class ExLob
ExLob(); // default constructor
~ExLob(); // default desctructor
- Ex_Lob_Error initialize(char *lobFile, Ex_Lob_Mode mode, char *dir,
+ Ex_Lob_Error initialize(const char *lobFile, Ex_Lob_Mode mode, char *dir,
LobsStorage storage, char *hdfsServer, Int64 hdfsPort,
char *lobLocation,
int bufferSize = 0, short replication =0,
int blocksize=0, Int64 lobMaxSize = 0,
ExLobGlobals *lobGlobals = NULL);
- Ex_Lob_Error initialize(char *lobFile);
+
Ex_Lob_Error writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOperation, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,Int64 lobGCLimit, char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
Ex_Lob_Error writeLobData(char *source, Int64 sourceLen,
LobsSubOper subOperation,
@@ -425,12 +425,12 @@ class ExLob
Ex_Lob_Error readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int32 handleInLen, Int64 &operLen,Int64 transId);
Ex_Lob_Error readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen,char *handleIn, Int32 handeLenIn,Int64 transId);
Ex_Lob_Error readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen);
- Ex_Lob_Error readDataCursorSimple(char *fileName, char *tgt, Int64 tgtSize, Int64 &operLen, ExLobGlobals *lobGlobals);
+ Ex_Lob_Error readDataCursorSimple(const char *fileName, char *tgt, Int64 tgtSize, Int64 &operLen, ExLobGlobals *lobGlobals);
bool hasNoOpenCursors() { return lobCursors_.empty(); }
Ex_Lob_Error openCursor(char *handleIn, Int32 handleInLen,Int64 transId);
- Ex_Lob_Error openDataCursor(char *fileName, LobsCursorType type, Int64 range,
+ Ex_Lob_Error openDataCursor(const char *fileName, LobsCursorType type, Int64 range,
Int64 bytesLeft, Int64 bufMaxSize, Int64 prefetch, ExLobGlobals *lobGlobals, Int32 *hdfsDetailError = NULL);
- Ex_Lob_Error deleteCursor(char *cursorName, ExLobGlobals *lobGlobals);
+ Ex_Lob_Error deleteCursor(const char *cursorName, ExLobGlobals *lobGlobals);
Ex_Lob_Error fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOffset, Int64 &outSize,NABoolean &isEOD,Int64 transId);
Ex_Lob_Error insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,char *handleIn,Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
Ex_Lob_Error append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemLen,Int64 lobGCLimit, char *handleIn,Int32 handleInLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
@@ -445,7 +445,7 @@ class ExLob
Ex_Lob_Error closeFile();
LobInputOutputFileType fileType(char *ioFileName);
Ex_Lob_Error closeCursor(char *handleIn, Int32 handleInLen);
- Ex_Lob_Error closeDataCursorSimple(char *fileName, ExLobGlobals *lobGlobals);
+ Ex_Lob_Error closeDataCursorSimple(const char *fileName, ExLobGlobals *lobGlobals);
Ex_Lob_Error doSanityChecks(char *dir, LobsStorage storage,
Int32 handleInLen, Int32 handleOutLen,
@@ -458,7 +458,7 @@ class ExLob
Ex_Lob_Error lockDesc();
Ex_Lob_Error unlockDesc();
- char *getDataFileName() { return lobDataFile_; }
+ const char *getDataFileName() { return lobDataFile_.c_str(); }
int getErrNo();
@@ -515,7 +515,7 @@ class ExLob
public:
- char lobDataFile_[MAX_LOB_FILE_NAME_LEN];
+ string lobDataFile_; // TODO: change to NAString when ExLobCursor is allocated off of an NAHeap
lobCursors_t lobCursors_;
ExLobLock lobCursorLock_;
LobsStorage storage_;
@@ -562,21 +562,24 @@ class ExLobPreOpen
{
public :
- ExLobPreOpen(ExLob *lobPtr, char *cursorName, Int64 range, Int64 bufMaxSize, Int64 maxBytes, Int64 waited)
+ ExLobPreOpen(ExLob *lobPtr, const char *cursorName, Int64 range,
+ Int64 bufMaxSize, Int64 maxBytes, Int64 waited,
+ NAHeap * heap)
: lobPtr_(lobPtr),
+ cursorName_(cursorName,heap),
range_(range),
bufMaxSize_(bufMaxSize),
maxBytes_(maxBytes),
waited_(waited)
{
- strcpy(cursorName_, cursorName);
+ // nothing else to do
}
~ExLobPreOpen();
public :
ExLob *lobPtr_;
- char cursorName_[MAX_LOB_FILE_NAME_LEN + 16];
+ NAString cursorName_;
Int64 range_;
Int64 bufMaxSize_;
Int64 maxBytes_;
[6/6] incubator-trafodion git commit: Merge [TRAFODION-2440] PR 915
Replace fixed len buffs with strings in LOB code
Posted by db...@apache.org.
Merge [TRAFODION-2440] PR 915 Replace fixed len buffs with strings in LOB code
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/660c80e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/660c80e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/660c80e3
Branch: refs/heads/master
Commit: 660c80e33581488753205ad17831e6731b62f078
Parents: a89a658 729472c
Author: Dave Birdsall <db...@apache.org>
Authored: Tue Jan 24 23:45:28 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Tue Jan 24 23:45:28 2017 +0000
----------------------------------------------------------------------
core/sql/bin/SqlciErrors.txt | 3 +-
core/sql/exp/ExpLOB.cpp | 10 +
core/sql/exp/ExpLOBaccess.cpp | 305 ++++++++++++-------
core/sql/exp/ExpLOBaccess.h | 42 +--
core/sql/exp/ExpLOBprocess.cpp | 2 +-
core/sql/regress/executor/EXPECTED130 | 97 +++---
core/sql/regress/executor/TEST130 | 6 +
.../src/asciidoc/_chapters/executor_msgs.adoc | 15 +
8 files changed, 305 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/660c80e3/core/sql/bin/SqlciErrors.txt
----------------------------------------------------------------------
[3/6] incubator-trafodion git commit: Second rework
Posted by db...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/729472c2/core/sql/regress/executor/TEST130
----------------------------------------------------------------------
diff --git a/core/sql/regress/executor/TEST130 b/core/sql/regress/executor/TEST130
index 208ca5f..cb7557f 100755
--- a/core/sql/regress/executor/TEST130
+++ b/core/sql/regress/executor/TEST130
@@ -296,6 +296,8 @@ sh regrhadoop.ksh fs -copyFromLocal lob_input_d1.txt /lobs/lob_input_d1.txt;
sh regrhadoop.ksh fs -copyFromLocal lob_input_e1.txt /lobs/lob_input_e1.txt;
sh regrhadoop.ksh fs -copyFromLocal deep.jpg /lobs/deep.jpg;
sh regrhadoop.ksh fs -copyFromLocal anoush.jpg /lobs/anoush.jpg;
+-- the next one is a really long file name intended to test error message 8557
+sh regrhadoop.ksh fs -copyFromLocal lob_input_a1.txt /lobs/reallyLongDirectoryName0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789/lob_input_a1012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789.txt;
sh sleep(20);
@@ -499,6 +501,9 @@ update tlob130ext set c4=stringtolob('cannot allow this') where c1=1;
delete from tlob130ext where c1=1;
insert into tlob130ext values(1, stringtolob('first lob'),externaltolob('hdfs:///lobs/lob_input_a1.txt'),externaltolob('hdfs:///lobs/lob_input_a1.txt'));
insert into tlob130ext values(1, stringtolob('first lob'), filetolob('hdfs:///lobs/lob_input_a1.txt'),externaltolob('hdfs:///lobs/lob_input_a1.txt'));
+-- the next one should see error 8557
+insert into tlob130ext values(1, stringtolob('first lob'), filetolob('hdfs:///lobs/lob_input_a1.txt'),
+externaltolob('hdfs:///lobs/reallyLongDirectoryName0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789/lob_input_a1012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789.txt'));
select lobtostring(c2,50),lobtostring(c3,50),lobtostring(c4,50) from tlob130ext;
update tlob130ext set c3=stringtolob('can allow this') where c1=1;
@@ -576,6 +581,7 @@ sh regrhadoop.ksh fs -rm /lobs/lobinput_b1.txt
sh regrhadoop.ksh fs -rm /lobs/lobinput_c1.txt
sh regrhadoop.ksh fs -rm /lobs/lobinput_d1.txt
sh regrhadoop.ksh fs -rm /lobs/lobinput_e1.txt
+sh regrhadoop.ksh fs -rm /lobs/reallyLongDirectoryName0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789/lob_input_a1012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789.txt;
sh rm lobinput_*;
sh rm deep.jpg;
sh rm anoush.jpg;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/729472c2/docs/messages_guide/src/asciidoc/_chapters/executor_msgs.adoc
----------------------------------------------------------------------
diff --git a/docs/messages_guide/src/asciidoc/_chapters/executor_msgs.adoc b/docs/messages_guide/src/asciidoc/_chapters/executor_msgs.adoc
index 191f033..06403c7 100644
--- a/docs/messages_guide/src/asciidoc/_chapters/executor_msgs.adoc
+++ b/docs/messages_guide/src/asciidoc/_chapters/executor_msgs.adoc
@@ -226,6 +226,21 @@ Stream overflow; subscription rate has fallen too far behind publishing rate.
*Recovery:* Reopen the statement or cursor and resume subscribing.
<<<
+[[SQL-8557]]
+== SQL 8557
+
+```
+The file name passed to externaltolob exceeds 256 bytes.
+```
+
+*Cause:* A DML statement attempted to execute the externaltolob function with a file name exceeding 256 bytes.
+
+*Effect:* The statement fails.
+
+*Recovery:* If the file name is incorrect, correct and rerun. If the file name is correct and is too long, rename the file so that
+it is within the 256 byte limit.
+
+<<<
[[SQL-8573]]
== SQL 8573
[2/6] incubator-trafodion git commit: Reworks
Posted by db...@apache.org.
Reworks
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/237f9e8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/237f9e8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/237f9e8e
Branch: refs/heads/master
Commit: 237f9e8e28cba752068756e6a4f0fe80408d334e
Parents: 49498df
Author: Dave Birdsall <db...@apache.org>
Authored: Thu Jan 19 23:48:14 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Thu Jan 19 23:48:14 2017 +0000
----------------------------------------------------------------------
core/sql/bin/SqlciErrors.txt | 3 +-
core/sql/exp/ExpLOB.cpp | 10 ++
core/sql/exp/ExpLOBaccess.cpp | 186 +++++++++++++++++++++++-------------
core/sql/exp/ExpLOBaccess.h | 17 ++--
core/sql/exp/ExpLOBprocess.cpp | 2 +-
5 files changed, 142 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/237f9e8e/core/sql/bin/SqlciErrors.txt
----------------------------------------------------------------------
diff --git a/core/sql/bin/SqlciErrors.txt b/core/sql/bin/SqlciErrors.txt
index a7f9bf9..d5110cb 100644
--- a/core/sql/bin/SqlciErrors.txt
+++ b/core/sql/bin/SqlciErrors.txt
@@ -1591,7 +1591,8 @@ $1~String1 --------------------------------
8553 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Stream overflow; subscription rate has fallen too far behind publishing rate.
8554 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU -- unused, although the error number reservation doc says it is used ---
8555 ZZZZZ 99999 ADVANCED CRTCL DIALOUT An internal error occurred in the SQL executor in the disk process.
-8556 ZZZZZ 99999 BEGINNER MAJOR DBADMIN An error occurred while accessing HBase table $0~string0. $1~string1
+8556 ZZZZZ 99999 BEGINNER MAJOR DBADMIN An error occurred while accessing HBase table $0~string0. $1~string1
+8557 ZZZZZ 99999 BEGINNER MAJOR DBADMIN The file name passed to externaltolob exceeds 256 bytes.
8570 ZZZZZ 99999 ADVANCED MAJOR DBADMIN SQL could not allocate sufficient memory to build query.
8571 ZZZZZ 99999 ADVANCED MAJOR DBADMIN SQL could not allocate sufficient memory to execute query.
8572 ZZZZZ 99999 ADVANCED CRTCL DIALOUT The statement has incurred a fatal error and must be deallocated.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/237f9e8e/core/sql/exp/ExpLOB.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.cpp b/core/sql/exp/ExpLOB.cpp
index feb9641..66da5fc 100644
--- a/core/sql/exp/ExpLOB.cpp
+++ b/core/sql/exp/ExpLOB.cpp
@@ -775,6 +775,16 @@ ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[],
Int64 lobLen = 0;
if(!fromEmpty())
lobLen = getOperand(1)->getLength();
+
+ // until SQL_EXEC_LOBcliInterface is changed to allow for unlimited
+ // black box sizes, we have to prevent over-sized file names from
+ // being stored
+ if ((so == Lob_External) && (lobLen > MAX_LOB_FILE_NAME_LEN))
+ {
+ ExRaiseSqlError(h, diagsArea,
+ (ExeErrorCode)(8557));
+ return ex_expr::EXPR_ERROR;
+ }
blackBoxLen_ = 0;
if (fromExternal())
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/237f9e8e/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index e5bd380..8c40bf7 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -75,7 +75,8 @@ extern int ms_transid_reinstate(MS_Mon_Transid_Type, MS_Mon_Transseq_Type);
// short LobServerFNum;
SB_Phandle_Type serverPhandle;
-ExLob::ExLob() :
+ExLob::ExLob(NAHeap * heap) :
+ lobDataFile_(heap),
storage_(Lob_Invalid_Storage),
lobStorageLocation_(string()),
lobGlobalHeap_(NULL),
@@ -167,14 +168,14 @@ Ex_Lob_Error ExLob::initialize(const char *lobFile, Ex_Lob_Mode mode,
if (mode == EX_LOB_CREATE)
{
// check if file is already created
- hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.data());
if (fInfo != NULL)
{
hdfsFreeFileInfo(fInfo, 1);
return LOB_DATA_FILE_CREATE_ERROR;
}
openFlags = O_WRONLY | O_CREAT;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags, bufferSize, replication, blockSize);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags, bufferSize, replication, blockSize);
if (!fdData_)
{
return LOB_DATA_FILE_CREATE_ERROR;
@@ -250,11 +251,10 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff
if (blackBox && blackBoxLen >0 )
{
// we have received the external data file name from the descriptor table
- // replace the contents of the lobDataFile with this name
- lobDataFile_.assign(blackBox,0,blackBoxLen); // copy blackBox safely (up to blackBoxLen chars)
- size_t found = lobDataFile_.find_last_not_of("0");
- if (found != string::npos)
- lobDataFile_.erase(found+1); // trim off any trailing '0'
+ // replace the contents of the lobDataFile with this name
+ char temp[blackBoxLen+1];
+ str_cpy_and_null(temp, blackBox, blackBoxLen, '\0', '0', TRUE);
+ lobDataFile_ = temp;
}
outOffset = offset;
outSize = size;
@@ -306,7 +306,7 @@ Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operL
if (!fdData_ || (openFlags_ != (O_WRONLY | O_APPEND))) // file is not open for write
{
// get file info
- hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.data());
if (fInfo == NULL) {
return LOB_DATA_FILE_NOT_FOUND_ERROR;
}
@@ -314,7 +314,7 @@ Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operL
hdfsCloseFile(fs_, fdData_);
fdData_=NULL;
openFlags_ = O_WRONLY | O_APPEND;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, 0, 0, 0);
if (!fdData_) {
openFlags_ = -1;
return LOB_DATA_FILE_OPEN_ERROR;
@@ -340,7 +340,7 @@ Ex_Lob_Error ExLob::writeDataSimple(char *data, Int64 size, LobsSubOper subOpera
if (!fdData_ || (openFlags_ != (O_WRONLY | O_APPEND))) // file is not open for write
{
// get file info
- hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.data());
if (fInfo == NULL) {
return LOB_DATA_FILE_NOT_FOUND_ERROR;
} else {
@@ -353,7 +353,7 @@ Ex_Lob_Error ExLob::writeDataSimple(char *data, Int64 size, LobsSubOper subOpera
hdfsCloseFile(fs_, fdData_);
fdData_=NULL;
openFlags_ = O_WRONLY | O_APPEND ;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, bufferSize, replication, blockSize);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, bufferSize, replication, blockSize);
if (!fdData_) {
openFlags_ = -1;
return LOB_DATA_FILE_OPEN_ERROR;
@@ -978,10 +978,9 @@ Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size, Int64 &operLen,char *
// we have received the external data file name from the descriptor table
// replace the contents of the lobDataFile with this name
- lobDataFile_.assign(blackBox,0,blackBoxLen); // copy blackBox safely (up to blackBoxLen chars)
- size_t found = lobDataFile_.find_last_not_of("0");
- if (found != string::npos)
- lobDataFile_.erase(found+1); // trim off any trailing '0'
+ char temp[blackBoxLen+1];
+ str_cpy_and_null(temp, blackBox, blackBoxLen, '\0', '0', TRUE);
+ lobDataFile_ = temp;
}
if (blackBoxLen == -1)
{
@@ -1055,10 +1054,9 @@ Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 tgtLength, Int64 &operLe
{
// we have received the external data file name from the descriptor table
// replace the contents of the lobDataFile with this name
- lobDataFile_.assign(blackBox,0,blackBoxLen); // copy blackBox safely (up to blackBoxLen chars)
- size_t found = lobDataFile_.find_last_not_of("0");
- if (found != string::npos)
- lobDataFile_.erase(found+1); // trim off any trailing '0'
+ char temp[blackBoxLen+1];
+ str_cpy_and_null(temp, blackBox, blackBoxLen, '\0', '0', TRUE);
+ lobDataFile_ = temp;
}
if (tgtType == HDFS_FILE)
{
@@ -1287,10 +1285,15 @@ Ex_Lob_Error ExLob::delDesc(char *handleIn, Int32 handleInLen, Int64 transId)
Ex_Lob_Error ExLob::purgeLob()
{
char logBuf[4096];
- if (hdfsDelete(fs_, lobDataFile_.c_str(), 0) != 0)
+ if (hdfsDelete(fs_, lobDataFile_.data(), 0) != 0)
{
- string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit
- str_sprintf(logBuf,"hdfsDelete of %s returned error",lobDataFileSubstr.c_str());
+ // extract a substring small enough to fit into logBuf
+ size_t len = MINOF(lobDataFile_.length(),sizeof(logBuf)-40);
+ char lobDataFileSubstr[len+1]; // +1 for trailing null
+ strncpy(lobDataFileSubstr,lobDataFile_.data(),len);
+ lobDataFileSubstr[len] = '\0';
+
+ str_sprintf(logBuf,"hdfsDelete of %s returned error",lobDataFileSubstr);
lobDebugInfo("In ExLob::purgeLob",0,__LINE__,lobTrace_);
return LOB_DATA_FILE_DELETE_ERROR;
}
@@ -1412,7 +1415,7 @@ Ex_Lob_Error ExLob::openDataCursor(const char *file, LobsCursorType type,
hdfsCloseFile(fs_, fdData_);
fdData_ = NULL;
openFlags_ = O_RDONLY;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, 0, 0, 0);
if (!fdData_)
{
@@ -1508,23 +1511,30 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset,
if (size == 0) //we are trying to empty this lob.
{
//rename lob datafile
- string saveLobDataFile = lobDataFile_ + "_save";
- Int32 rc2 = hdfsRename(fs_,lobDataFile_.c_str(),saveLobDataFile.c_str());
+ char saveLobDataFile[lobDataFile_.length() + sizeof("_save")]; // sizeof includes room for null terminator
+ strcpy(saveLobDataFile,lobDataFile_.data());
+ strcpy(saveLobDataFile+lobDataFile_.length(),"_save");
+ Int32 rc2 = hdfsRename(fs_,lobDataFile_.data(),saveLobDataFile);
if (rc2 == -1)
{
lobDebugInfo("Problem renaming datafile to save data file",0,__LINE__,lobTrace_);
return LOB_DATA_FILE_WRITE_ERROR;
}
//create a new file of the same name.
- hdfsFile fdNew = hdfsOpenFile(fs_, lobDataFile_.c_str(),O_WRONLY|O_CREAT,0,0,0);
+ hdfsFile fdNew = hdfsOpenFile(fs_, lobDataFile_.data(),O_WRONLY|O_CREAT,0,0,0);
if (!fdNew)
{
- string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit
- str_sprintf(logBuf,"Could not create/open file:%s",lobDataFileSubstr.c_str());
+ // extract a substring small enough to fit into logBuf
+ size_t len = MINOF(lobDataFile_.length(),sizeof(logBuf)-40);
+ char lobDataFileSubstr[len+1]; // +1 for trailing null
+ strncpy(lobDataFileSubstr,lobDataFile_.data(),len);
+ lobDataFileSubstr[len] = '\0';
+
+ str_sprintf(logBuf,"Could not create/open file:%s",lobDataFileSubstr);
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
//restore previous version
- Int32 rc2 = hdfsRename(fs_,saveLobDataFile.c_str(),lobDataFile_.c_str());
+ Int32 rc2 = hdfsRename(fs_,saveLobDataFile,lobDataFile_.data());
if (rc2 == -1)
{
lobDebugInfo("Problem restoring datafile . Will need to retry the update",0,__LINE__,lobTrace_);
@@ -1537,7 +1547,7 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset,
{
//A new empty data file has been created.
// delete the saved data file
- Int32 rc2 = hdfsDelete(fs_,saveLobDataFile.c_str(),FALSE);//ok to ignore error.nt32
+ Int32 rc2 = hdfsDelete(fs_,saveLobDataFile,FALSE);//ok to ignore error.nt32
if (rc2 == -1)
{
lobDebugInfo("Problem deleting saved datafile . Will need to manually cleanup saved datafile",0,__LINE__,lobTrace_);
@@ -1546,7 +1556,7 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset,
fdNew = NULL;
}
}
- hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_.data());
if (fInfo)
dataOffset = fInfo->mSize;
@@ -1574,19 +1584,24 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset,
if (GCDone) // recalculate the new offset
{
hdfsFreeFileInfo(fInfo, 1);
- fInfo = hdfsGetPathInfo(fs_, lobDataFile_.c_str());
+ fInfo = hdfsGetPathInfo(fs_, lobDataFile_.data());
}
if (fInfo)
dataOffset = fInfo->mSize;
- string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 70)); // make small enough to fit below
+ // extract a substring small enough to fit into logBuf
+ size_t len = MINOF(lobDataFile_.length(),sizeof(logBuf)-70);
+ char lobDataFileSubstr[len+1]; // +1 for trailing null
+ strncpy(lobDataFileSubstr,lobDataFile_.data(),len);
+ lobDataFileSubstr[len] = '\0';
+
if (GCDone)
str_sprintf(logBuf,"Done GC. Allocating new Offset %Ld in %s",
- dataOffset,lobDataFileSubstr.c_str());
+ dataOffset,lobDataFileSubstr);
else
str_sprintf(logBuf,"Allocating new Offset %Ld in %s ",
- dataOffset,lobDataFileSubstr.c_str());
+ dataOffset,lobDataFileSubstr);
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
//Find the last offset in the file
// dataOffset = hdfsTell(fs_,fdData_); //commenting out.hdfsTell always returns 0 !!
@@ -1599,14 +1614,32 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
char logBuf[4096];
lobDebugInfo("In ExLob::compactLobDataFile",0,__LINE__,lobTrace_);
Int64 maxMemChunk = 1024*1024*1024; //1GB limit for intermediate buffer for transfering data
- string saveLobDataFile = lobDataFile_ + "_save";
- string tmpLobDataFile = lobDataFile_ + "_tmp";
- string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf)/3 - 20); // make small enough to fit
- string tmpLobDataFileSubstr = tmpLobDataFile.substr(0,sizeof(logBuf)/3 - 20);
- string saveLobDataFileSubstr = saveLobDataFile.substr(0,sizeof(logBuf)/3 - 20);
+ // make some temporary file names
+ size_t len = lobDataFile_.length();
+ char saveLobDataFile[len + sizeof("_save")]; // sizeof includes room for null terminator
+ strcpy(saveLobDataFile,lobDataFile_.data());
+ strcpy(saveLobDataFile+len,"_save");
+ char tmpLobDataFile[len + sizeof("_tmp")]; // sizeof includes room for null terminator
+ strcpy(tmpLobDataFile,lobDataFile_.data());
+ strcpy(tmpLobDataFile+len,"_tmp");
+
+ // extract small enough bits of these file names to fit in logBuf
+ len = MINOF(lobDataFile_.length(),sizeof(logBuf)/3 - 20);
+ char lobDataFileSubstr[len + 1];
+ strncpy(lobDataFileSubstr,lobDataFile_.data(),len);
+ lobDataFileSubstr[len] = '\0';
+ len = MINOF(sizeof(tmpLobDataFile),sizeof(logBuf)/3 - 20);
+ char tmpLobDataFileSubstr[len + 1];
+ strncpy(tmpLobDataFileSubstr,tmpLobDataFile,len);
+ tmpLobDataFileSubstr[len] = '\0';
+ len = MINOF(sizeof(saveLobDataFile),sizeof(logBuf)/3 - 20);
+ char saveLobDataFileSubstr[len + 1];
+ strncpy(saveLobDataFileSubstr,saveLobDataFile,len);
+ saveLobDataFileSubstr[len] = '\0';
+
str_sprintf(logBuf,"DataFile %s, TempDataFile : %s, SaveDataFile : %s ",
- lobDataFileSubstr.c_str(),tmpLobDataFileSubstr.c_str(), saveLobDataFileSubstr.c_str());
+ lobDataFileSubstr,tmpLobDataFileSubstr, saveLobDataFileSubstr);
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_);
@@ -1614,12 +1647,17 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
return LOB_DATA_FILE_OPEN_ERROR;
- hdfsFile fdData = hdfsOpenFile(fs, lobDataFile_.c_str(), O_RDONLY, 0, 0,0);
+ hdfsFile fdData = hdfsOpenFile(fs, lobDataFile_.data(), O_RDONLY, 0, 0,0);
if (!fdData)
{
- string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit
- str_sprintf(logBuf,"Could not open file:%s",lobDataFileSubstr.c_str());
+ // extract substring small enough to fit in logBuf
+ len = MINOF(lobDataFile_.length(),sizeof(logBuf) - 40);
+ char lobDataFileSubstr2[len + 1];
+ strncpy(lobDataFileSubstr2,lobDataFile_.data(),len);
+ lobDataFileSubstr2[len] = '\0';
+
+ str_sprintf(logBuf,"Could not open file:%s",lobDataFileSubstr2);
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
hdfsCloseFile(fs,fdData);
fdData = NULL;
@@ -1627,11 +1665,16 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
}
- hdfsFile fdTemp = hdfsOpenFile(fs, tmpLobDataFile.c_str(),O_WRONLY|O_CREAT,0,0,0);
- if (!fdTemp)
+ hdfsFile fdTemp = hdfsOpenFile(fs, tmpLobDataFile,O_WRONLY|O_CREAT,0,0,0);
+ if (!fdTemp)
{
- string tmpLobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 40)); // make small enough to fit
- str_sprintf(logBuf,"Could not open file:%s",tmpLobDataFileSubstr.c_str());
+ // extract substring small enough to fit in logBuf
+ len = MINOF(sizeof(tmpLobDataFile),sizeof(logBuf)/3 - 20);
+ char tmpLobDataFileSubstr2[len + 1];
+ strncpy(tmpLobDataFileSubstr2,tmpLobDataFile,len);
+ tmpLobDataFileSubstr2[len] = '\0';
+
+ str_sprintf(logBuf,"Could not open file:%s",tmpLobDataFileSubstr2);
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
hdfsCloseFile(fs,fdTemp);
fdTemp = NULL;
@@ -1700,13 +1743,13 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
//Now save the data file and rename the tempfile to the original datafile
- Int32 rc2 = hdfsRename(fs,lobDataFile_.c_str(),saveLobDataFile.c_str());
+ Int32 rc2 = hdfsRename(fs,lobDataFile_.data(),saveLobDataFile);
if (rc2 == -1)
{
lobDebugInfo("Problem renaming datafile to save data file",0,__LINE__,lobTrace_);
return LOB_DATA_FILE_WRITE_ERROR;
}
- rc2 = hdfsRename(fs,tmpLobDataFile.c_str(), lobDataFile_.c_str());
+ rc2 = hdfsRename(fs,tmpLobDataFile, lobDataFile_.data());
if (rc2 == -1)
{
lobDebugInfo("Problem renaming temp datafile to data file",0,__LINE__,lobTrace_);
@@ -1723,9 +1766,11 @@ Ex_Lob_Error ExLob::restoreLobDataFile()
hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_);
if (fs == NULL)
return LOB_DATA_FILE_OPEN_ERROR;
- string saveLobDataFile = lobDataFile_ + "_save";
- Int32 rc2 = hdfsDelete(fs,lobDataFile_.c_str(),FALSE);//ok to ignore error.
- rc2 = hdfsRename(fs,saveLobDataFile.c_str(), lobDataFile_.c_str());
+ char saveLobDataFile[lobDataFile_.length() + sizeof("_save")]; // sizeof includes room for null terminator
+ strcpy(saveLobDataFile,lobDataFile_.data());
+ strcpy(saveLobDataFile+lobDataFile_.length(),"_save");
+ Int32 rc2 = hdfsDelete(fs,lobDataFile_.data(),FALSE);//ok to ignore error.
+ rc2 = hdfsRename(fs,saveLobDataFile, lobDataFile_.data());
if (rc2)
{
lobDebugInfo("Problem renaming savedatafile to data file",0,__LINE__,lobTrace_);
@@ -1742,8 +1787,10 @@ Ex_Lob_Error ExLob::purgeBackupLobDataFile()
hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_);
if (fs == NULL)
return LOB_DATA_FILE_OPEN_ERROR;
- string saveLobDataFile = lobDataFile_ + "_save";
- Int32 rc2 = hdfsDelete(fs,saveLobDataFile.c_str(),FALSE);//ok to ignore error.
+ char saveLobDataFile[lobDataFile_.length() + sizeof("_save")]; // sizeof includes room for null terminator
+ strcpy(saveLobDataFile,lobDataFile_.data());
+ strcpy(saveLobDataFile+lobDataFile_.length(),"_save");
+ Int32 rc2 = hdfsDelete(fs,saveLobDataFile,FALSE);//ok to ignore error.
return rc;
}
@@ -1830,7 +1877,7 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
hdfsCloseFile(fs_, fdData_);
fdData_=NULL;
openFlags_ = O_RDONLY;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, 0, 0, 0);
if (!fdData_)
{
@@ -1905,7 +1952,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
hdfsCloseFile(fs_, fdData_);
fdData_=NULL;
openFlags_ = O_RDONLY;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, 0, 0, 0);
if (!fdData_)
{
@@ -1917,7 +1964,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
}
else
{
- fdData_ = hdfsOpenFile(fs_, lobDataFile_.c_str(), openFlags_, 0, 0, 0);
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_.data(), openFlags_, 0, 0, 0);
if (!fdData_)
{
@@ -1939,9 +1986,14 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
return LOB_DATA_READ_ERROR;
}
- string lobDataFileSubstr = lobDataFile_.substr(0,sizeof(logBuf - 100)); // make small enough to fit
+ // extract a substring small enough to fit into logBuf
+ size_t len = MINOF(lobDataFile_.length(),sizeof(logBuf)-100);
+ char lobDataFileSubstr[len+1]; // +1 for trailing null
+ strncpy(lobDataFileSubstr,lobDataFile_.data(),len);
+ lobDataFileSubstr[len] = '\0';
+
str_sprintf(logBuf,"After hdfsPread: File:%s, Offset:%Ld, Size:%Ld,Target Mem Addr:%Ld",
- lobDataFileSubstr.c_str(),offset,size,memAddr);
+ lobDataFileSubstr,offset,size,memAddr);
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
operLen = bytesRead;
return LOB_OPER_OK;
@@ -2334,8 +2386,7 @@ Ex_Lob_Error ExLobsOper (
if (it == lobMap->end())
{
- //lobPtr = new (lobGlobals->getHeap())ExLob();
- lobPtr = new ExLob();
+ lobPtr = new (lobGlobals->getHeap())ExLob(lobGlobals->getHeap());
if (lobPtr == NULL)
return LOB_ALLOC_ERROR;
@@ -3105,10 +3156,6 @@ ExLobGlobals::~ExLobGlobals()
preOpenList_.clear();
preOpenListLock_.unlock();
-
- if (lobMap_)
- delete lobMap_;
-
if (numWorkerThreads_ > 0) {
for (int i=0; numWorkerThreads_-i > 0 && i < NUM_WORKER_THREADS; i++) {
QRLogger::log(CAT_SQL_EXE, LL_DEBUG, 0, NULL,
@@ -3139,6 +3186,11 @@ ExLobGlobals::~ExLobGlobals()
c_it = postfetchBufList_.erase(c_it);
}
postfetchBufListLock_.unlock();
+
+ //delete the lobMap AFTER the worker threads have finished their pending
+ //work since they may still be using an objetc that was fetched off the lobMap_
+ if (lobMap_)
+ delete lobMap_;
//msg_mon_close_process(&serverPhandle);
if (threadTraceFile_)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/237f9e8e/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index a793487..3d981b4 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -399,12 +399,12 @@ void cleanupLOBDataDescFiles(const char *hdfsServer, int hdfsPort, const char *h
// ExLob
///////////////////////////////////////////////////////////////////////////////
-class ExLob
+class ExLob : public NABasicObject
{
public:
- ExLob(); // default constructor
- ~ExLob(); // default desctructor
+ ExLob(NAHeap * heap); // default constructor
+ virtual ~ExLob(); // default desctructor
Ex_Lob_Error initialize(const char *lobFile, Ex_Lob_Mode mode, char *dir,
LobsStorage storage, char *hdfsServer, Int64 hdfsPort,
@@ -458,7 +458,7 @@ class ExLob
Ex_Lob_Error lockDesc();
Ex_Lob_Error unlockDesc();
- const char *getDataFileName() { return lobDataFile_.c_str(); }
+ const char *getDataFileName() { return lobDataFile_.data(); }
int getErrNo();
@@ -515,7 +515,7 @@ class ExLob
public:
- string lobDataFile_; // TODO: change to NAString when ExLobCursor is allocated off of an NAHeap
+ NAString lobDataFile_;
lobCursors_t lobCursors_;
ExLobLock lobCursorLock_;
LobsStorage storage_;
@@ -558,7 +558,7 @@ class ExLobHdfsRequest
};
-class ExLobPreOpen
+class ExLobPreOpen : public NABasicObject
{
public :
@@ -575,7 +575,10 @@ class ExLobPreOpen
// nothing else to do
}
- ~ExLobPreOpen();
+ virtual ~ExLobPreOpen()
+ {
+ // nothing else to do
+ };
public :
ExLob *lobPtr_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/237f9e8e/core/sql/exp/ExpLOBprocess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBprocess.cpp b/core/sql/exp/ExpLOBprocess.cpp
index 92779ef..1bf0598 100644
--- a/core/sql/exp/ExpLOBprocess.cpp
+++ b/core/sql/exp/ExpLOBprocess.cpp
@@ -502,7 +502,7 @@ Ex_Lob_Error ExLobGlobals::getLobPtr(char *lobName, ExLob *& lobPtr)
if (it == lobMap->end())
{
- lobPtr = new ExLob();
+ lobPtr = new (lobGlobals->getHeap())ExLob(lobGlobals->getHeap());
if (lobPtr == NULL)
return LOB_ALLOC_ERROR;
[5/6] incubator-trafodion git commit: Second rework
Posted by db...@apache.org.
Second rework
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/729472c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/729472c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/729472c2
Branch: refs/heads/master
Commit: 729472c2e647332d9084eefe5504db99e2e1f4bd
Parents: 237f9e8
Author: Dave Birdsall <db...@apache.org>
Authored: Mon Jan 23 21:16:21 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Mon Jan 23 21:16:21 2017 +0000
----------------------------------------------------------------------
core/sql/exp/ExpLOBaccess.cpp | 72 ++++++++-------
core/sql/regress/executor/EXPECTED130 | 97 +++++++++++---------
core/sql/regress/executor/TEST130 | 6 ++
.../src/asciidoc/_chapters/executor_msgs.adoc | 15 +++
4 files changed, 114 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/729472c2/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 8c40bf7..3e06949 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -199,7 +199,7 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff
lobCursors_it it = lobCursors_.find(string(handleIn, handleLenIn));
char logBuf[4096];
lobDebugInfo("In ExLob::fetchCursor",0,__LINE__,lobTrace_);
- char *blackBox = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; // TODO: do we ever go this way for Hive?
+ char *blackBox = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
Int32 blackBoxLen = 0;
if (it == lobCursors_.end())
@@ -2338,7 +2338,6 @@ Ex_Lob_Error ExLobsOper (
{
Ex_Lob_Error err = LOB_OPER_OK;
ExLob *lobPtr = NULL;
- string fn;
struct timespec startTime;
struct timespec endTime;
Int64 secs, nsecs, totalnsecs;
@@ -2491,26 +2490,31 @@ Ex_Lob_Error ExLobsOper (
err = lobPtr->openCursor(handleIn, handleInLen,transId);
break;
- case Lob_OpenDataCursorSimple:
- if (openType == 1) { // preopen
- char temp1[30]; // big enough for :%Lx:
- sprintf(temp1, ":%Lx:",(long long unsigned int)lobName);
- fn = lobPtr->getDataFileName();
- fn += temp1;
- fn += cursorId;
- preOpenObj = new (lobGlobals->getHeap()) ExLobPreOpen(lobPtr, fn.c_str(), descNumIn, sourceLen,
- cursorBytes, waited, lobGlobals->getHeap());
- lobGlobals->addToPreOpenList(preOpenObj);
- } else if (openType == 2) { // must open
- char temp2[30]; // big enough for :%Lx:
- sprintf(temp2, ":%Lx:",(long long unsigned int)lobName);
- fn = lobPtr->getDataFileName();
- fn += temp2;
- fn += cursorId;
- fileName = fn.c_str();
- err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals, (Int32 *)blackBox);
- } else
- err = LOB_SUBOPER_ERROR;
+ case Lob_OpenDataCursorSimple:
+ {
+ size_t dataFileNameLen = strlen(lobPtr->getDataFileName());
+ size_t cursorIdLen = strlen(cursorId);
+ if (openType == 1) { // preopen
+ char temp1[30]; // big enough for :%Lx:
+ sprintf(temp1, ":%Lx:",(long long unsigned int)lobName);
+ char fn[dataFileNameLen + sizeof(temp1) + cursorIdLen + 1];
+ strcpy(fn,lobPtr->getDataFileName());
+ strcpy(fn + dataFileNameLen, temp1);
+ strcpy(fn + dataFileNameLen + strlen(temp1), cursorId);
+ preOpenObj = new (lobGlobals->getHeap()) ExLobPreOpen(lobPtr, fn, descNumIn, sourceLen,
+ cursorBytes, waited, lobGlobals->getHeap());
+ lobGlobals->addToPreOpenList(preOpenObj);
+ } else if (openType == 2) { // must open
+ char temp2[30]; // big enough for :%Lx:
+ sprintf(temp2, ":%Lx:",(long long unsigned int)lobName);
+ char fn[dataFileNameLen + sizeof(temp2) + cursorIdLen + 1];
+ strcpy(fn,lobPtr->getDataFileName());
+ strcpy(fn + dataFileNameLen, temp2);
+ strcpy(fn + dataFileNameLen + strlen(temp2), cursorId);
+ err = lobPtr->openDataCursor(fn, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals, (Int32 *)blackBox);
+ } else
+ err = LOB_SUBOPER_ERROR;
+ }
break;
case Lob_ReadCursor:
@@ -2526,11 +2530,13 @@ Ex_Lob_Error ExLobsOper (
{
char temp3[30]; // big enough for :%Lx:
sprintf(temp3, ":%Lx:",(long long unsigned int)lobName);
- fn = lobPtr->getDataFileName();
- fn += temp3;
- fn += cursorId;
- fileName = fn.c_str();
- err = lobPtr->readDataCursorSimple(fileName, source, sourceLen, retOperLen, lobGlobals);
+ size_t dataFileNameLen = strlen(lobPtr->getDataFileName());
+ size_t cursorIdLen = strlen(cursorId);
+ char fn[dataFileNameLen + sizeof(temp3) + cursorIdLen + 1];
+ strcpy(fn,lobPtr->getDataFileName());
+ strcpy(fn + dataFileNameLen, temp3);
+ strcpy(fn + dataFileNameLen + strlen(temp3), cursorId);
+ err = lobPtr->readDataCursorSimple(fn, source, sourceLen, retOperLen, lobGlobals);
}
break;
@@ -2553,11 +2559,13 @@ Ex_Lob_Error ExLobsOper (
{
char temp4[30]; // big enough for :%Lx:
sprintf(temp4, ":%Lx:",(long long unsigned int)lobName);
- fn = lobPtr->getDataFileName();
- fn += temp4;
- fn += cursorId;
- fileName = fn.c_str();
- err = lobPtr->closeDataCursorSimple(fileName, lobGlobals);
+ size_t dataFileNameLen = strlen(lobPtr->getDataFileName());
+ size_t cursorIdLen = strlen(cursorId);
+ char fn[dataFileNameLen + sizeof(temp4) + cursorIdLen + 1];
+ strcpy(fn,lobPtr->getDataFileName());
+ strcpy(fn + dataFileNameLen, temp4);
+ strcpy(fn + dataFileNameLen + strlen(temp4), cursorId);
+ err = lobPtr->closeDataCursorSimple(fn, lobGlobals);
}
break;
[4/6] incubator-trafodion git commit: Second rework
Posted by db...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/729472c2/core/sql/regress/executor/EXPECTED130
----------------------------------------------------------------------
diff --git a/core/sql/regress/executor/EXPECTED130 b/core/sql/regress/executor/EXPECTED130
index d4f218f..9c8fb51 100644
--- a/core/sql/regress/executor/EXPECTED130
+++ b/core/sql/regress/executor/EXPECTED130
@@ -52,9 +52,9 @@ C1
C1 C2
----------- ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------
- 1 LOBH0000000200010693284694666931821619693284694675520439318212350402808136646020"TRAFODION"."LOB130"
- 2 LOBH0000000200010693284694666931821619693284694676238320918212350402817603197020"TRAFODION"."LOB130"
- 3 LOBH0000000200010693284694666931821619693284694676863363518212350402823890682020"TRAFODION"."LOB130"
+ 1 LOBH0000000200010559696826186435416219559696826195743244718212351957479965039020"TRAFODION"."LOB130"
+ 2 LOBH0000000200010559696826186435416219559696826196364100518212351957487711692020"TRAFODION"."LOB130"
+ 3 LOBH0000000200010559696826186435416219559696826196975611918212351957493829420020"TRAFODION"."LOB130"
--- 3 row(s) selected.
>>
@@ -477,7 +477,7 @@ And the dish ran away with the fork !
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_txt1.txt');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH0000000200010693284694666936172119693284694718802636218212350403243416736020"TRAFODION"."LOB130"
' , 'tlob130_txt1.txt');
+>>extract lobtofile(LOB 'LOBH0000000200010559696826186439964319559696826241512955218212351957939199849020"TRAFODION"."LOB130"
' , 'tlob130_txt1.txt');
Success. Targetfile :tlob130_txt1.txt Length : 19
--- SQL operation complete.
@@ -493,7 +493,7 @@ Success. Targetfile :tlob130_txt1.txt Length : 19
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_deep.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH0000000200010693284694666936277719693284694725950422618212350403314769123020"TRAFODION"."LOB130"
' , 'tlob130_deep.jpg');
+>>extract lobtofile(LOB 'LOBH0000000200010559696826186440101919559696826248930896218212351958013356713020"TRAFODION"."LOB130"
' , 'tlob130_deep.jpg');
Success. Targetfile :tlob130_deep.jpg Length : 159018
--- SQL operation complete.
@@ -509,7 +509,7 @@ Success. Targetfile :tlob130_deep.jpg Length : 159018
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_anoush.jpg');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH0000000200010693284694666936277719693284694725950422618212350403314769123020"TRAFODION"."LOB130"
' , 'tlob130_anoush.jpg');
+>>extract lobtofile(LOB 'LOBH0000000200010559696826186440101919559696826248930896218212351958013356713020"TRAFODION"."LOB130"
' , 'tlob130_anoush.jpg');
Success. Targetfile :tlob130_anoush.jpg Length : 230150
--- SQL operation complete.
@@ -536,6 +536,8 @@ Success. Targetfile :tlob130_anoush.jpg Length : 230150
>>sh regrhadoop.ksh fs -copyFromLocal lob_input_e1.txt /lobs/lob_input_e1.txt;
>>sh regrhadoop.ksh fs -copyFromLocal deep.jpg /lobs/deep.jpg;
>>sh regrhadoop.ksh fs -copyFromLocal anoush.jpg /lobs/anoush.jpg;
+>>-- the next one is a really long file name intended to test error message 8557
+>>sh regrhadoop.ksh fs -copyFromLocal lob_input_a1.txt /lobs/reallyLongDirectoryName0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789/lob_input_a1012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789.txt;
>>sh sleep(20);
>>
>>
@@ -629,7 +631,7 @@ And the dish ran away with the fork !
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'hdfs:\/\/\/lobs\/tlob130_txt2.txt');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH0000000200010693284694666937699219693284694733387350918212350403389004357020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_txt2.txt');
+>>extract lobtofile(LOB 'LOBH0000000200010559696826186441630019559696826257581553118212351958099882804020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_txt2.txt');
Success. Targetfile :hdfs:///lobs/tlob130_txt2.txt Length : 19
--- SQL operation complete.
@@ -645,7 +647,7 @@ Success. Targetfile :hdfs:///lobs/tlob130_txt2.txt Length : 19
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'hdfs:\/\/\/lobs\/tlob130_deep.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH0000000200010693284694666937807719693284694740112181518212350403456401952020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_deep.jpg');
+>>extract lobtofile(LOB 'LOBH0000000200010559696826186441758719559696826264825758518212351958172401525020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_deep.jpg');
Success. Targetfile :hdfs:///lobs/tlob130_deep.jpg Length : 159018
--- SQL operation complete.
@@ -661,7 +663,7 @@ Success. Targetfile :hdfs:///lobs/tlob130_deep.jpg Length : 159018
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'hdfs:\/\/\/lobs\/tlob130_anoush.jpg');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH0000000200010693284694666936277719693284694725950422618212350403314769123020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_anoush.jpg');
+>>extract lobtofile(LOB 'LOBH0000000200010559696826186440101919559696826248930896218212351958013356713020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_anoush.jpg');
Success. Targetfile :hdfs:///lobs/tlob130_anoush.jpg Length : 230150
--- SQL operation complete.
@@ -680,7 +682,7 @@ Column Name : c2
Input a filename to extract to :
Output File Name : lobc2out.jpg
Extracting lob handle for column c2...
-LOB handle for c2: LOBH0000000200010693284694666936277719693284694725950422618212350403314769123020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010559696826186440101919559696826248930896218212351958013356713020"TRAFODION"."LOB130"
Extracting LOB data length for the above handle...
LOB data length :230150
Extracting lob data into file in chunks of 1000 ...
@@ -961,7 +963,7 @@ And the dish ran away with the spoon.
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_deep2.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH0000000200020693284694666939887019693284694753101879518212350403586346705020"TRAFODION"."LOB130"
' , 'tlob130_deep2.jpg');
+>>extract lobtofile(LOB 'LOBH0000000200020559696826186444079619559696826279514148718212351958319078382020"TRAFODION"."LOB130"
' , 'tlob130_deep2.jpg');
Success. Targetfile :tlob130_deep2.jpg Length : 159018
--- SQL operation complete.
@@ -970,7 +972,7 @@ Success. Targetfile :tlob130_deep2.jpg Length : 159018
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'hdfs:\/\/\/lobs\/tlob130_anoush2.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH0000000200030693284694666939887019693284694753661757718212350403591996291020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_anoush2.jpg');
+>>extract lobtofile(LOB 'LOBH0000000200030559696826186444079619559696826280142532318212351958325419986020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_anoush2.jpg');
Success. Targetfile :hdfs:///lobs/tlob130_anoush2.jpg Length : 230150
--- SQL operation complete.
@@ -997,7 +999,7 @@ Hey diddle diddle,
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_anoush3.jpg',create,truncate);/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH0000000200030693284694666940355019693284694758323311318212350403638460438020"TRAFODION"."LOB130"
' , 'tlob130_anoush3.jpg',create,truncate);
+>>extract lobtofile(LOB 'LOBH0000000200030559696826186444611919559696826285313364218212351958377158869020"TRAFODION"."LOB130"
' , 'tlob130_anoush3.jpg',create,truncate);
Success. Targetfile :tlob130_anoush3.jpg Length : 230150
--- SQL operation complete.
@@ -1098,12 +1100,12 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT2
ColumnName : C2
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669434383_0001
+ LOB Data File: LOBP_05596968261864479209_0001
LOB EOD : 0
LOB Used Len : 0
ColumnName : C3
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669434383_0002
+ LOB Data File: LOBP_05596968261864479209_0002
LOB EOD : 0
LOB Used Len : 0
ColumnName : C4
@@ -1118,8 +1120,8 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT2
CATALOG_NAME SCHEMA_NAME OBJECT_NAME COLUMN_NAME
LOB_LOCATION LOB_DATA_FILE LOB_DATA_FILE_SIZE_EOD LOB_DATA_FILE_SIZE_USED
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------- -----------------------
-TRAFODION LOB130 TLOB130GT2 C2
/lobs LOBP_06932846946669434383_0001 0 0
-TRAFODION LOB130 TLOB130GT2 C3
/lobs LOBP_06932846946669434383_0002 0 0
+TRAFODION LOB130 TLOB130GT2 C2
/lobs LOBP_05596968261864479209_0001 0 0
+TRAFODION LOB130 TLOB130GT2 C3
/lobs LOBP_05596968261864479209_0002 0 0
TRAFODION LOB130 TLOB130GT2 C4
External HDFS Location External HDFS File 0 0
--- 3 row(s) selected.
@@ -1140,17 +1142,17 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT
ColumnName : C2
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669431934_0001
+ LOB Data File: LOBP_05596968261864476749_0001
LOB EOD : 15
LOB Used Len : 15
ColumnName : C3
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669431934_0002
+ LOB Data File: LOBP_05596968261864476749_0002
LOB EOD : 15
LOB Used Len : 15
ColumnName : C4
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669431934_0003
+ LOB Data File: LOBP_05596968261864476749_0003
LOB EOD : 45
LOB Used Len : 45
@@ -1160,9 +1162,9 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT
CATALOG_NAME SCHEMA_NAME OBJECT_NAME COLUMN_NAME
LOB_LOCATION LOB_DATA_FILE LOB_DATA_FILE_SIZE_EOD LOB_DATA_FILE_SIZE_USED
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------- -----------------------
-TRAFODION LOB130 TLOB130GT C2
/lobs LOBP_06932846946669431934_0001 15 15
-TRAFODION LOB130 TLOB130GT C3
/lobs LOBP_06932846946669431934_0002 15 15
-TRAFODION LOB130 TLOB130GT C4
/lobs LOBP_06932846946669431934_0003 45 45
+TRAFODION LOB130 TLOB130GT C2
/lobs LOBP_05596968261864476749_0001 15 15
+TRAFODION LOB130 TLOB130GT C3
/lobs LOBP_05596968261864476749_0002 15 15
+TRAFODION LOB130 TLOB130GT C4
/lobs LOBP_05596968261864476749_0003 45 45
--- 3 row(s) selected.
>>delete from tlob130gt where c1=2;
@@ -1179,17 +1181,17 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT
ColumnName : C2
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669431934_0001
+ LOB Data File: LOBP_05596968261864476749_0001
LOB EOD : 30
LOB Used Len : 25
ColumnName : C3
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669431934_0002
+ LOB Data File: LOBP_05596968261864476749_0002
LOB EOD : 31
LOB Used Len : 26
ColumnName : C4
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669431934_0003
+ LOB Data File: LOBP_05596968261864476749_0003
LOB EOD : 71
LOB Used Len : 56
@@ -1199,9 +1201,9 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT
CATALOG_NAME SCHEMA_NAME OBJECT_NAME COLUMN_NAME
LOB_LOCATION LOB_DATA_FILE LOB_DATA_FILE_SIZE_EOD LOB_DATA_FILE_SIZE_USED
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------- -----------------------
-TRAFODION LOB130 TLOB130GT C2
/lobs LOBP_06932846946669431934_0001 30 25
-TRAFODION LOB130 TLOB130GT C3
/lobs LOBP_06932846946669431934_0002 31 26
-TRAFODION LOB130 TLOB130GT C4
/lobs LOBP_06932846946669431934_0003 71 56
+TRAFODION LOB130 TLOB130GT C2
/lobs LOBP_05596968261864476749_0001 30 25
+TRAFODION LOB130 TLOB130GT C3
/lobs LOBP_05596968261864476749_0002 31 26
+TRAFODION LOB130 TLOB130GT C4
/lobs LOBP_05596968261864476749_0003 71 56
--- 3 row(s) selected.
>>
@@ -1270,6 +1272,13 @@ update tlob130ext set c4=externaltolob('hdfs:///lobs/lob_input_a1.txt', append)
>>insert into tlob130ext values(1, stringtolob('first lob'), filetolob('hdfs:///lobs/lob_input_a1.txt'),externaltolob('hdfs:///lobs/lob_input_a1.txt'));
--- 1 row(s) inserted.
+>>-- the next one should see error 8557
+>>insert into tlob130ext values(1, stringtolob('first lob'), filetolob('hdfs:///lobs/lob_input_a1.txt'),
++>externaltolob('hdfs:///lobs/reallyLongDirectoryName0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789/lob_input_a1012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789.txt'));
+
+*** ERROR[8557] The file name passed to externaltolob exceeds 256 bytes.
+
+--- 0 row(s) inserted.
>>
>>select lobtostring(c2,50),lobtostring(c3,50),lobtostring(c4,50) from tlob130ext;
@@ -1356,13 +1365,13 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130EXT
ColumnName : C2
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669445784_0001
- LOB EOD : 61
+ LOB Data File: LOBP_05596968261864491728_0001
+ LOB EOD : 70
LOB Used Len : 42
ColumnName : C3
Lob Location : /lobs
- LOB Data File: LOBP_06932846946669445784_0002
- LOB EOD : 106
+ LOB Data File: LOBP_05596968261864491728_0002
+ LOB EOD : 125
LOB Used Len : 68
ColumnName : C4
Lob Location : External HDFS Location
@@ -1376,8 +1385,8 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130EXT
CATALOG_NAME SCHEMA_NAME OBJECT_NAME COLUMN_NAME
LOB_LOCATION LOB_DATA_FILE LOB_DATA_FILE_SIZE_EOD LOB_DATA_FILE_SIZE_USED
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------- -----------------------
-TRAFODION LOB130 TLOB130EXT C2
/lobs LOBP_06932846946669445784_0001 61 42
-TRAFODION LOB130 TLOB130EXT C3
/lobs LOBP_06932846946669445784_0002 106 68
+TRAFODION LOB130 TLOB130EXT C2
/lobs LOBP_05596968261864491728_0001 70 42
+TRAFODION LOB130 TLOB130EXT C3
/lobs LOBP_05596968261864491728_0002 125 68
TRAFODION LOB130 TLOB130EXT C4
External HDFS Location External HDFS File 0 0
--- 3 row(s) selected.
@@ -1394,7 +1403,7 @@ Column Name : c4
Input a filename to extract to :
Output File Name : lobc4ext.txt
Extracting lob handle for column c4...
-LOB handle for c4: LOBH0000000800030693284694666944578419693284694806495512618212350404120317726020"TRAFODION"."LOB130"
+LOB handle for c4: LOBH0000000800030559696826186449172819559696826337745983818212351958901392808020"TRAFODION"."LOB130"
Extracting LOB data length for the above handle...
LOB data length :19
Extracting lob data into file in chunks of 1000 ...
@@ -1419,7 +1428,7 @@ Table name : TRAFODION.LOB130.t130lob5
Input lob column name to get handle from :
Column Name : c2
Extracting lob handle for column c2...
-LOB handle for c2: LOBH0000000200010693284694666947044019693284694824175608718212350404297138941020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010559696826186452001719559696826357840916318212351959102589093020"TRAFODION"."LOB130"
>>select lobtostring(c2,20) from t130lob5;
(EXPR)
@@ -1436,7 +1445,7 @@ Table name : TRAFODION.LOB130.t130lob5
Input lob column name to get handle from :
Column Name : c2
Extracting lob handle for column c2...
-LOB handle for c2: LOBH0000000200010693284694666947044019693284694824175608718212350404297138941020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010559696826186452001719559696826357840916318212351959102589093020"TRAFODION"."LOB130"
>>select lobtostring(c2,40) from t130lob5;
(EXPR)
@@ -1453,7 +1462,7 @@ Table name : TRAFODION.LOB130.t130lob5
Input lob column name to get handle from :
Column Name : c2
Extracting lob handle for column c2...
-LOB handle for c2: LOBH0000000200010693284694666947044019693284694824175608718212350404297138941020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010559696826186452001719559696826357840916318212351959102589093020"TRAFODION"."LOB130"
>>select lobtostring(c2,20) from t130lob5;
(EXPR)
@@ -1470,7 +1479,7 @@ Table name : TRAFODION.LOB130.t130lob5
Input lob column name to get handle from :
Column Name : c2
Extracting lob handle for column c2...
-LOB handle for c2: LOBH0000000200010693284694666947044019693284694824175608718212350404297138941020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010559696826186452001719559696826357840916318212351959102589093020"TRAFODION"."LOB130"
>>select lobtostring(c2,40) from t130lob5;
(EXPR)
@@ -1497,7 +1506,7 @@ zzzzzzzzzzzzzzzzzzzz
C2
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
---------------------------
-LOBH0000000200010693284694666944578419693284694836695682418212350404422379045020"TRAFODION"."LOB130"
+LOBH0000000200010559696826186449172819559696826371325150818212351959237435210020"TRAFODION"."LOB130"
--- 1 row(s) selected.
>>-- following should return error since only external lobs will be allowed
@@ -1540,9 +1549,9 @@ LOBH0000000200010693284694666944578419693284694836695682418212350404422379045020
Tables in Schema TRAFODION.LOBSCH
=================================
-LOBDescChunks__06932846946669487886_0001
-LOBDescHandle__06932846946669487886_0001
-LOBMD__06932846946669487886
+LOBDescChunks__05596968261864538503_0001
+LOBDescHandle__05596968261864538503_0001
+LOBMD__05596968261864538503
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
SB_PERSISTENT_SAMPLES