You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sa...@apache.org on 2016/03/09 19:43:59 UTC
[01/11] incubator-trafodion git commit: Add a new table to keep track
of lob offsets for GC.
Repository: incubator-trafodion
Updated Branches:
refs/heads/master a1bc4654a -> 5a6549b23
Add a new table to keep track of lob offsets for GC.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/93ccb080
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/93ccb080
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/93ccb080
Branch: refs/heads/master
Commit: 93ccb080a87e2f55df05d50cd4a1cc9cb1d7fe78
Parents: f365667
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Thu Jan 21 23:51:33 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Thu Jan 21 23:51:33 2016 +0000
----------------------------------------------------------------------
core/sql/cli/Cli.cpp | 139 ++++++++++++++++++++++-----
core/sql/exp/ExpLOB.cpp | 4 +-
core/sql/exp/ExpLOB.h | 2 +-
core/sql/exp/ExpLOBaccess.cpp | 11 +++
core/sql/exp/ExpLOBaccess.h | 2 +-
core/sql/exp/ExpLOBprocess.cpp | 2 +-
core/sql/generator/GenPreCode.cpp | 4 +-
core/sql/generator/GenRelUpdate.cpp | 1 +
core/sql/sqlcomp/CmpSeabaseDDLindex.cpp | 2 +-
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 2 +-
10 files changed, 137 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index 58a77ba..fea7913 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -93,6 +93,7 @@
#include "LmLangManagerC.h"
#include "LmLangManagerJava.h"
#include "LmRoutine.h"
+#include "CmpDDLCatErrorCodes.h"
#define DISPLAY_DONE_WARNING 1032
extern Lng32 getTotalTcbSpace(char * tdb, char * otherInfo,
@@ -9458,11 +9459,11 @@ Lng32 SQLCLI_LOBcliInterface
ExpLOBoper::ExpGetLOBDescChunksName(schNameLen, schName, uid, lobNum,
lobDescChunksNameBuf, lobDescChunksNameLen);
- char lobHdrNameBuf[1024];
- Lng32 lobHdrNameLen = 1024;
- char * lobHdrName =
- ExpLOBoper::ExpGetLOBHdrName(schNameLen, schName, uid, lobNum,
- lobHdrNameBuf, lobHdrNameLen);
+ char lobOffsetsNameBuf[1024];
+ Lng32 lobOffsetsNameLen = 1024;
+ char * lobOffsetsName =
+ ExpLOBoper::ExpGetLOBOffsetsName(schNameLen, schName, uid, lobNum,
+ lobOffsetsNameBuf, lobOffsetsNameLen);
char * query = new(currContext.exHeap()) char[4096];
@@ -9565,8 +9566,7 @@ Lng32 SQLCLI_LOBcliInterface
case LOB_CLI_CREATE:
{
- cliRC = cliInterface->executeImmediate("cqd pos hold;");
- cliRC = cliInterface->executeImmediate("cqd pos 'OFF';");
+
// create lob descriptor handle table salted
str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, numChunks int not null, lobLen largeint not null) salt using 8 partitions store by (descPartnKey, syskey) ",
@@ -9581,8 +9581,7 @@ Lng32 SQLCLI_LOBcliInterface
if (cliRC < 0)
{
- cliInterface->executeImmediate("cqd pos restore;");
-
+
cliInterface->retrieveSQLDiagnostics(myDiags);
goto error_return;
@@ -9602,14 +9601,35 @@ Lng32 SQLCLI_LOBcliInterface
if (cliRC < 0)
{
- cliInterface->executeImmediate("cqd pos restore;");
-
+
cliInterface->retrieveSQLDiagnostics(myDiags);
goto error_return;
}
- cliInterface->executeImmediate("cqd pos restore;");
+
+
+ // create lob datafile descriptor unaudited table. Use a cqd
+ // traf_no_dtm_xn while doing any dml to this table.
+ //dataState = 1 - indicates used chunk
+ //dataState = 0 - indicates unused chunk
+ str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, descSysKey largeint not null, chunkNum int not null, chunkLen largeint not null, offset largeint, dataState int , primary key(descPartnKey, descSysKey, chunkNum)) ",
+ lobOffsetsName);
+
+
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->executeImmediate(query);
+
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
cliRC = 0;
}
@@ -9626,7 +9646,8 @@ Lng32 SQLCLI_LOBcliInterface
cliRC = cliInterface->executeImmediate(query);
currContext.resetSqlParserFlags(0x1);
-
+ if (cliRC == -CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION)
+ cliRC = 0;
if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(myDiags);
@@ -9644,13 +9665,34 @@ Lng32 SQLCLI_LOBcliInterface
currContext.resetSqlParserFlags(0x1);
+ if (cliRC == -CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION)
+ cliRC = 0;
+
if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(myDiags);
goto error_return;
}
+ str_sprintf(query, "drop ghost table %s",
+ lobOffsetsName);
+
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->executeImmediate(query);
+
+ currContext.resetSqlParserFlags(0x1);
+ if (cliRC == -CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION)
+ cliRC = 0;
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
cliRC = 0;
}
@@ -9671,7 +9713,12 @@ Lng32 SQLCLI_LOBcliInterface
if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(myDiags);
-
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+ if (myDiags->containsError(-CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION))
+ {
+ cliRC = 0;
+ break;
+ }
goto error_return;
}
@@ -9684,11 +9731,15 @@ Lng32 SQLCLI_LOBcliInterface
cliRC = cliInterface->executeImmediate(query);
currContext.resetSqlParserFlags(0x1);
-
+
if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(myDiags);
-
+ if (myDiags->containsError(-CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION))
+ {
+ cliRC = 0;
+ break;
+ }
goto error_return;
}
@@ -9751,6 +9802,28 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
+ //Insert into the lob offsets table (without transaction)
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
+ str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld,1, %Ld, %Ld, 1)",
+ lobOffsetsName, descPartnKey, descSyskey,
+ (dataLen ? *dataLen : 0),
+ (dataOffset ? *dataOffset : 0) );
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->executeImmediate(query);
+
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
+
if (outDescPartnKey)
*outDescPartnKey = descPartnKey;
@@ -9825,7 +9898,8 @@ Lng32 SQLCLI_LOBcliInterface
str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld, %d, %Ld, %Ld, NULL)",
lobDescChunksName, descPartnKey, inDescSyskey,
numChunks, (dataLen ? *dataLen : 0),
- (dataOffset ? *dataOffset : 0));
+ (dataOffset ? *dataOffset : 0)
+ );
}
// set parserflags to allow ghost table
@@ -9976,6 +10050,25 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
+ //Delete from lob offsets table (without transaction)
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
+ str_sprintf(query, "update table(ghost table %s) set dataState = 0 where descPartnKey = %Ld and syskey = %Ld",lobOffsetsName, descPartnKey, inDescSyskey);
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->executeImmediate(query);
+
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
+
cliRC = 0;
// if (outDescSyskey)
// *outDescSyskey = inDescSyskey;
@@ -10343,9 +10436,7 @@ Lng32 SQLCLI_LOBddlInterface
{
case LOB_CLI_CREATE:
{
- cliRC = cliInterface->executeImmediate("cqd pos hold;");
- cliRC = cliInterface->executeImmediate("cqd pos 'OFF';");
-
+
// create lob metadata table
str_sprintf(query, "create ghost table %s (lobnum smallint not null, storagetype smallint not null, location varchar(4096) not null, primary key (lobnum)) location $system",
// str_sprintf(query, "create ghost table %s (lobnum smallint not null, storagetype smallint not null, location varchar(4096) not null, primary key (lobnum)) no partition",
@@ -10356,7 +10447,7 @@ Lng32 SQLCLI_LOBddlInterface
cliRC = cliInterface->executeImmediate(query);
- cliInterface->executeImmediate("cqd pos restore;");
+
currContext.resetSqlParserFlags(0x1);
@@ -10457,10 +10548,12 @@ Lng32 SQLCLI_LOBddlInterface
cliRC = cliInterface->executeImmediate(query);
currContext.resetSqlParserFlags(0x1);
-
+ if (cliRC == -CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION)
+ cliRC = 0;
if (cliRC < 0)
{
- cliInterface->retrieveSQLDiagnostics(&diags);
+ cliInterface->retrieveSQLDiagnostics(&diags);
+
goto error_return;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/exp/ExpLOB.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.cpp b/core/sql/exp/ExpLOB.cpp
index 18470d6..00116b6 100644
--- a/core/sql/exp/ExpLOB.cpp
+++ b/core/sql/exp/ExpLOB.cpp
@@ -137,7 +137,7 @@ char * ExpLOBoper::ExpGetLOBDescChunksName(Lng32 schNameLen, char * schName,
return outBuf;
}
-char * ExpLOBoper::ExpGetLOBHdrName(Lng32 schNameLen, char * schName,
+char * ExpLOBoper::ExpGetLOBOffsetsName(Lng32 schNameLen, char * schName,
Int64 uid, Lng32 num,
char * outBuf, Lng32 outBufLen)
{
@@ -146,7 +146,7 @@ char * ExpLOBoper::ExpGetLOBHdrName(Lng32 schNameLen, char * schName,
(schName == NULL))
return NULL;
- str_sprintf(outBuf, "%s.\"LOBHdr_%020Ld_%04d\"",
+ str_sprintf(outBuf, "%s.\"LOBOffset_%020Ld_%04d\"",
schName, uid, num);
return outBuf;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/exp/ExpLOB.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.h b/core/sql/exp/ExpLOB.h
index de74b4a..316c259 100644
--- a/core/sql/exp/ExpLOB.h
+++ b/core/sql/exp/ExpLOB.h
@@ -225,7 +225,7 @@ public:
Int64 uid, Lng32 lobNum,
char * outBuf, Lng32 outBufLen);
- static char * ExpGetLOBHdrName(Lng32 schNameLen, char * schName,
+ static char * ExpGetLOBOffsetsName(Lng32 schNameLen, char * schName,
Int64 uid, Lng32 lobNum,
char * outBuf, Lng32 outBufLen);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index f16d48b..9c66eed 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -302,6 +302,8 @@ Ex_Lob_Error ExLob::putDesc(ExLobDesc &desc, Int64 descNum)
return err;
}
+
+
Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operLen)
{
Ex_Lob_Error err;
@@ -1368,7 +1370,16 @@ Ex_Lob_Error ExLob::doSanityChecks(char *dir, LobsStorage storage,
return LOB_OPER_OK;
}
+Ex_Lob_Error ExLob::getNextFreeOffset(Int32 size,Int64 &freeOffset, Int64 lobMaxSize)
+{
+ Ex_Lob_Error err = LOB_OPER_OK;
+ Lng32 retval = 0;
+ Int32 openFlags = O_RDONLY ;
+ Int64 currentOffset = 0;
+
+ return LOB_OPER_OK;
+}
Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, Int64 lobMaxSize)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index e68c2eb..a4cf509 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -414,6 +414,7 @@ class ExLob
Int64 handleInLen, Int64 handleOutLen,
Int64 blackBoxLen);
Ex_Lob_Error allocateDesc(unsigned int size, Int64 &descNum, Int64 &dataOffset,Int64 lobMaxSize);
+ Ex_Lob_Error getNextFreeOffset(Int32 size,Int64 &freeOffset, Int64 lobMaxSize);
Ex_Lob_Error readStats(char *buffer);
Ex_Lob_Error initStats();
@@ -446,7 +447,6 @@ class ExLob
Ex_Lob_Error readDataFromFile(char *memAddr, Int64 len, Int64 &operLen);
Ex_Lob_Error emptyDirectory();
-
ExLobStats *getStats() { return &stats_; }
NAHeap *getLobGlobalHeap() { return lobGlobalHeap_;}
public:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/exp/ExpLOBprocess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBprocess.cpp b/core/sql/exp/ExpLOBprocess.cpp
index d456c34..cabc7c3 100644
--- a/core/sql/exp/ExpLOBprocess.cpp
+++ b/core/sql/exp/ExpLOBprocess.cpp
@@ -554,7 +554,7 @@ Ex_Lob_Error ExLob::purgeLob()
return err;
} */
-
+
Ex_Lob_Error ExLob::insertDesc(ExLobRequest *request)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/generator/GenPreCode.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp
index c730c4e..1adce80 100644
--- a/core/sql/generator/GenPreCode.cpp
+++ b/core/sql/generator/GenPreCode.cpp
@@ -4837,7 +4837,7 @@ RelExpr * HbaseDelete::preCodeGen(Generator * generator,
}
NABoolean isAlignedFormat = getTableDesc()->getNATable()->isAlignedFormat(getIndexDesc());
- if (producesOutputs())
+ if (producesOutputs() || getTableDesc()->getNATable()->hasLobColumn())
{
retColRefSet_ = getIndexDesc()->getIndexColumns();
}
@@ -5001,7 +5001,7 @@ RelExpr * HbaseDelete::preCodeGen(Generator * generator,
// flag for hbase tables
generator->setHdfsAccess(TRUE);
-
+
markAsPreCodeGenned();
return this;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/generator/GenRelUpdate.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelUpdate.cpp b/core/sql/generator/GenRelUpdate.cpp
index da7c8ea..3aedb68 100644
--- a/core/sql/generator/GenRelUpdate.cpp
+++ b/core/sql/generator/GenRelUpdate.cpp
@@ -787,6 +787,7 @@ short HbaseDelete::codeGen(Generator * generator)
ex_expr *convert_expr = NULL;
ex_expr * keyColValExpr = NULL;
ex_expr *preCondExpr = NULL;
+ ex_expr *lobExpr = NULL;
ex_cri_desc * givenDesc
= generator->getCriDesc(Generator::DOWN);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/sqlcomp/CmpSeabaseDDLindex.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLindex.cpp b/core/sql/sqlcomp/CmpSeabaseDDLindex.cpp
index 94ebf17..37f8111 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLindex.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLindex.cpp
@@ -152,7 +152,7 @@ CmpSeabaseDDL::createIndexColAndKeyInfoArrays(
const NAType * naType = tableCol->getType();
if ((naType->getFSDatatype() == REC_BLOB) || (naType->getFSDatatype() == REC_CLOB))
{
- *CmpCommon::diags() << DgSqlCode(CAT_LOB_COL_CANNOT_BE_INDEX_OR_KEY)
+ *CmpCommon::diags() << DgSqlCode(-CAT_LOB_COL_CANNOT_BE_INDEX_OR_KEY)
<< DgColumnName(col_name);
processReturn();
return -1;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/93ccb080/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
index eb4efbd..d5a0efa 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
@@ -1875,7 +1875,7 @@ short CmpSeabaseDDL::createSeabaseTable2(
if (colType->getFSDatatype() == REC_BLOB || colType->getFSDatatype() == REC_CLOB)
//Cannot allow LOB in primary or clustering key
{
- *CmpCommon::diags() << DgSqlCode(CAT_LOB_COL_CANNOT_BE_INDEX_OR_KEY)
+ *CmpCommon::diags() << DgSqlCode(-CAT_LOB_COL_CANNOT_BE_INDEX_OR_KEY)
<< DgColumnName(colName);
deallocEHI(ehi);
[02/11] incubator-trafodion git commit: Anoop's delete expresssion
change
Posted by sa...@apache.org.
Anoop's delete expresssion change
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/57ae601c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/57ae601c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/57ae601c
Branch: refs/heads/master
Commit: 57ae601cb3fc41d39836574027fef229a4d78e20
Parents: 93ccb08
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Fri Jan 22 21:26:41 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Fri Jan 22 21:26:41 2016 +0000
----------------------------------------------------------------------
core/sql/cli/Cli.cpp | 75 ++++++++++++++++++++++++++++--
core/sql/comexe/ComTdbHbaseAccess.cpp | 2 +-
core/sql/comexe/ComTdbHbaseAccess.h | 2 +-
core/sql/executor/ExHbaseAccess.h | 3 ++
core/sql/executor/ExHbaseIUD.cpp | 27 ++++++++++-
core/sql/exp/ExpLOBaccess.h | 2 +-
core/sql/exp/ExpLOBprocess.cpp | 13 ++++++
core/sql/generator/GenPreCode.cpp | 25 ++++++++--
core/sql/generator/GenRelUpdate.cpp | 46 +++++++++++++++++-
9 files changed, 182 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/57ae601c/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index fea7913..018bbfa 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -9613,7 +9613,7 @@ Lng32 SQLCLI_LOBcliInterface
// traf_no_dtm_xn while doing any dml to this table.
//dataState = 1 - indicates used chunk
//dataState = 0 - indicates unused chunk
- str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, descSysKey largeint not null, chunkNum int not null, chunkLen largeint not null, offset largeint, dataState int , primary key(descPartnKey, descSysKey, chunkNum)) ",
+ str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, descSysKey largeint not null, chunkNum int not null, chunkLen largeint not null, offset largeint, dataState int , primary key(descPartnKey, descSysKey, chunkLen)) ",
lobOffsetsName);
@@ -9622,7 +9622,7 @@ Lng32 SQLCLI_LOBcliInterface
cliRC = cliInterface->executeImmediate(query);
- currContext.resetSqlParserFlags(0x1);
+ currContext.resetSqlParserFlags(0x1);x
if (cliRC < 0)
{
@@ -9916,6 +9916,29 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
+ //Insert into the lob offsets table (without transaction)
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
+ str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld,%d, %Ld, %Ld, 1)",
+ lobOffsetsName, descPartnKey, descSyskey,numChunks
+ (dataLen ? *dataLen : 0),
+ (dataOffset ? *dataOffset : 0) );
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->executeImmediate(query);
+
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
+
cliRC = 0;
// if (outDescSyskey)
@@ -9998,6 +10021,51 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
+ // Update the lob offsets table. Set all the previous chunks to be
+ // unused. Insert the new chunk as the new used chunk.
+
+
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
+
+ //update old chunks as unused
+
+ str_sprintf(query, "update table(ghost table %s) set dataState = 0 where descPartnKey = %Ld and descSyskey = %Ld",lobOffsetsName, descPartnKey, inDescSyskey);
+
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+ cliRC = cliInterface->executeImmediate(query);
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+
+ //Insert new chunk offset and length
+ str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld,1, %Ld, %Ld, 1)",
+ lobOffsetsName, descPartnKey, descSyskey,
+ (dataLen ? *dataLen : 0),
+ (dataOffset ? *dataOffset : 0) );
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->executeImmediate(query);
+
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
+
// update lob handle with the returned values
if (outLobHandle)
{
@@ -10053,7 +10121,7 @@ Lng32 SQLCLI_LOBcliInterface
//Delete from lob offsets table (without transaction)
cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
- str_sprintf(query, "update table(ghost table %s) set dataState = 0 where descPartnKey = %Ld and syskey = %Ld",lobOffsetsName, descPartnKey, inDescSyskey);
+ str_sprintf(query, "update table(ghost table %s) set dataState = 0 where descPartnKey = %Ld and descSyskey = %Ld",lobOffsetsName, descPartnKey, inDescSyskey);
// set parserflags to allow ghost table
currContext.setSqlParserFlags(0x1);
@@ -10063,6 +10131,7 @@ Lng32 SQLCLI_LOBcliInterface
if (cliRC < 0)
{
+ cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
cliInterface->retrieveSQLDiagnostics(myDiags);
goto error_return;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/57ae601c/core/sql/comexe/ComTdbHbaseAccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbHbaseAccess.cpp b/core/sql/comexe/ComTdbHbaseAccess.cpp
index 050018a..762a676 100644
--- a/core/sql/comexe/ComTdbHbaseAccess.cpp
+++ b/core/sql/comexe/ComTdbHbaseAccess.cpp
@@ -334,7 +334,7 @@ ComTdbHbaseAccess::getExpressionName(Int32 expNum) const
case 3:
return "UpdateExpr";
case 4:
- return "MergeInsertExpr";
+ return ((getAccessType() == DELETE_) ? "LobDeleteExpr" : "MergeInsertExpr");
case 5:
return "LowKeyExpr";
case 6:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/57ae601c/core/sql/comexe/ComTdbHbaseAccess.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbHbaseAccess.h b/core/sql/comexe/ComTdbHbaseAccess.h
index 1a19dd9..ded7c0a 100644
--- a/core/sql/comexe/ComTdbHbaseAccess.h
+++ b/core/sql/comexe/ComTdbHbaseAccess.h
@@ -154,7 +154,7 @@ public:
return NULL;
};
- ComTdbAccessType getAccessType() { return ComTdbAccessType(accessType_); }
+ ComTdbAccessType getAccessType() const { return ComTdbAccessType(accessType_); }
class HbaseScanRows : public NAVersionedObject
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/57ae601c/core/sql/executor/ExHbaseAccess.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.h b/core/sql/executor/ExHbaseAccess.h
index 85c50be..59b62f6 100644
--- a/core/sql/executor/ExHbaseAccess.h
+++ b/core/sql/executor/ExHbaseAccess.h
@@ -205,6 +205,9 @@ protected:
inline ex_expr *mergeInsertExpr() const
{ return hbaseAccessTdb().mergeInsertExpr_; }
+ inline ex_expr *lobDelExpr() const
+ { return hbaseAccessTdb().mergeInsertExpr_; }
+
inline ex_expr *mergeInsertRowIdExpr() const
{ return hbaseAccessTdb().mergeInsertRowIdExpr_; }
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/57ae601c/core/sql/executor/ExHbaseIUD.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseIUD.cpp b/core/sql/executor/ExHbaseIUD.cpp
index e869620..6f87e06 100644
--- a/core/sql/executor/ExHbaseIUD.cpp
+++ b/core/sql/executor/ExHbaseIUD.cpp
@@ -1851,6 +1851,7 @@ ExWorkProcRetcode ExHbaseUMDtrafUniqueTaskTcb::work(short &rc)
step_ = HANDLE_ERROR;
else if ((tcb_->hbaseAccessTdb().getAccessType() == ComTdbHbaseAccess::DELETE_) &&
(! tcb_->scanExpr()) &&
+ (! tcb_->lobDelExpr()) &&
(NOT tcb_->hbaseAccessTdb().returnRow()))
step_ = DELETE_ROW;
else
@@ -2239,7 +2240,19 @@ ExWorkProcRetcode ExHbaseUMDtrafUniqueTaskTcb::work(short &rc)
step_ = HANDLE_ERROR;
break;
}
-
+
+ // delete entries from LOB desc table, if needed
+ if (tcb_->lobDelExpr())
+ {
+ ex_expr::exp_return_type exprRetCode =
+ tcb_->lobDelExpr()->eval(pentry_down->getAtp(), tcb_->workAtp_);
+ if (exprRetCode == ex_expr::EXPR_ERROR)
+ {
+ step_ = HANDLE_ERROR;
+ break;
+ }
+ }
+
if (tcb_->getHbaseAccessStats())
tcb_->getHbaseAccessStats()->incUsedRows();
@@ -3000,6 +3013,18 @@ ExWorkProcRetcode ExHbaseUMDtrafSubsetTaskTcb::work(short &rc)
break;
}
+ // delete entries from LOB desc table, if needed
+ if (tcb_->lobDelExpr())
+ {
+ ex_expr::exp_return_type exprRetCode =
+ tcb_->lobDelExpr()->eval(pentry_down->getAtp(), tcb_->workAtp_);
+ if (exprRetCode == ex_expr::EXPR_ERROR)
+ {
+ step_ = HANDLE_ERROR;
+ break;
+ }
+ }
+
if (tcb_->getHbaseAccessStats())
tcb_->getHbaseAccessStats()->incUsedRows();
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/57ae601c/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index a4cf509..6062775 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -428,7 +428,7 @@ class ExLob
hdfsFile *getfdDesc() { return &fdDesc_;}
char *getDescFileName() { return lobDescFile_; }
char *getDataFileName() { return lobDataFile_; }
-
+ Ex_Lob_Error getNextFreeOffset(Int64 size, Int64 &dataOffset, Int64 lobMaxSize);
Ex_Lob_Error print();
int getErrNo();
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/57ae601c/core/sql/exp/ExpLOBprocess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBprocess.cpp b/core/sql/exp/ExpLOBprocess.cpp
index cabc7c3..e121c1e 100644
--- a/core/sql/exp/ExpLOBprocess.cpp
+++ b/core/sql/exp/ExpLOBprocess.cpp
@@ -509,6 +509,19 @@ Ex_Lob_Error ExLob::purgeLob()
return LOB_OPER_OK;
}
+
+Ex_Lob_Error ExLob::getNextFreeOffset(Int64 size, Int64 &dataOffset, Int64 lobMaxSize)
+{
+ dataOffset = 0;
+ // Retrieve the next available, best-fit free offset to write into.
+
+ if (sourceLen <= 0 || sourceLen > lobMaxSize)
+ {
+ return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size
+ }
+}
+
+
/*Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset)
{
Ex_Lob_Error err = LOB_OPER_OK;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/57ae601c/core/sql/generator/GenPreCode.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp
index 1adce80..c720b0a 100644
--- a/core/sql/generator/GenPreCode.cpp
+++ b/core/sql/generator/GenPreCode.cpp
@@ -4837,7 +4837,7 @@ RelExpr * HbaseDelete::preCodeGen(Generator * generator,
}
NABoolean isAlignedFormat = getTableDesc()->getNATable()->isAlignedFormat(getIndexDesc());
- if (producesOutputs() || getTableDesc()->getNATable()->hasLobColumn())
+ if (producesOutputs())
{
retColRefSet_ = getIndexDesc()->getIndexColumns();
}
@@ -4903,6 +4903,15 @@ RelExpr * HbaseDelete::preCodeGen(Generator * generator,
// value is needed to retrieve a row.
HbaseAccess::addColReferenceFromVIDlist(getIndexDesc()->getIndexKey(), retColRefSet_);
}
+
+ if (getTableDesc()->getNATable()->hasLobColumn())
+ {
+ for (Lng32 i = 0; i < getIndexDesc()->getIndexColumns().entries(); i++)
+ {
+ const ValueId vid = getIndexDesc()->getIndexColumns()[i];
+ retColRefSet_.insert(vid);
+ }
+ }
}
NABoolean inlinedActions = FALSE;
@@ -4928,8 +4937,9 @@ RelExpr * HbaseDelete::preCodeGen(Generator * generator,
uniqueHbaseOper() = FALSE;
if ((generator->oltOptInfo()->multipleRowsReturned()) &&
(CmpCommon::getDefault(HBASE_ROWSET_VSBB_OPT) == DF_ON) &&
- (NOT generator->isRIinliningForTrafIUD()))
- uniqueRowsetHbaseOper() = TRUE;
+ (NOT generator->isRIinliningForTrafIUD()) &&
+ (NOT getTableDesc()->getNATable()->hasLobColumn()))
+ uniqueRowsetHbaseOper() = TRUE;
}
else
if (isUnique)
@@ -4946,7 +4956,8 @@ RelExpr * HbaseDelete::preCodeGen(Generator * generator,
{
if ((generator->oltOptInfo()->multipleRowsReturned()) &&
(CmpCommon::getDefault(HBASE_ROWSET_VSBB_OPT) == DF_ON) &&
- (NOT generator->isRIinliningForTrafIUD()))
+ (NOT generator->isRIinliningForTrafIUD()) &&
+ (NOT getTableDesc()->getNATable()->hasLobColumn()))
uniqueRowsetHbaseOper() = TRUE;
else if ((NOT generator->oltOptInfo()->multipleRowsReturned()) &&
(listOfDelUniqueRows_.entries() == 0))
@@ -4967,6 +4978,12 @@ RelExpr * HbaseDelete::preCodeGen(Generator * generator,
generator->oltOptInfo()->setOltCliOpt(FALSE);
}
+ if (getTableDesc()->getNATable()->hasLobColumn())
+ {
+ canDoCheckAndUpdel() = FALSE;
+ uniqueRowsetHbaseOper() = FALSE;
+ }
+
generator->setUpdSavepointOnError(FALSE);
generator->setUpdPartialOnError(FALSE);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/57ae601c/core/sql/generator/GenRelUpdate.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelUpdate.cpp b/core/sql/generator/GenRelUpdate.cpp
index 3aedb68..3c29d54 100644
--- a/core/sql/generator/GenRelUpdate.cpp
+++ b/core/sql/generator/GenRelUpdate.cpp
@@ -892,6 +892,7 @@ short HbaseDelete::codeGen(Generator * generator)
ULng32 convertRowLen = 0;
+ ValueIdList lobDelVIDlist;
for (CollIndex ii = 0; ii < numColumns; ii++)
{
ItemExpr * col_node = ((columnList[ii]).getValueDesc())->getItemExpr();
@@ -918,6 +919,15 @@ short HbaseDelete::codeGen(Generator * generator)
castValue->bindNode(generator->getBindWA());
convertExprCastVids.insert(castValue->getValueId());
+
+ if (col_node->getValueId().getType().isLob())
+ {
+ ItemExpr * ld = new(generator->wHeap())
+ LOBdelete(castValue);
+ ld->bindNode(generator->getBindWA());
+ lobDelVIDlist.insert(ld->getValueId());
+ }
+
} // for (ii = 0; ii < numCols; ii++)
// Add ascii columns to the MapTable. After this call the MapTable
@@ -1061,7 +1071,39 @@ short HbaseDelete::codeGen(Generator * generator)
expGen->generateExpr(newPredTree->getValueId(), ex_expr::exp_SCAN_PRED,
&scanExpr);
}
-
+
+ ex_expr * lobDelExpr = NULL;
+ if (getTableDesc()->getNATable()->hasLobColumn())
+ {
+ // generate code to delete rows from LOB desc table
+ expGen->generateListExpr(lobDelVIDlist,
+ ex_expr::exp_ARITH_EXPR, &lobDelExpr);
+ }
+
+#ifdef __ignore
+ ex_expr * lobDelExpr = NULL;
+ if (getTableDesc()->getNATable()->hasLobColumn())
+ {
+ // generate code to delete rows from LOB desc table
+
+ ValueIdList lobDelVIDlist;
+ for (Lng32 i = 0; i < getIndexDesc()->getIndexColumns().entries(); i++)
+ {
+ const ValueId vid = getIndexDesc()->getIndexColumns()[i];
+ if (vid.getType().isLob())
+ {
+ ItemExpr * ld = new(generator->wHeap())
+ LOBdelete(vid.getItemExpr());
+ ld->bindNode(generator->getBindWA());
+ lobDelVIDlist.insert(ld->getValueId());
+ }
+ }
+
+ expGen->generateListExpr(lobDelVIDlist,
+ ex_expr::exp_ARITH_EXPR, &lobDelExpr);
+ }
+#endif
+
ULng32 rowIdAsciiRowLen = 0;
ExpTupleDesc * rowIdAsciiTupleDesc = 0;
ex_expr * rowIdExpr = NULL;
@@ -1220,7 +1262,7 @@ short HbaseDelete::codeGen(Generator * generator)
scanExpr,
rowIdExpr,
NULL, // updateExpr
- NULL, // mergeInsertExpr
+ lobDelExpr, // NULL, // mergeInsertExpr
NULL, // mergeInsertRowIdExpr
NULL, // mergeUpdScanExpr
NULL, // projExpr
[11/11] incubator-trafodion git commit: Merge remote branch
'origin/pr/363/head' into merge_pr363
Posted by sa...@apache.org.
Merge remote branch 'origin/pr/363/head' into merge_pr363
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/5a6549b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/5a6549b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/5a6549b2
Branch: refs/heads/master
Commit: 5a6549b23e28c0195cbe2653f6f7b306574a452d
Parents: a1bc465 0ba0df9
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Wed Mar 9 18:43:31 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Wed Mar 9 18:43:31 2016 +0000
----------------------------------------------------------------------
.../sqf/conf/log4cxx.trafodion.masterexe.config | 1 +
core/sqf/sql/scripts/gensq.pl | 35 +-
core/sqf/sql/scripts/sqstart | 3 +-
core/sqf/sql/scripts/sqstop | 2 +-
core/sql/cli/Cli.cpp | 442 ++-
core/sql/cli/Cli.h | 15 +-
core/sql/cli/CliExpExchange.cpp | 18 +-
core/sql/cli/CliExtern.cpp | 54 +-
core/sql/cli/Descriptor.cpp | 6 +-
core/sql/comexe/ComTdbHbaseAccess.cpp | 2 +-
core/sql/comexe/ComTdbHbaseAccess.h | 2 +-
core/sql/common/CharType.cpp | 3 +-
core/sql/common/CharType.h | 2 +-
core/sql/executor/ExExeUtilLoad.cpp | 3 +-
core/sql/executor/ExHbaseAccess.h | 3 +
core/sql/executor/ExHbaseIUD.cpp | 27 +-
core/sql/exp/ExpLOB.cpp | 142 +-
core/sql/exp/ExpLOB.h | 41 +-
core/sql/exp/ExpLOBaccess.cpp | 2905 +++++++++---------
core/sql/exp/ExpLOBaccess.h | 288 +-
core/sql/exp/ExpLOBenums.h | 5 +-
core/sql/exp/ExpLOBexternal.h | 20 +-
core/sql/exp/ExpLOBinterface.cpp | 291 +-
core/sql/exp/ExpLOBinterface.h | 24 +-
core/sql/exp/ExpLOBprocess.cpp | 494 +--
core/sql/exp/ExpLOBprocess.h | 2 +-
core/sql/generator/GenExpGenerator.cpp | 3 +-
core/sql/generator/GenItemFunc.cpp | 17 +-
core/sql/generator/GenPreCode.cpp | 27 +-
core/sql/generator/GenRelUpdate.cpp | 47 +-
core/sql/optimizer/ItemFunc.h | 17 +-
core/sql/optimizer/NATable.cpp | 5 +-
core/sql/regress/executor/EXPECTED130 | 106 +-
core/sql/regress/executor/TEST130 | 32 +
core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 7 +-
core/sql/sqlcomp/CmpSeabaseDDLindex.cpp | 2 +-
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 25 +-
core/sql/sqlcomp/DefaultConstants.h | 1 +
core/sql/sqlcomp/nadefaults.cpp | 12 +-
39 files changed, 2759 insertions(+), 2372 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a6549b2/core/sqf/sql/scripts/gensq.pl
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a6549b2/core/sql/generator/GenPreCode.cpp
----------------------------------------------------------------------
[05/11] incubator-trafodion git commit: Changes for performing GC.
Changes to pass in hdfsServer and port via CQD and remove hardcoding. Changes
to pass in lobLocation via CQD and remove hardcoding. Changes to remove all
mxlobsrvr process functionality
Posted by sa...@apache.org.
Changes for performing GC.
Changes to pass in hdfsServer and port via CQD and remove hardcoding.
Changes to pass in lobLocation via CQD and remove hardcoding.
Changes to remove all mxlobsrvr process functionality from ExLOBprocess.cpp.
Retaining only the infrasturcture so mxlobsrvr process can be used in the
future. The executable is still built but the process is not started as part of
Trafodion anymore. We may use it in the future.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/c29103fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/c29103fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/c29103fd
Branch: refs/heads/master
Commit: c29103fdd67342631bd5b0b3c01ab8295e46a1d8
Parents: 57ae601
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Mon Feb 29 06:12:17 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Mon Feb 29 06:12:17 2016 +0000
----------------------------------------------------------------------
core/sqf/conf/log4j.dtm.config | 2 +-
core/sqf/conf/log4j.hdfs.config | 4 +-
core/sqf/sql/scripts/gensq.pl | 17 -
core/sqf/sql/scripts/sqstart | 3 +-
core/sqf/sql/scripts/sqstop | 2 +-
core/sql/cli/Cli.cpp | 558 +++--
core/sql/cli/Cli.h | 15 +-
core/sql/cli/CliExpExchange.cpp | 16 +-
core/sql/cli/CliExtern.cpp | 54 +-
core/sql/common/CharType.cpp | 1 +
core/sql/common/CharType.h | 2 +-
core/sql/executor/ExExeUtilLoad.cpp | 3 +-
core/sql/exp/ExpLOB.cpp | 142 +-
core/sql/exp/ExpLOB.h | 41 +-
core/sql/exp/ExpLOBaccess.cpp | 2920 +++++++++++++-----------
core/sql/exp/ExpLOBaccess.h | 288 ++-
core/sql/exp/ExpLOBenums.h | 5 +-
core/sql/exp/ExpLOBexternal.h | 20 +-
core/sql/exp/ExpLOBinterface.cpp | 291 +--
core/sql/exp/ExpLOBinterface.h | 24 +-
core/sql/exp/ExpLOBprocess.cpp | 507 +---
core/sql/exp/ExpLOBprocess.h | 2 +-
core/sql/generator/GenItemFunc.cpp | 17 +-
core/sql/optimizer/ItemFunc.h | 17 +-
core/sql/optimizer/NATable.cpp | 5 +-
core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 7 +-
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 10 +-
core/sql/sqlcomp/DefaultConstants.h | 1 +
core/sql/sqlcomp/nadefaults.cpp | 12 +-
29 files changed, 2489 insertions(+), 2497 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/conf/log4j.dtm.config
----------------------------------------------------------------------
diff --git a/core/sqf/conf/log4j.dtm.config b/core/sqf/conf/log4j.dtm.config
index 69bea7e..b7fdb16 100644
--- a/core/sqf/conf/log4j.dtm.config
+++ b/core/sqf/conf/log4j.dtm.config
@@ -53,5 +53,5 @@ log4j.appender.hbaseclient.Append=true
log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.hadoop.hbase=ERROR
log4j.logger.org.trafodion.dtm=INFO
-#log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
+log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/conf/log4j.hdfs.config
----------------------------------------------------------------------
diff --git a/core/sqf/conf/log4j.hdfs.config b/core/sqf/conf/log4j.hdfs.config
index 9505331..0a0ac49 100644
--- a/core/sqf/conf/log4j.hdfs.config
+++ b/core/sqf/conf/log4j.hdfs.config
@@ -50,5 +50,5 @@ log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.hadoop.hbase=ERROR
log4j.logger.org.trafodion.sql=ERROR
-#log4j.logger.org.apache.hadoop.hbase=DEBUG
-#log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/sql/scripts/gensq.pl
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/gensq.pl b/core/sqf/sql/scripts/gensq.pl
index e826c99..ea18311 100755
--- a/core/sqf/sql/scripts/gensq.pl
+++ b/core/sqf/sql/scripts/gensq.pl
@@ -479,24 +479,7 @@ sub genComponentWait {
printScript(1, "fi\n");
}
-sub genLOBConfig {
- # Generate sqconfig.db config for LOB.
- # This allows the process startup daemon (pstartd)
- # to start it up after a node failure.
- for ($i=0; $i < $gdNumNodes; $i++) {
- my $l_progname="mxlobsrvr";
- my $l_procargs="";
- my $l_procname="\$ZLOBSRV$i";
- my $l_procname_config = sprintf('$ZLOBSRV%d', $i);
- my $l_stdout="stdout_\$ZLOBSRV_$i";
- addDbProcData($l_procname_config, "PERSIST_RETRIES", "10,60");
- addDbProcData($l_procname_config, "PERSIST_ZONES", $i);
- addDbPersistProc($l_procname_config, $i, 1);
- addDbProcDef( $ProcessType_Generic, $l_procname_config, $i, $l_progname, $l_stdout, $l_procargs);
- }
-
-}
sub genIdTmSrv {
if ($SQ_IDTMSRV > 0) {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/sql/scripts/sqstart
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/sqstart b/core/sqf/sql/scripts/sqstart
index 7c63eb1..93e49bd 100755
--- a/core/sqf/sql/scripts/sqstart
+++ b/core/sqf/sql/scripts/sqstart
@@ -542,7 +542,8 @@ dcsstart
sleep 10
sqcheck | tee -a $SQMON_LOG
# Start the LOB Service
-lobstart
+#comment out since it's unused right now
+#lobstart
# Start the REST Service
reststart
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/sql/scripts/sqstop
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/sqstop b/core/sqf/sql/scripts/sqstop
index 55cfa84..e4ecbb1 100755
--- a/core/sqf/sql/scripts/sqstop
+++ b/core/sqf/sql/scripts/sqstop
@@ -92,7 +92,7 @@ reststop
dcsstop
# Stop the lob Service
-lobstop
+#lobstop
# Issue the shutdown request
echoLog "Shutting down ($shutdowntype) the SQ environment!"
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index 018bbfa..e762714 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -94,6 +94,7 @@
#include "LmLangManagerJava.h"
#include "LmRoutine.h"
#include "CmpDDLCatErrorCodes.h"
+#include "ExpLOBaccess.h"
#define DISPLAY_DONE_WARNING 1032
extern Lng32 getTotalTcbSpace(char * tdb, char * otherInfo,
@@ -9394,7 +9395,7 @@ Lng32 SQLCLI_LOBcliInterface
/*IN*/ char * inLobHandle,
/*IN*/ Lng32 inLobHandleLen,
/*IN*/ char * blackBox,
- /*INOUT*/ Lng32 *blackBoxLen,
+ /*INOUT*/ Int32 *blackBoxLen,
/*OUT*/ char * outLobHandle,
/*OUT*/ Lng32 * outLobHandleLen,
/*IN*/ LOBcliQueryType qType,
@@ -9403,8 +9404,9 @@ Lng32 SQLCLI_LOBcliInterface
OUT: for select */
/*INOUT*/ Int64 * dataLen, /* length of data.
IN: for insert, out: for select */
- /*OUT*/ Int64 * outDescPartnKey, /* returned after insert and select */
- /*OUT*/ Int64 * outDescSyskey, /* returned after insert and select */
+ /*INOUT*/ Int64 * inoutDescPartnKey, /* returned after insert and select
+
+ /*INOUT*/ Int64 * inoutDescSyskey, /* returned after insert and select
/*INOUT*/ void* *inCliInterface,
/*IN*/ Int64 xnId /* xn id of the parent process, if non-zero */
)
@@ -9459,13 +9461,7 @@ Lng32 SQLCLI_LOBcliInterface
ExpLOBoper::ExpGetLOBDescChunksName(schNameLen, schName, uid, lobNum,
lobDescChunksNameBuf, lobDescChunksNameLen);
- char lobOffsetsNameBuf[1024];
- Lng32 lobOffsetsNameLen = 1024;
- char * lobOffsetsName =
- ExpLOBoper::ExpGetLOBOffsetsName(schNameLen, schName, uid, lobNum,
- lobOffsetsNameBuf, lobOffsetsNameLen);
-
-
+
char * query = new(currContext.exHeap()) char[4096];
if (outLobHandleLen)
@@ -9499,52 +9495,6 @@ Lng32 SQLCLI_LOBcliInterface
SQL_EXEC_Xact(SQLTRANS_INHERIT, NULL);
}
- /* if ((qType == LOB_CLI_CREATE) ||
- (qType == LOB_CLI_DROP) ||
- (qType == LOB_CLI_INSERT) ||
- (qType == LOB_CLI_INSERT_APPEND) ||
- (qType == LOB_CLI_UPDATE_UNIQUE) ||
- (qType == LOB_CLI_DELETE) ||
- (qType == LOB_CLI_PURGEDATA))
- */
- if (0)
- {
- Int64 transId;
- cliRC = GETTRANSID((short *)&transId);
-
- if (xnId == 0) // no xn passed in
- {
- if (cliRC == 75) // no transid present
- {
- strcpy(query, "set transaction autocommit on;");
- cliRC = cliInterface->executeImmediate(query);
-
- xnAutoCommit = TRUE;
- }
- }
- else
- {
- if ((cliRC == 0) && // transid present
- (xnId == transId))
- {
- // Same as the current transaction in progress.
- // do nothing.
- }
- else
- {
- cliRC = JOINTRANSACTION(xnId);
- if (cliRC)
- return -EXE_BEGIN_TRANSACTION_ERROR;
-
- cliRC = GETTRANSID((short *)&transId);
- if ((cliRC) ||
- (xnId != transId))// transid present
- return -EXE_BEGIN_TRANSACTION_ERROR;
-
- xnJoined = TRUE;
- }
- }
- }
switch (qType)
{
@@ -9588,7 +9538,7 @@ Lng32 SQLCLI_LOBcliInterface
}
// create lob descriptor chunks table salted
- str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, descSysKey largeint not null, chunkNum int not null, chunkLen largeint not null, intParam largeint, stringParam varchar(400), primary key(descPartnKey, descSysKey, chunkNum)) salt using 8 partitions",
+ str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, descSysKey largeint not null, chunkNum int not null, chunkLen largeint not null, dataOffset largeint, stringParam varchar(400), primary key(descPartnKey, descSysKey, chunkNum)) salt using 8 partitions",
lobDescChunksName);
@@ -9606,30 +9556,6 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
-
-
-
- // create lob datafile descriptor unaudited table. Use a cqd
- // traf_no_dtm_xn while doing any dml to this table.
- //dataState = 1 - indicates used chunk
- //dataState = 0 - indicates unused chunk
- str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, descSysKey largeint not null, chunkNum int not null, chunkLen largeint not null, offset largeint, dataState int , primary key(descPartnKey, descSysKey, chunkLen)) ",
- lobOffsetsName);
-
-
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
-
- currContext.resetSqlParserFlags(0x1);x
-
- if (cliRC < 0)
- {
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
cliRC = 0;
}
@@ -9674,25 +9600,8 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
- str_sprintf(query, "drop ghost table %s",
- lobOffsetsName);
-
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC == -CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION)
- cliRC = 0;
-
- if (cliRC < 0)
- {
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
+
cliRC = 0;
}
@@ -9741,9 +9650,7 @@ Lng32 SQLCLI_LOBcliInterface
break;
}
goto error_return;
- }
-
-
+ }
cliRC = 0;
}
break;
@@ -9802,33 +9709,11 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
- //Insert into the lob offsets table (without transaction)
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
- str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld,1, %Ld, %Ld, 1)",
- lobOffsetsName, descPartnKey, descSyskey,
- (dataLen ? *dataLen : 0),
- (dataOffset ? *dataOffset : 0) );
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
-
- if (outDescPartnKey)
- *outDescPartnKey = descPartnKey;
-
- if (outDescSyskey)
- *outDescSyskey = descSyskey;
+ if (inoutDescSyskey)
+ *inoutDescSyskey = descSyskey;
// update lob handle with the returned values
if (outLobHandle)
@@ -9839,7 +9724,9 @@ Lng32 SQLCLI_LOBcliInterface
if (outLobHandleLen)
*outLobHandleLen = inLobHandleLen;
}
+
+
cliRC = 0;
}
break;
@@ -9915,30 +9802,14 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
- //Insert into the lob offsets table (without transaction)
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
- str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld,%d, %Ld, %Ld, 1)",
- lobOffsetsName, descPartnKey, descSyskey,numChunks
- (dataLen ? *dataLen : 0),
- (dataOffset ? *dataOffset : 0) );
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
-
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
+ if (inoutDescSyskey)
+ *inoutDescSyskey = inDescSyskey;
+ if(blackBoxLen)
+ *blackBoxLen = numChunks;
cliRC = 0;
// if (outDescSyskey)
@@ -10020,52 +9891,7 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
-
- // Update the lob offsets table. Set all the previous chunks to be
- // unused. Insert the new chunk as the new used chunk.
-
-
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
-
- //update old chunks as unused
-
- str_sprintf(query, "update table(ghost table %s) set dataState = 0 where descPartnKey = %Ld and descSyskey = %Ld",lobOffsetsName, descPartnKey, inDescSyskey);
-
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
- cliRC = cliInterface->executeImmediate(query);
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
-
- //Insert new chunk offset and length
- str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld,1, %Ld, %Ld, 1)",
- lobOffsetsName, descPartnKey, descSyskey,
- (dataLen ? *dataLen : 0),
- (dataOffset ? *dataOffset : 0) );
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
-
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
-
+
// update lob handle with the returned values
if (outLobHandle)
{
@@ -10075,7 +9901,11 @@ Lng32 SQLCLI_LOBcliInterface
if (outLobHandleLen)
*outLobHandleLen = inLobHandleLen;
}
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
+ if (inoutDescSyskey)
+ *inoutDescSyskey = inDescSyskey;
cliRC = 0;
}
break;
@@ -10117,30 +9947,7 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
-
- //Delete from lob offsets table (without transaction)
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
- str_sprintf(query, "update table(ghost table %s) set dataState = 0 where descPartnKey = %Ld and descSyskey = %Ld",lobOffsetsName, descPartnKey, inDescSyskey);
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
-
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
-
- cliRC = 0;
- // if (outDescSyskey)
- // *outDescSyskey = inDescSyskey;
+
}
break;
@@ -10200,7 +10007,7 @@ Lng32 SQLCLI_LOBcliInterface
}
// This lob has only one chunk. Read and return the single descriptor.
- str_sprintf(query, "select c.chunkLen, c.intParam from table(ghost table %s) h, table(ghost table %s) c where h.descPartnKey = c.descPartnKey and h.syskey = c.descSyskey and h.descPartnKey = %Ld and h.syskey = %Ld and c.chunkNum = h.numChunks for read committed access",
+ str_sprintf(query, "select c.chunkLen, c.dataOffset from table(ghost table %s) h, table(ghost table %s) c where h.descPartnKey = c.descPartnKey and h.syskey = c.descSyskey and h.descPartnKey = %Ld and h.syskey = %Ld and c.chunkNum = h.numChunks for read committed access",
lobDescHandleName, lobDescChunksName,
descPartnKey, inDescSyskey);
@@ -10243,11 +10050,11 @@ Lng32 SQLCLI_LOBcliInterface
if (dataOffset)
str_cpy_all((char*)dataOffset, ptr, len);
- if (outDescPartnKey)
- *outDescPartnKey = descPartnKey;
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
- if (outDescSyskey)
- *outDescSyskey = inDescSyskey;
+ if (inoutDescSyskey)
+ *inoutDescSyskey = inDescSyskey;
}
// else
//cliRC = -100;
@@ -10268,7 +10075,7 @@ Lng32 SQLCLI_LOBcliInterface
case LOB_CLI_SELECT_CURSOR:
{
- str_sprintf(query, "select intParam, chunkLen from table(ghost table %s) where descPartnKey = %Ld and descSyskey = %Ld order by chunkNum for read committed access",
+ str_sprintf(query, "select dataOffset, chunkLen from table(ghost table %s) where descPartnKey = %Ld and descSyskey = %Ld order by chunkNum for read committed access",
lobDescChunksName, descPartnKey, inDescSyskey);
// set parserflags to allow ghost table
@@ -10395,7 +10202,7 @@ Lng32 SQLCLI_LOBcliInterface
//aggregate on chunklen for this lob.
- str_sprintf (query, "select sum(chunklen) from %s where descpartnkey = %Ld and descsyskey = %Ld ", lobDescChunksName, descPartnKey, inDescSyskey );
+ str_sprintf (query, "select sum(chunklen) from %s where descpartnkey = %Ld and descsyskey = %Ld ", lobDescChunksName, descPartnKey, inDescSyskey );
// set parserflags to allow ghost table
currContext.setSqlParserFlags(0x1);
@@ -10403,11 +10210,11 @@ Lng32 SQLCLI_LOBcliInterface
Int64 outlen = 0;Lng32 len = 0;
cliRC = cliInterface->executeImmediate(query,(char *)dataLen, &len, FALSE);
- if (outDescPartnKey)
- *outDescPartnKey = descPartnKey;
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
- if (outDescSyskey)
- *outDescSyskey = inDescSyskey;
+ if (inoutDescSyskey)
+ *inoutDescSyskey = inDescSyskey;
@@ -10424,6 +10231,7 @@ Lng32 SQLCLI_LOBcliInterface
cliRC = saveCliErr;
}
break;
+
} // switch
// normal return. Fall down to deallocate of structures.
@@ -10464,10 +10272,274 @@ Lng32 SQLCLI_LOBcliInterface
else
return 0;
}
+/* The process for GC is as follows:
+1. Check LOB descriptor chunks table (per column) for any holes in the LOB sections.
+2. Create an in memory array that can be used to calculate new offsets.
+3. Update the lob descriptor chunks table to reflect the new offsets
+ if any error, return an error and rollback the user transaction.
+4. Compact the lob data file buy doing hte following :
+ a) Save a copy of the LOB data file.
+ b) Create a temp lob file to copy the sections contiguously from the lob data file to the temp file.
+ If any error return an error and rollback all updates to the lob descriptor chunks table.
+ c) Delete the lob data file.
+ d) Rename the tempfile to original lob data file name.
+ If any error, restore the lob data file from the saved backup in step (a)
+ e) Purge the saved backup.
+*/
+
+
+Lng32 SQLCLI_LOB_GC_Interface
+(
+ /*IN*/ CliGlobals *cliGlobals,
+ /*IN*/ void *lobGlobals, // can be passed or NULL
+ /*IN*/ char * handle,
+ /*IN*/ Lng32 handleLen,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
+ /*IN*/ char *lobLocation,
+ /*IN*/ Int64 lobMaxMemChunkLen // if passed in as 0, will use default value of 1G for the in memory buffer to do compaction.
+ )
+{
+ Lng32 cliRC = 0;
+
+ ContextCli & currContext = *(cliGlobals->currContext());
+ ComDiagsArea & diags = currContext.diags();
+
+ ComDiagsArea * myDiags = ComDiagsArea::allocate(currContext.exHeap());
+
+ ExeCliInterface *cliInterface = NULL;
+ cliInterface = new (currContext.exHeap())
+ ExeCliInterface(currContext.exHeap(),
+ SQLCHARSETCODE_UTF8,
+ &currContext,
+ NULL);
+ Int32 rc = 0;
+ Int16 flags = 0;
+ Lng32 lobType = 1;
+ Lng32 lobNum = 0;
+ Int64 uid = 0;
+ Int64 inDescSyskey, inDescPartnKey;
+ short schNameLen = 0;
+ char schName[512];
+ if (handle)
+ {
+ ExpLOBoper::extractFromLOBhandle(&flags, &lobType, &lobNum, &uid,
+ &inDescSyskey, &inDescPartnKey,
+ &schNameLen, schName,
+ handle);
+ }
+
+ char tgtLobNameBuf[100];
+ char * tgtLobName =
+ ExpLOBoper::ExpGetLOBname(uid, lobNum, tgtLobNameBuf, 100);
+
+ char lobDescHandleNameBuf[1024];
+ Lng32 lobDescHandleNameLen = 1024;
+ char * lobDescHandleName =
+ ExpLOBoper::ExpGetLOBDescHandleName(schNameLen, schName, uid, lobNum,
+ lobDescHandleNameBuf, lobDescHandleNameLen);
+
+ char lobDescChunksNameBuf[1024];
+ Lng32 lobDescChunksNameLen = 1024;
+ char * lobDescChunksName =
+ ExpLOBoper::ExpGetLOBDescChunksName(schNameLen, schName, uid, lobNum,
+ lobDescChunksNameBuf, lobDescChunksNameLen);
+
+
+ char * query = new(currContext.exHeap()) char[4096];
+ //Find how many entries in the descchunks table to allocate
+ //in memory array.
+ str_sprintf(query, "select count(*) from table(ghost table %s) ",
+ lobDescChunksName);
+
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ Lng32 numEntries = 0;
+ Lng32 len;
+ cliRC = cliInterface->executeImmediate(query, (char*)&numEntries, &len, FALSE);
+
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+ {
+ //Allocate an inmemory array of numEntries.
+ ExLobInMemoryDescChunksEntry *dcInMemoryArray = new ExLobInMemoryDescChunksEntry[numEntries];
+ //Read the desc chunks table into memory
+
+ str_sprintf(query, "select dataOffset, descPartnKey,descSyskey,chunkLen,chunkNum from table(ghost table %s) order by dataOffset for read committed access",
+ lobDescChunksName);
+
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->fetchRowsPrologue(query);
+
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+ cliRC = cliInterface->fetch();
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ cliInterface->fetchRowsEpilogue(0);
+
+ goto error_return;
+ }
+
+ short i = 0;
+ Int64 currentOffset = 0;
+ Int64 descPartnKey = 0;
+ Int64 descSyskey = 0;
+ Int64 chunkLen = 0;
+ Int64 chunkNum = 0;
+ while (cliRC != 100)
+ {
+ char * ptr;
+ Lng32 len;
+
+ cliInterface->getPtrAndLen(1, ptr, len);
+ str_cpy_all((char*)¤tOffset, ptr, len);
+
+ cliInterface->getPtrAndLen(2, ptr, len);
+ str_cpy_all((char*)&descPartnKey, ptr, len);
+
+ cliInterface->getPtrAndLen(3, ptr, len);
+ str_cpy_all((char*)&descSyskey, ptr, len);
+
+ cliInterface->getPtrAndLen(4, ptr, len);
+ str_cpy_all((char*)&chunkLen, ptr, len);
+
+ cliInterface->getPtrAndLen(5, ptr, len);
+ str_cpy_all((char*)&chunkNum, ptr, len);
+
+ dcInMemoryArray[i].setCurrentOffset(currentOffset);
+ dcInMemoryArray[i].setNewOffset(currentOffset);
+ dcInMemoryArray[i].setDescPartnKey(descPartnKey);
+ dcInMemoryArray[i].setSyskey(descSyskey);
+ dcInMemoryArray[i].setChunkLen(chunkLen);
+ dcInMemoryArray[i].setChunkNum(chunkNum);
+ cliRC = cliInterface->fetch();
+ i++;
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ cliInterface->fetchRowsEpilogue(0);
+
+ goto error_return;
+ }
+ }
+
+ cliRC = cliInterface->fetchRowsEpilogue(0);
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+
+ // adjust in memory array to calculate holes and new offsets.
+ ExpLOBoper::calculateNewOffsets(dcInMemoryArray,numEntries);
+
+
+ // Update the descChunks table with new offsets
+ //TBD start a separate transaction to manage this.
+ //For now use the transaction associated with the IUD operation
+ //that triggered this GC
+
+ i = 0;
+ while (i < numEntries)
+ {
+ if (dcInMemoryArray[i].getCurrentOffset() == dcInMemoryArray[i].getNewOffset())
+ i++;
+ else
+ {
+ str_sprintf(query, "update table(ghost table %s) set dataOffset=%Ld, chunkLen = %Ld where descPartnKey = %Ld and descSysKey = %Ld",
+ lobDescChunksName,
+ dcInMemoryArray[i].getNewOffset(),
+ dcInMemoryArray[i].getChunkLen(),
+ dcInMemoryArray[i].getDescPartnKey(),
+ dcInMemoryArray[i].getSyskey());
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->executeImmediate(query);
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ //tbd Give warning and rollback transaction and return with warning. For now return error and abort the iud operation itself.
+ goto error_return;
+ }
+ i++;
+ }
+ }
+
+ // Compact into new temp file
+
+
+ rc = ExpLOBoper::compactLobDataFile(lobGlobals,dcInMemoryArray,numEntries,tgtLobName,lobMaxMemChunkLen, (void *)currContext.exHeap(), hdfsServer, hdfsPort,lobLocation);
+
+ if (rc )
+ {
+ cliRC = 9999; // Warning
+ ComDiagsArea * da = &diags;
+ ExRaiseSqlError(currContext.exHeap(), &da,
+ (ExeErrorCode)(8442), NULL, &cliRC ,
+ &rc, NULL, (char*)"Lob GC call",
+ getLobErrStr(rc));
+ // TBD When local transaction support is in
+ // rollback all the updates to the lob desc chunks file too.
+ // return with warning
+ // For now, return error for the IUD operation
+
+ // Restore original data file.
+ Int32 rc2=ExpLOBoper::restoreLobDataFile(lobGlobals,tgtLobName, (void *)currContext.exHeap(),hdfsServer,hdfsPort,lobLocation);
+ // if error restoring, mark this lob as corrupt.
+ goto error_return;
+ }
+ else
+ {
+ //TBD :commit all updates and remove the saved copy of datafile
+ ExpLOBoper::purgeBackupLobDataFile(lobGlobals, tgtLobName,(void *)currContext.exHeap(),hdfsServer,hdfsPort,lobLocation);
+ }
+ }
+ error_return:
+
+ Lng32 tempCliRC = 0;
+ NADELETEBASIC(query, currContext.exHeap());
+
+ if (cliRC < 0)
+ {
+ if (myDiags->getNumber() > 0)
+ {
+ diags.mergeAfter(*myDiags);
+ }
+ return cliRC;
+ }
+ else if (cliRC == 100)
+ return 100;
+ else
+ return 0;
+}
Lng32 SQLCLI_LOBddlInterface
(
- /*IN*/ CliGlobals *cliGlobals,
+/*IN*/ CliGlobals *cliGlobals,
/*IN*/ char * schName,
/*IN*/ Lng32 schNameLen,
/*IN*/ Int64 objectUID,
@@ -10476,8 +10548,11 @@ Lng32 SQLCLI_LOBddlInterface
/*IN*/ short *lobNumList,
/*IN*/ short *lobTypList,
/*IN*/ char* *lobLocList,
+ /*IN*/ char *hdfsServer,
+ /*IN*/ Int32 hdfsPort,
/*IN*/ Int64 lobMaxSize
)
+
{
Lng32 cliRC = 0;
@@ -10507,18 +10582,12 @@ Lng32 SQLCLI_LOBddlInterface
{
// create lob metadata table
- str_sprintf(query, "create ghost table %s (lobnum smallint not null, storagetype smallint not null, location varchar(4096) not null, primary key (lobnum)) location $system",
- // str_sprintf(query, "create ghost table %s (lobnum smallint not null, storagetype smallint not null, location varchar(4096) not null, primary key (lobnum)) no partition",
- lobMDName);
+ str_sprintf(query, "create ghost table %s (lobnum smallint not null, storagetype smallint not null, location varchar(4096) not null, primary key (lobnum)) ",lobMDName);
// set parserflags to allow ghost table
currContext.setSqlParserFlags(0x1);
- cliRC = cliInterface->executeImmediate(query);
-
-
-
-
+ cliRC = cliInterface->executeImmediate(query);
currContext.resetSqlParserFlags(0x1);
if (cliRC < 0)
@@ -10557,8 +10626,8 @@ Lng32 SQLCLI_LOBddlInterface
// create lob data tables
Lng32 rc = ExpLOBoper::createLOB
(NULL, currContext.exHeap(),
- lobLocList[i],
- objectUID, lobNumList[i], lobMaxSize);
+ lobLocList[i], hdfsPort,hdfsServer,
+ objectUID, lobNumList[i],lobMaxSize);
if (rc)
{
@@ -10567,6 +10636,7 @@ Lng32 SQLCLI_LOBddlInterface
ExRaiseSqlError(currContext.exHeap(), &da,
(ExeErrorCode)(8442), NULL, &cliRC ,
&rc, NULL, (char*)"ExpLOBInterfaceCreate",
+
getLobErrStr(rc));
goto error_return;
}
@@ -10631,7 +10701,7 @@ Lng32 SQLCLI_LOBddlInterface
{
Lng32 rc = ExpLOBoper::dropLOB
(NULL, currContext.exHeap(),
- lobLocList[i],
+ lobLocList[i],hdfsPort,hdfsServer,
objectUID, lobNumList[i]);
if (rc)
@@ -10703,7 +10773,7 @@ Lng32 SQLCLI_LOBddlInterface
{
Lng32 rc = ExpLOBoper::dropLOB
(NULL, currContext.exHeap(),
- lobLocList[i],
+ lobLocList[i],hdfsPort, hdfsServer,
objectUID, lobNumList[i]);
if (rc)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/cli/Cli.h
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.h b/core/sql/cli/Cli.h
index 9bffc5c..db96c87 100644
--- a/core/sql/cli/Cli.h
+++ b/core/sql/cli/Cli.h
@@ -895,7 +895,7 @@ Lng32 SQLCLI_LOBcliInterface
/*IN*/ char * inLobHandle,
/*IN*/ Lng32 inLobHandleLen,
/*IN*/ char * blackBox,
- /*IN*/ Lng32* blackBoxLen,
+ /*IN*/ Int32* blackBoxLen,
/*OUT*/ char * outLobHandle,
/*OUT*/ Lng32 * outLobHandleLen,
/*IN*/ LOBcliQueryType qType,
@@ -909,6 +909,17 @@ Lng32 SQLCLI_LOBcliInterface
/*INOUT*/ void* *inCliInterface,
/*IN*/ Int64 xnId /* xn id of the parent process, if non-zero */
);
+Lng32 SQLCLI_LOB_GC_Interface
+(
+ /*IN*/ CliGlobals *cliGlobals,
+ /*IN*/ void *lobGlobals, // can be passed or NULL
+ /*IN*/ char * handle,
+ /*IN*/ Lng32 handleLen,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
+ /*IN*/ char *lobLocation,
+ /*IN*/ Int64 lobMaxMemChunkLen // if passed in as 0, will use default value of 1G for the in memory buffer to do compaction.
+ );
Lng32 SQLCLI_LOBddlInterface
(
@@ -921,6 +932,8 @@ Lng32 SQLCLI_LOBddlInterface
/*IN*/ short *lobNumList,
/*IN*/ short *lobTypList,
/*IN*/ char* *lobLocList,
+ /*IN*/ char *hdfsServer,
+ /*IN*/ Int32 hdfsPort,
/*IN*/ Int64 lobMaxSize
);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/cli/CliExpExchange.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/CliExpExchange.cpp b/core/sql/cli/CliExpExchange.cpp
index 24853a3..f3b1967 100644
--- a/core/sql/cli/CliExpExchange.cpp
+++ b/core/sql/cli/CliExpExchange.cpp
@@ -207,14 +207,15 @@ ex_expr::exp_return_type InputOutputExpr::describeOutput(void * output_desc_,
// Use SQLDESC_CHAR_SET_NAM (one-part name) for charset
- if ( DFS2REC::isAnyCharacter(operand->getDatatype()) ) {
- output_desc->setDescItem(entry, SQLDESC_CHAR_SET_NAM, 0,
- (char*)CharInfo::getCharSetName(operand->getCharSet()));
+ if ( DFS2REC::isAnyCharacter(operand->getDatatype()) )
+ {
+ output_desc->setDescItem(entry, SQLDESC_CHAR_SET_NAM, 0,
+ (char*)CharInfo::getCharSetName(operand->getCharSet()));
- // reset the length for Unicode
+ // reset the length for Unicode
if ( operand->getCharSet() == CharInfo::UNICODE ||
CharInfo::is_NCHAR_MP(operand->getCharSet())
- )
+ )
{
length = operand->getLength()/SQL_DBCHAR_SIZE;
}
@@ -2240,8 +2241,9 @@ ex_expr::exp_return_type InputOutputExpr::describeInput(void * input_desc_,
}
// Use SQLDESC_CHAR_SET_NAM (one-part name) for charset
- if ((dataType >= REC_MIN_CHARACTER) &&
- (dataType <= REC_MAX_CHARACTER)) {
+ if (((dataType >= REC_MIN_CHARACTER) &&
+ (dataType <= REC_MAX_CHARACTER)) )
+ {
input_desc->setDescItem(entry, SQLDESC_CHAR_SET_NAM, 0,
(char*)CharInfo::getCharSetName(operand->getCharSet()));
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/cli/CliExtern.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/CliExtern.cpp b/core/sql/cli/CliExtern.cpp
index 127380f..e5dd796 100644
--- a/core/sql/cli/CliExtern.cpp
+++ b/core/sql/cli/CliExtern.cpp
@@ -7359,7 +7359,7 @@ SQLCLI_LIB_FUNC Lng32 SQL_EXEC_LOBcliInterface
/*IN*/ char * inLobHandle,
/*IN*/ Lng32 inLobHandleLen,
/*IN*/ char * blackBox,
- /*IN*/ Lng32* blackBoxLen,
+ /*IN*/ Int32* blackBoxLen,
/*OUT*/ char * outLobHandle,
/*OUT*/ Lng32 * outLobHandleLen,
/*IN*/ LOBcliQueryType qType,
@@ -7415,6 +7415,51 @@ SQLCLI_LIB_FUNC Lng32 SQL_EXEC_LOBcliInterface
RecordError(NULL, retcode);
return retcode;
}
+Lng32 SQL_EXEC_LOB_GC_Interface
+(
+ /*IN*/ void *lobGlobals, // can be passed or NULL
+ /*IN*/ char * handle,
+ /*IN*/ Lng32 handleLen,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
+ /*IN*/ char *lobLocation,
+ /*IN*/ Int64 lobMaxMemChunkLen // if passed in as 0, will use default value of 1G for the in memory buffer to do compaction.
+ )
+{
+ Lng32 retcode;
+ CLISemaphore *tmpSemaphore;
+ ContextCli *threadContext;
+ CLI_NONPRIV_PROLOGUE(retcode);
+ try
+ {
+ tmpSemaphore = getCliSemaphore(threadContext);
+ tmpSemaphore->get();
+ threadContext->incrNumOfCliCalls();
+ retcode = SQLCLI_LOB_GC_Interface(GetCliGlobals(),
+ lobGlobals,
+ handle,
+ handleLen,
+ hdfsServer,
+ hdfsPort,lobLocation,
+ lobMaxMemChunkLen);
+ }
+ catch(...)
+ {
+ retcode = -CLI_INTERNAL_ERROR;
+#if defined(_THROW_EXCEPTIONS)
+ if (cliWillThrow())
+ {
+ threadContext->decrNumOfCliCalls();
+ tmpSemaphore->release();
+ throw;
+ }
+#endif
+ }
+ threadContext->decrNumOfCliCalls();
+ tmpSemaphore->release();
+ RecordError(NULL, retcode);
+ return retcode;
+ }
Lng32 SQL_EXEC_LOBddlInterface
(
@@ -7426,6 +7471,8 @@ Lng32 SQL_EXEC_LOBddlInterface
/*IN*/ short *lobNumList,
/*IN*/ short *lobTypList,
/*IN*/ char* *lobLocList,
+ /*IN*/ char *hdfsServer,
+ /*IN*/ Int32 hdfsPort,
/*IN */ Int64 lobMaxSize
)
{
@@ -7446,7 +7493,10 @@ Lng32 SQL_EXEC_LOBddlInterface
qType,
lobNumList,
lobTypList,
- lobLocList, lobMaxSize);
+ lobLocList,
+ hdfsServer,
+ hdfsPort,
+ lobMaxSize);
}
catch(...)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/common/CharType.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/CharType.cpp b/core/sql/common/CharType.cpp
index 40d258d..1b21fd1 100644
--- a/core/sql/common/CharType.cpp
+++ b/core/sql/common/CharType.cpp
@@ -1536,6 +1536,7 @@ SQLBlob::SQLBlob(
externalFormat,
extFormatLen)
{
+ setCharSet(CharInfo::DefaultCharSet);
}
NAType *SQLBlob::newCopy(NAMemory* h) const
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/common/CharType.h
----------------------------------------------------------------------
diff --git a/core/sql/common/CharType.h b/core/sql/common/CharType.h
index f76f574..7d0c383 100644
--- a/core/sql/common/CharType.h
+++ b/core/sql/common/CharType.h
@@ -817,7 +817,7 @@ public:
NABoolean allowSQLnull = TRUE,
NABoolean inlineIfPossible = FALSE,
NABoolean externalFormat = FALSE,
- Lng32 extFormatLen = 100);
+ Lng32 extFormatLen = 1024);
SQLClob(const SQLClob & aClob,NAMemory * heap)
:SQLlob(aClob,heap)
{}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index 1db4bee..cc481a2 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -2980,7 +2980,8 @@ short ExExeUtilLobExtractTcb::work()
LOB_CLI_SELECT_UNIQUE,
lobNumList,
lobTypList,
- lobLocList,0);
+ lobLocList,lobTdb().getLobHdfsServer(),
+ lobTdb().getLobHdfsPort(),0);
if (cliRC < 0)
{
getDiagsArea()->mergeAfter(diags);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOB.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.cpp b/core/sql/exp/ExpLOB.cpp
index 00116b6..2b534ee 100644
--- a/core/sql/exp/ExpLOB.cpp
+++ b/core/sql/exp/ExpLOB.cpp
@@ -137,20 +137,7 @@ char * ExpLOBoper::ExpGetLOBDescChunksName(Lng32 schNameLen, char * schName,
return outBuf;
}
-char * ExpLOBoper::ExpGetLOBOffsetsName(Lng32 schNameLen, char * schName,
- Int64 uid, Lng32 num,
- char * outBuf, Lng32 outBufLen)
-{
- if ((outBufLen < 512) ||
- (schNameLen == 0) ||
- (schName == NULL))
- return NULL;
- str_sprintf(outBuf, "%s.\"LOBOffset_%020Ld_%04d\"",
- schName, uid, num);
-
- return outBuf;
-}
char * ExpLOBoper::ExpGetLOBMDName(Lng32 schNameLen, char * schName,
Int64 uid,
@@ -165,8 +152,8 @@ char * ExpLOBoper::ExpGetLOBMDName(Lng32 schNameLen, char * schName,
return outBuf;
}
Lng32 ExpLOBoper::createLOB(void * lobGlob, void * lobHeap,
- char * lobLoc,
- Int64 uid, Lng32 num, Int64 lobMaxSize)
+ char * lobLoc,Int32 hdfsPort,char *hdfsServer,
+ Int64 uid, Lng32 num, Int64 lobMaxSize )
{
char buf[100];
@@ -186,13 +173,114 @@ Lng32 ExpLOBoper::createLOB(void * lobGlob, void * lobHeap,
else
lobGlobL = lobGlob;
- rc = ExpLOBinterfaceCreate(lobGlobL, lobName, lobLoc, Lob_HDFS_File,(char *)"default",lobMaxSize);
+ rc = ExpLOBinterfaceCreate(lobGlobL, lobName, lobLoc, Lob_HDFS_File,hdfsServer,lobMaxSize, hdfsPort);
+
+ return rc;
+}
+void ExpLOBoper::calculateNewOffsets(ExLobInMemoryDescChunksEntry *dcArray, Lng32 numEntries)
+{
+ Int32 i = 0;
+ //Check if there is a hole right up front for the first entry. If so start compacting with the first entry.
+ if (dcArray[0].getCurrentOffset() != 0)
+ {
+ dcArray[0].setNewOffset(0);
+ for (i = 1; i < numEntries; i++)
+ {
+ dcArray[i].setNewOffset(dcArray[i-1].getNewOffset() + dcArray[i-1].getChunkLen());
+ }
+ }
+ else
+ //Look for the first unused section and start compacting from there.
+ {
+ NABoolean done = FALSE;
+ i = 0;
+ Int32 j = 0;
+ while (i < numEntries && !done )
+ {
+ if ((dcArray[i].getCurrentOffset()+dcArray[i].getChunkLen()) !=
+ dcArray[i+1].getCurrentOffset())
+ {
+ j = i+1;
+ while (j < numEntries)
+ {
+ dcArray[j].setNewOffset(dcArray[j-1].getNewOffset()+dcArray[j-1].getChunkLen());
+ j++;
+ }
+ done = TRUE;
+ }
+ i++;
+ }
+ }
+ return ;
+}
+
+Lng32 ExpLOBoper::compactLobDataFile(void *lobGlob,ExLobInMemoryDescChunksEntry *dcArray,Int32 numEntries,char *tgtLobName,Int64 lobMaxChunkMemSize, void *lobHeap, char *hdfsServer, Int32 hdfsPort, char *lobLoc)
+{
+ Int32 rc = 0;
+ void * lobGlobL = NULL;
+ // Call ExeLOBinterface to create the LOB
+ if (lobGlob == NULL)
+ {
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+ }
+ else
+ lobGlobL = lobGlob;
+
+ if (rc)
+ return -1;
+
+ rc = ExpLOBinterfacePerformGC(lobGlobL,tgtLobName, (void *)dcArray, numEntries,hdfsServer,hdfsPort,lobLoc,lobMaxChunkMemSize);
+
+ return rc;
+}
+
+Int32 ExpLOBoper::restoreLobDataFile(void *lobGlob, char *lobName, void *lobHeap, char *hdfsServer, Int32 hdfsPort, char *lobLoc)
+{
+ Int32 rc = 0;
+ void * lobGlobL = NULL;
+ if (lobGlob == NULL)
+ {
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+ }
+ else
+ lobGlobL = lobGlob;
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+
+ rc = ExpLOBinterfaceRestoreLobDataFile(lobGlobL,hdfsServer,hdfsPort,lobLoc,lobName);
+ return rc;
+}
+
+Int32 ExpLOBoper::purgeBackupLobDataFile(void *lobGlob,char *lobName, void *lobHeap, char * hdfsServer, Int32 hdfsPort, char *lobLoc)
+{
+ Int32 rc = 0;
+ void * lobGlobL = NULL;
+ if (lobGlob == NULL)
+ {
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+ }
+ else
+ lobGlobL = lobGlob;
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+
+
+ rc = ExpLOBinterfacePurgeBackupLobDataFile(lobGlobL,(char *)hdfsServer,hdfsPort,lobLoc,lobName);
return rc;
}
+
Lng32 ExpLOBoper::dropLOB(void * lobGlob, void * lobHeap,
- char * lobLoc,
+ char * lobLoc,Int32 hdfsPort, char *hdfsServer,
Int64 uid, Lng32 num)
{
char buf[101];
@@ -214,12 +302,13 @@ Lng32 ExpLOBoper::dropLOB(void * lobGlob, void * lobHeap,
lobGlobL = lobGlob;
// Call ExeLOBinterface to drop the LOB
- rc = ExpLOBinterfaceDrop(lobGlobL,(char *)"default", 0, lobName, lobLoc);
+ rc = ExpLOBinterfaceDrop(lobGlobL,hdfsServer, hdfsPort, lobName, lobLoc);
return rc;
}
Lng32 ExpLOBoper::purgedataLOB(void * lobGlob, char * lobLoc,
+
Int64 uid, Lng32 num)
{
char buf[100];
@@ -229,7 +318,7 @@ Lng32 ExpLOBoper::purgedataLOB(void * lobGlob, char * lobLoc,
return -1;
// Call ExeLOBinterface to purgedata the LOB
- Lng32 rc = ExpLOBInterfacePurgedata(lobGlob,(char *)"default", 0, lobName, lobLoc);
+ Lng32 rc = ExpLOBInterfacePurgedata(lobGlob, lobName, lobLoc);
if (rc < 0)
return ex_expr::EXPR_ERROR;
@@ -252,11 +341,12 @@ ExpLOBoper::ExpLOBoper(OperatorTypeEnum oper_type,
lobHandleSaved_[0] = 0;
lobStorageLocation_[0] = 0;
lobHdfsServer_[0] = 0;
- strcpy(lobHdfsServer_,"default");
- lobHdfsPort_ = 0;
+ strcpy(lobHdfsServer_,"");
+ lobHdfsPort_ = -1;
descSchName_[0] = 0;
lobMaxSize_ = 0;
lobMaxChunkMemSize_ = 0;
+ lobGCLimit_ = 0;
};
@@ -725,7 +815,7 @@ ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[],
&cliError,
so,
waitedOp,
- lobData, lobLen, getLobMaxSize(), getLobMaxChunkMemSize());
+ lobData, lobLen, getLobMaxSize(), getLobMaxChunkMemSize(),getLobGCLimit());
if (rc == LOB_ACCESS_PREEMPT)
{
@@ -891,7 +981,9 @@ ex_expr::exp_return_type ExpLOBiud::insertData(Lng32 handleLen,
so,
waitedOp,
lobData,
- lobLen,getLobMaxSize(), getLobMaxChunkMemSize());
+ lobLen,getLobMaxSize(),
+ getLobMaxChunkMemSize(),
+ getLobGCLimit());
}
if (rc == LOB_ACCESS_PREEMPT)
@@ -1231,7 +1323,7 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[],
data,
fromLobName, fromSchNameLen, fromSchName,
fromDescKey, fromDescTS,
- getLobMaxSize(), getLobMaxChunkMemSize());
+ getLobMaxSize(), getLobMaxChunkMemSize(),getLobGCLimit());
}
else
{
@@ -1254,7 +1346,7 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[],
data,
fromLobName, fromSchNameLen, fromSchName,
fromDescKey, fromDescTS,
- getLobMaxSize(), getLobMaxChunkMemSize());
+ getLobMaxSize(), getLobMaxChunkMemSize(),getLobGCLimit());
}
if (rc < 0)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOB.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.h b/core/sql/exp/ExpLOB.h
index 316c259..d71a1d9 100644
--- a/core/sql/exp/ExpLOB.h
+++ b/core/sql/exp/ExpLOB.h
@@ -47,7 +47,7 @@
#define LOB_HANDLE_LEN 1024
-
+class ExLobInMemoryDescChunksEntry;
////////////////////////////////
// class LOBglobals
////////////////////////////////
@@ -225,22 +225,23 @@ public:
Int64 uid, Lng32 lobNum,
char * outBuf, Lng32 outBufLen);
- static char * ExpGetLOBOffsetsName(Lng32 schNameLen, char * schName,
- Int64 uid, Lng32 lobNum,
- char * outBuf, Lng32 outBufLen);
static Lng32 ExpGetLOBnumFromDescName(char * descName, Lng32 descNameLen);
static char * ExpGetLOBMDName(Lng32 schNameLen, char * schName,
Int64 uid,
char * outBuf, Lng32 outBufLen);
+ static void calculateNewOffsets(ExLobInMemoryDescChunksEntry *dcArray, Lng32 numEntries);
+ static Lng32 compactLobDataFile(void *lobGlob, ExLobInMemoryDescChunksEntry *dcArray, Int32 numEntries, char *tgtLobName, Int64 lobMaxChunkSize, void *lobHeap,char *hdfsServer, Int32 hdfsPort,char *lobLocation);
+ static Int32 restoreLobDataFile(void *lobGlob, char *lobName, void *lobHeap, char *hdfsServer, Int32 hdfsPort,char *lobLocation );
+ static Int32 purgeBackupLobDataFile(void *lobGlob,char *lobName, void *lobHeap, char *hdfsServer, Int32 hdfsPort, char *lobLocation);
static Lng32 createLOB(void * lobGlob, void * lobHeap,
- char * lobLoc,
+ char * lobLoc, Int32 hdfsPort, char *hdfsServer,
Int64 uid, Lng32 lobNum, Int64 lobMAxSize);
static Lng32 dropLOB(void * lobGlob, void * lobHeap,
- char * lobLoc,
+ char * lobLoc,Int32 hdfsPort, char *hdfsServer,
Int64 uid, Lng32 lobNum);
static Lng32 purgedataLOB(void * lobGlob,
@@ -303,6 +304,12 @@ public:
Int64 getLobMaxSize() { return lobMaxSize_;}
void setLobMaxChunkMemSize(Int64 maxsize) { lobMaxChunkMemSize_ = maxsize;}
Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_;}
+ void setLobGCLimit(Int64 gclimit) { lobGCLimit_ = gclimit;}
+ Int64 getLobGCLimit() { return lobGCLimit_;}
+ void setLobHdfsServer(char *hdfsServer)
+ {strcpy(lobHdfsServer_,hdfsServer);}
+ void setLobHdfsPort(Int32 hdfsPort)
+ {lobHdfsPort_ = hdfsPort;}
protected:
typedef enum
{
@@ -349,13 +356,14 @@ public:
};
Lng32 checkLobOperStatus();
-
+
protected:
char * descSchName() { return descSchName_; }
char * getLobHdfsServer() { return (strlen(lobHdfsServer_) == 0 ? NULL : lobHdfsServer_); }
Lng32 getLobHdfsPort() { return lobHdfsPort_; }
-
+
+
short flags_; // 00-02
short lobNum_;
@@ -371,7 +379,7 @@ protected:
char lobHandleSaved_[LOB_HANDLE_LEN];
char outLobHandle_[LOB_HANDLE_LEN];
- Int64 outHandleLen_;
+ Int32 outHandleLen_;
char blackBox_[1024];
Int64 blackBoxLen_;
@@ -386,6 +394,7 @@ protected:
Int64 lobMaxSize_;
Int64 lobMaxChunkMemSize_;
+ Int64 lobGCLimit_;
// NABasicPtr lobStorageLocation_;
}
;
@@ -478,6 +487,7 @@ class ExpLOBiud : public ExpLOBoper {
{
(v) ? liudFlags_ |= FROM_EXTERNAL: liudFlags_ &= ~FROM_EXTERNAL;
};
+
protected:
Int64 objectUID_;
@@ -534,18 +544,12 @@ public:
virtual short getClassSize() { return (short)sizeof(*this); }
// ---------------------------------------------------------------------
-
-
+
private:
- // char * descSchName() { return descSchName_; }
-
+
Lng32 liFlags_;
char filler1_[4];
- // Int64 objectUID_;
-
- // short descSchNameLen_;
- // char descSchName_[510];
- // NABasicPtr descSchName_;
+
};
class ExpLOBdelete : public ExpLOBiud {
@@ -705,6 +709,7 @@ public:
{
return tgtLocation_;
}
+
private:
enum
{
[08/11] incubator-trafodion git commit: Merge remote branch
'origin/master' into lob_gc_work
Posted by sa...@apache.org.
Merge remote branch 'origin/master' into lob_gc_work
Conflicts:
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/69fbcc98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/69fbcc98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/69fbcc98
Branch: refs/heads/master
Commit: 69fbcc981d64a6dfc3f1340166ed5f4a09ec3604
Parents: 9503519 1824c7e
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Thu Mar 3 17:41:00 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Thu Mar 3 17:41:00 2016 +0000
----------------------------------------------------------------------
core/.gitignore | 1 +
core/Makefile | 5 +-
core/sqf/Makefile | 32 +-
core/sqf/hbase_utilities/pom.xml | 2 +-
core/sqf/makelink | 33 -
core/sqf/sqenvcom.sh | 43 +-
core/sqf/sql/scripts/install_local_hadoop | 6 +-
core/sqf/src/seatrans/hbase-trx/Makefile | 16 +-
core/sqf/src/seatrans/hbase-trx/genvers | 3 +-
.../seatrans/hbase-trx/hbase-protocol/pom.xml | 191 --
core/sqf/src/seatrans/hbase-trx/pom.xml | 342 ---
.../transactional/TransactionManager.java | 22 +-
.../transactional/TransactionalScanner.java | 14 +-
core/sqf/src/seatrans/tm/hbasetmlib2/pom.xml | 6 +-
.../java/org/trafodion/dtm/HBaseTxClient.java | 14 +-
.../java/org/trafodion/dtm/TmAuditTlog.java | 23 +-
core/sql/arkcmp/CmpContext.cpp | 12 +-
core/sql/arkcmp/CmpContext.h | 18 +
core/sql/arkcmp/CmpStatement.cpp | 19 +-
core/sql/arkcmp/CmpStatement.h | 1 +
core/sql/bin/SqlciErrors.txt | 1 +
core/sql/cli/Context.cpp | 106 +-
core/sql/cli/Context.h | 13 +-
core/sql/cli/ExSqlComp.cpp | 5 +
core/sql/cli/Statement.cpp | 36 +-
core/sql/comexe/CmpMessage.h | 23 +
core/sql/comexe/ComTdbHbaseAccess.cpp | 3 +
core/sql/comexe/ComTdbHbaseAccess.h | 4 +-
core/sql/common/ComSmallDefs.h | 11 +
core/sql/common/DateTimeType.cpp | 3 +-
core/sql/executor/ExExeUtil.h | 1 +
core/sql/executor/ExExeUtilGet.cpp | 55 +-
core/sql/executor/ExFirstN.cpp | 45 +-
core/sql/executor/ExFirstN.h | 4 +-
core/sql/executor/ExHbaseAccess.cpp | 22 +-
core/sql/executor/ExHbaseAccess.h | 1 +
core/sql/executor/ExHbaseIUD.cpp | 11 +-
core/sql/executor/ExStats.cpp | 13 +-
core/sql/executor/HBaseClient_JNI.cpp | 41 +-
core/sql/executor/HBaseClient_JNI.h | 7 +-
core/sql/executor/ex_ddl.cpp | 2 +
core/sql/executor/ex_transaction.cpp | 278 ++-
core/sql/executor/ex_transaction.h | 2 +-
core/sql/exp/ExpHbaseInterface.cpp | 34 +-
core/sql/exp/ExpHbaseInterface.h | 16 +-
core/sql/generator/GenPreCode.cpp | 13 -
core/sql/generator/GenRelExeUtil.cpp | 31 +-
core/sql/generator/GenRelMisc.cpp | 13 +-
core/sql/generator/GenRelUpdate.cpp | 112 +-
core/sql/generator/Generator.cpp | 5 +
core/sql/lib_mgmt/.gitignore | 7 +
core/sql/lib_mgmt/Makefile | 35 +
core/sql/lib_mgmt/README.rst | 63 +
core/sql/lib_mgmt/pom.xml | 108 +
.../java/org/trafodion/libmgmt/FileMgmt.java | 645 ++++++
.../java/org/trafodion/libmgmt/Vproc.java.tmpl | 35 +
.../lib_mgmt/src/main/resources/init_libmgmt.sh | 206 ++
.../com/trafodion/libmgmt/JarFileMgmtTest.java | 27 +
core/sql/nskgmake/Makerules.linux | 19 +-
core/sql/optimizer/BindRelExpr.cpp | 86 +-
core/sql/optimizer/ItemOther.h | 7 +
core/sql/optimizer/NAColumn.h | 43 +-
core/sql/optimizer/NARoutine.cpp | 2 +-
core/sql/optimizer/NARoutineDB.h | 5 +-
core/sql/optimizer/NATable.cpp | 58 +-
core/sql/optimizer/NATable.h | 10 +-
core/sql/optimizer/RelExeUtil.cpp | 155 +-
core/sql/optimizer/RelExeUtil.h | 30 +-
core/sql/optimizer/RelExpr.cpp | 2 +-
core/sql/optimizer/RelUpdate.h | 2 +
core/sql/parser/SqlParserAux.cpp | 2 +-
core/sql/parser/StmtDDLAlter.cpp | 11 +
.../sql/parser/StmtDDLAlterTableAlterColumn.cpp | 20 +-
core/sql/parser/StmtDDLAlterTableAlterColumn.h | 17 +-
core/sql/parser/StmtDDLNode.cpp | 14 +
core/sql/parser/StmtDDLNode.h | 15 +-
core/sql/parser/sqlparser.y | 6 +-
core/sql/pom.xml | 10 +-
core/sql/regress/charsets/DIFF012.KNOWN.SB | 2 +
core/sql/regress/compGeneral/EXPECTED071 | 8 +-
core/sql/regress/core/DIFF116.KNOWN.SB | 21 +
core/sql/regress/core/EXPECTED056.SB | 75 +-
core/sql/regress/core/EXPECTED116 | 542 +++--
core/sql/regress/core/TEST029 | 2 +-
core/sql/regress/core/TEST056 | 7 +-
core/sql/regress/core/TEST116 | 123 +-
core/sql/regress/executor/EXPECTED013.SB | 39 +-
core/sql/regress/executor/EXPECTED140 | 40 +-
core/sql/regress/fullstack2/EXPECTED062 | 3 +
core/sql/regress/fullstack2/EXPECTED062.RELEASE | 3 +
core/sql/regress/privs2/EXPECTED135 | 329 ---
core/sql/regress/privs2/EXPECTED138 | 2 +-
core/sql/regress/privs2/EXPECTED142 | 760 +++++++
core/sql/regress/privs2/TEST135 | 43 -
core/sql/regress/privs2/TEST142 | 280 +++
core/sql/regress/seabase/EXPECTED010 | 460 ++--
core/sql/regress/seabase/EXPECTED011 | 102 +-
core/sql/regress/seabase/EXPECTED016 | 35 +-
core/sql/regress/seabase/EXPECTED020 | 233 ++
core/sql/regress/seabase/EXPECTED027 | 433 +++-
core/sql/regress/seabase/TEST011 | 16 -
core/sql/regress/seabase/TEST020 | 54 +
core/sql/regress/seabase/TEST027 | 58 +
core/sql/regress/tools/runregr_catman1.ksh | 3 +-
core/sql/regress/tools/runregr_charsets.ksh | 3 +-
core/sql/regress/tools/runregr_compGeneral.ksh | 3 +-
core/sql/regress/tools/runregr_core.ksh | 3 +-
core/sql/regress/tools/runregr_executor.ksh | 3 +-
core/sql/regress/tools/runregr_fullstack2.ksh | 3 +-
core/sql/regress/tools/runregr_hive.ksh | 4 +-
core/sql/regress/tools/runregr_privs1.ksh | 3 +-
core/sql/regress/tools/runregr_privs2.ksh | 3 +-
core/sql/regress/tools/runregr_qat.ksh | 3 +-
core/sql/regress/tools/runregr_seabase.ksh | 4 +-
core/sql/regress/tools/runregr_udr.ksh | 9 +-
core/sql/regress/tools/sbdefs | 1 +
core/sql/sqlcomp/CmpDescribe.cpp | 186 +-
core/sql/sqlcomp/CmpSeabaseDDL.h | 104 +-
core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 42 +-
core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 490 +++--
core/sql/sqlcomp/CmpSeabaseDDLindex.cpp | 98 +-
core/sql/sqlcomp/CmpSeabaseDDLrepos.cpp | 2 +-
core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp | 8 +-
core/sql/sqlcomp/CmpSeabaseDDLschema.cpp | 26 +-
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 1999 ++++++++++++------
core/sql/sqlcomp/CmpSeabaseDDLupgrade.cpp | 14 +-
core/sql/sqlcomp/CmpSeabaseDDLupgrade.h | 6 +-
core/sql/sqlcomp/CmpSeabaseDDLview.cpp | 30 +-
core/sql/sqlcomp/DefaultConstants.h | 8 +
core/sql/sqlcomp/PrivMgrDefs.h | 4 +-
core/sql/sqlcomp/PrivMgrMD.cpp | 171 +-
core/sql/sqlcomp/PrivMgrMD.h | 132 +-
core/sql/sqlcomp/PrivMgrPrivileges.cpp | 793 ++++---
core/sql/sqlcomp/PrivMgrPrivileges.h | 48 +-
core/sql/sqlcomp/nadefaults.cpp | 9 +-
.../java/org/trafodion/sql/HBaseClient.java | 70 +-
licenses/lic-clients-bin | 36 +-
137 files changed, 7688 insertions(+), 3657 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/comexe/ComTdbHbaseAccess.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/comexe/ComTdbHbaseAccess.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/executor/ExHbaseAccess.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/executor/ExHbaseIUD.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/generator/GenPreCode.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/generator/GenRelUpdate.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/sqlcomp/CmpSeabaseDDLindex.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
diff --cc core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
index 276673f,fb9c802..ce3349b
--- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
@@@ -2143,105 -2171,104 +2171,111 @@@ short CmpSeabaseDDL::createSeabaseTable
return -1;
}
}
-
- HbaseStr hbaseTable;
- hbaseTable.val = (char*)extNameForHbase.data();
- hbaseTable.len = extNameForHbase.length();
- if (createHbaseTable(ehi, &hbaseTable, trafColFamVec,
- &hbaseCreateOptions,
- numSplits, keyLength,
- encodedKeysBuffer) == -1)
- {
- deallocEHI(ehi);
-
- processReturn();
-
- return -2;
- }
-
- // if this table has lob columns, create the lob files
- short *lobNumList = new (STMTHEAP) short[numCols];
- short *lobTypList = new (STMTHEAP) short[numCols];
- char **lobLocList = new (STMTHEAP) char*[numCols];
- Lng32 j = 0;
- for (Int32 i = 0; i < colArray.entries(); i++)
- {
- ElemDDLColDef *column = colArray[i];
+
+ NABoolean ddlXns = createTableNode->ddlXns();
+ HbaseStr hbaseTable;
+ hbaseTable.val = (char*)extNameForHbase.data();
+ hbaseTable.len = extNameForHbase.length();
+ if (createHbaseTable(ehi, &hbaseTable, trafColFamVec,
+ &hbaseCreateOptions,
+ numSplits, keyLength,
+ encodedKeysBuffer,
+ FALSE, ddlXns
+ ) == -1)
+ {
+ deallocEHI(ehi);
- Lng32 datatype = column->getColumnDataType()->getFSDatatype();
- if ((datatype == REC_BLOB) ||
- (datatype == REC_CLOB))
- {
-
- lobNumList[j] = i; //column->getColumnNumber();
- lobTypList[j] =
- (short)(column->getLobStorage() == Lob_Invalid_Storage
+ processReturn();
+
+ return -2;
+ }
+
+ // if this table has lob columns, create the lob files
+ short *lobNumList = new (STMTHEAP) short[numCols];
+ short *lobTypList = new (STMTHEAP) short[numCols];
+ char **lobLocList = new (STMTHEAP) char*[numCols];
+ Lng32 j = 0;
+ for (Int32 i = 0; i < colArray.entries(); i++)
+ {
+ ElemDDLColDef *column = colArray[i];
+
+ Lng32 datatype = column->getColumnDataType()->getFSDatatype();
+ if ((datatype == REC_BLOB) ||
+ (datatype == REC_CLOB))
+ {
+
+ lobNumList[j] = i; //column->getColumnNumber();
+ lobTypList[j] =
+ (short)(column->getLobStorage() == Lob_Invalid_Storage
? Lob_HDFS_File : column->getLobStorage());
- // lobTypList[j] = (short)
- // CmpCommon::getDefaultNumeric(LOB_STORAGE_TYPE);
- char * loc = new (STMTHEAP) char[1024];
+ // lobTypList[j] = (short)
+ // CmpCommon::getDefaultNumeric(LOB_STORAGE_TYPE);
+ char * loc = new (STMTHEAP) char[1024];
- const char* f = ActiveSchemaDB()->getDefaults().
- getValue(LOB_STORAGE_FILE_DIR);
+ const char* f = ActiveSchemaDB()->getDefaults().
+ getValue(LOB_STORAGE_FILE_DIR);
- strcpy(loc, f);
+ strcpy(loc, f);
- lobLocList[j] = loc;
- j++;
- }
+ lobLocList[j] = loc;
+ j++;
+ }
}
-
+
Int64 lobMaxSize = CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024;
++
+ const char *lobHdfsServer = CmpCommon::getDefaultString(LOB_HDFS_SERVER);
+ Int32 lobHdfsPort = (Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT);
- if (j > 0)
- {
- //if the table is a volatile table return an error
- if (createTableNode->isVolatile())
- {
- *CmpCommon::diags()
++
+ if (j > 0)
+ {
+ //if the table is a volatile table return an error
+ if (createTableNode->isVolatile())
+ {
+ *CmpCommon::diags()
<< DgSqlCode(-CAT_LOB_COLUMN_IN_VOLATILE_TABLE)
<< DgTableName(extTableName);
deallocEHI(ehi);
processReturn();
return -1;
- }
- Int64 objUID = getObjectUID(&cliInterface,
- catalogNamePart.data(), schemaNamePart.data(),
- objectNamePart.data(),
- COM_BASE_TABLE_OBJECT_LIT);
+ }
+ Int64 objUID = getObjectUID(&cliInterface,
+ catalogNamePart.data(), schemaNamePart.data(),
+ objectNamePart.data(),
+ COM_BASE_TABLE_OBJECT_LIT);
-
+
- ComString newSchName = "\"";
- newSchName += catalogNamePart;
- newSchName.append("\".\"");
- newSchName.append(schemaNamePart);
- newSchName += "\"";
- Lng32 rc = SQL_EXEC_LOBddlInterface((char*)newSchName.data(),
- newSchName.length(),
- objUID,
- j,
- LOB_CLI_CREATE,
- lobNumList,
- lobTypList,
- lobLocList,
- (char *)lobHdfsServer,
- lobHdfsPort,
- lobMaxSize);
+ ComString newSchName = "\"";
+ newSchName += catalogNamePart;
+ newSchName.append("\".\"");
+ newSchName.append(schemaNamePart);
+ newSchName += "\"";
+ Lng32 rc = SQL_EXEC_LOBddlInterface((char*)newSchName.data(),
+ newSchName.length(),
+ objUID,
+ j,
+ LOB_CLI_CREATE,
+ lobNumList,
+ lobTypList,
+ lobLocList,
++ (char *)lobHdfsServer,
++ lobHdfsPort,
+ lobMaxSize);
-
+
- if (rc < 0)
- {
- //sss TBD need to retrive the cli diags here.
- *CmpCommon::diags() << DgSqlCode(-CAT_CREATE_OBJECT_ERROR)
- << DgTableName(extTableName);
- deallocEHI(ehi);
- processReturn();
+ if (rc < 0)
+ {
+ //sss TBD need to retrive the cli diags here.
+ *CmpCommon::diags() << DgSqlCode(-CAT_CREATE_OBJECT_ERROR)
+ << DgTableName(extTableName);
+ deallocEHI(ehi);
+ processReturn();
-
+
- return -2;
- }
- }
+ return -2;
+ }
- } // j > 0
-
++ }
++
+
// if not a compound create, update valid def to true.
if (NOT ((createTableNode->getAddConstraintUniqueArray().entries() > 0) ||
(createTableNode->getAddConstraintRIArray().entries() > 0) ||
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/69fbcc98/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
[03/11] incubator-trafodion git commit: Changes for performing GC.
Changes to pass in hdfsServer and port via CQD and remove hardcoding. Changes
to pass in lobLocation via CQD and remove hardcoding. Changes to remove all
mxlobsrvr process functionality
Posted by sa...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index 6062775..bcdba44 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -46,7 +46,7 @@
#include "ExpLOBstats.h"
#include "ExpLOBinterface.h"
#include "ComSmallDefs.h"
-
+#include "Globals.h"
#include <seabed/ms.h>
#include <seabed/fs.h>
@@ -79,6 +79,79 @@ using namespace std;
#define LOB_CURSOR_PREFETCH_BYTES_MAX (1 << 27) // 128MB
class ExLobGlobals;
+// This class defines the request used to construct the message to send over
+// to the mxlobsrvr process. It's currently not used. All lob functionailty is
+// now in te master process. We are retaining it here for furture use.
+class ExLobRequest
+{
+ public:
+ ExLobRequest();
+ ~ExLobRequest();
+ Ex_Lob_Error send();
+
+ void setValues(char *descFileName, Int64 descNumIn, Int64 handleInLen,
+ char *handleIn, LobsStorage storage, Int64 transId,
+ SB_Transid_Type transIdBig,
+ SB_Transseq_Type transStartId,
+ char *blackBox, Int64 blackBoxLen);
+ void getValues(Int64 &descNumOut, Int64 &handleOutLen,
+ char *handleOut, Ex_Lob_Error &requestStatus,
+ Int64 &cliError,
+ char *blackBox, Int64 &blackBoxLen);
+
+ Int64 getDescNumIn() { return descNumIn_; }
+ void setDescNumIn(Int64 descNum) { descNumIn_ = descNum; }
+ Int64 getDescNumOut() { return descNumOut_; }
+ void setDescNumOut(Int64 descNum) { descNumOut_ = descNum; }
+ Int64 getDataOffset() { return dataOffset_; }
+ void setDataOffset(int dataOffset) { dataOffset_ = dataOffset; }
+ LobsRequest getType() { return type_; }
+ void setType(LobsRequest type) { type_ = type; }
+ char *getDescFileName() { return descFileName_; }
+ Int64 getOperLen() { return operLen_; }
+ void setOperLen(Int64 len) { operLen_ = len; }
+ void log();
+ void setError(Ex_Lob_Error err) { error_ = err; }
+ Ex_Lob_Error getError() { return error_; }
+ Ex_Lob_Error getStatus() { return status_; }
+ char *getHandleIn() { return handleIn_; }
+ Int64 getHandleInLen() { return handleInLen_; }
+ char *getHandleOut() { return handleOut_; }
+ void setHandleOutLen(Lng32 len) { handleOutLen_ = len; }
+ void setCliError(int cliErr) { cliError_ = cliErr; }
+ int getCliError() { return (int)cliError_; }
+ Int64 getTransId() { return transId_; }
+ SB_Transid_Type getTransIdBig() { return transIdBig_; }
+ SB_Transseq_Type getTransStartId() { return transStartId_; }
+ Int64 getBlackBoxLen() { return blackBoxLen_; };
+ void setBlackBoxLen(Int64 len) { blackBoxLen_ = len; }
+ char *getBlackBox() { return blackBox_; }
+
+ void incrReqNum() { reqNum_++; }
+ Int64 getReqNum() { return reqNum_; }
+
+ private:
+ Int64 reqNum_;
+ Int64 descNumIn_;
+ Int64 descNumOut_;
+ char handleIn_[MAX_HANDLE_IN_LEN];
+ Int64 handleInLen_;
+ char handleOut_[MAX_HANDLE_OUT_LEN];
+ Int64 handleOutLen_;
+ Int64 dataOffset_;
+ LobsRequest type_;
+ LobsStorage storage_;
+ Int64 operLen_;
+ Ex_Lob_Error error_;
+ Int64 cliError_;
+ Ex_Lob_Error status_;
+ Int64 transId_;
+ SB_Transid_Type transIdBig_;
+ SB_Transseq_Type transStartId_;
+ char descFileName_[MAX_LOB_FILE_NAME_LEN];
+ char blackBox_[MAX_BLACK_BOX_LEN];
+ Int64 blackBoxLen_;
+};
///////////////////////////////////////////////////////////////////////////////
// THE SQL LOB API
@@ -87,11 +160,11 @@ class ExLobGlobals;
Ex_Lob_Error ExLobsOper (
char *lobName, // lob name
char *handleIn, // input handle (for cli calls)
- Int64 handleInLen, // input handle len
+ Int32 handleInLen, // input handle len
char *hdfsServer, // server where hdfs fs resides
Int64 hdfsPort, // port number to access hdfs server
char *handleOut, // output handle (for cli calls)
- Int64 &handleOutLen, // output handle len
+ Int32 &handleOutLen, // output handle len
Int64 descNumIn, // input desc Num (for flat files only)
Int64 &descNumOut, // output desc Num (for flat files only)
Int64 &retOperLen, // length of data involved in this operation
@@ -111,9 +184,10 @@ Ex_Lob_Error ExLobsOper (
void *&globPtr, // ptr to the Lob objects.
Int64 transId,
void *blackBox, // black box to be sent to cli
- Int64 blackBoxLen, // length of black box
+ Int32 blackBoxLen, // length of black box
Int64 lobMaxSize = 0, // max size of lob.
Int64 lobMaxChunkMemSize = 0 ,//max length of intermediate mem buffer used to do i/o.
+ Int64 lobGCLimit = 0, // size at which GC must be triggered
int bufferSize =0,
short replication =0,
int blocksize=0,
@@ -160,7 +234,7 @@ class ExLobDescHeader
int getDataOffset() { return dataOffset_; }
void incDataOffset(int size) { dataOffset_ += size; }
- Ex_Lob_Error print(char *descFileName);
+
private:
@@ -200,7 +274,6 @@ class ExLobDesc
void setOffset(int offset) { dataOffset_ = offset; }
void setDataState(int dataState) { dataState_ = dataState; }
- Ex_Lob_Error print();
private:
@@ -213,79 +286,46 @@ class ExLobDesc
int nextFree_;
};
-class ExLobRequest
+class ExLobInMemoryDescChunksEntry
{
- public:
- ExLobRequest();
- ~ExLobRequest();
- Ex_Lob_Error send();
-
- void setValues(char *descFileName, Int64 descNumIn, Int64 handleInLen,
- char *handleIn, LobsStorage storage, Int64 transId,
- SB_Transid_Type transIdBig,
- SB_Transseq_Type transStartId,
- char *blackBox, Int64 blackBoxLen);
- void getValues(Int64 &descNumOut, Int64 &handleOutLen,
- char *handleOut, Ex_Lob_Error &requestStatus,
- Int64 &cliError,
- char *blackBox, Int64 &blackBoxLen);
-
- Int64 getDescNumIn() { return descNumIn_; }
- void setDescNumIn(Int64 descNum) { descNumIn_ = descNum; }
- Int64 getDescNumOut() { return descNumOut_; }
- void setDescNumOut(Int64 descNum) { descNumOut_ = descNum; }
- Int64 getDataOffset() { return dataOffset_; }
- void setDataOffset(int dataOffset) { dataOffset_ = dataOffset; }
- LobsRequest getType() { return type_; }
- void setType(LobsRequest type) { type_ = type; }
- char *getDescFileName() { return descFileName_; }
- Int64 getOperLen() { return operLen_; }
- void setOperLen(Int64 len) { operLen_ = len; }
- void log();
- void getDescOut(ExLobDesc &desc);
- void putDescIn(ExLobDesc &desc);
- ExLobDesc &getDesc() { return desc_; }
- void setError(Ex_Lob_Error err) { error_ = err; }
- Ex_Lob_Error getError() { return error_; }
- Ex_Lob_Error getStatus() { return status_; }
- char *getHandleIn() { return handleIn_; }
- Int64 getHandleInLen() { return handleInLen_; }
- char *getHandleOut() { return handleOut_; }
- void setHandleOutLen(Lng32 len) { handleOutLen_ = len; }
- void setCliError(int cliErr) { cliError_ = cliErr; }
- int getCliError() { return (int)cliError_; }
- Int64 getTransId() { return transId_; }
- SB_Transid_Type getTransIdBig() { return transIdBig_; }
- SB_Transseq_Type getTransStartId() { return transStartId_; }
- Int64 getBlackBoxLen() { return blackBoxLen_; };
- void setBlackBoxLen(Int64 len) { blackBoxLen_ = len; }
- char *getBlackBox() { return blackBox_; }
-
- void incrReqNum() { reqNum_++; }
- Int64 getReqNum() { return reqNum_; }
-
- private:
- Int64 reqNum_;
- Int64 descNumIn_;
- Int64 descNumOut_;
- char handleIn_[MAX_HANDLE_IN_LEN];
- Int64 handleInLen_;
- char handleOut_[MAX_HANDLE_OUT_LEN];
- Int64 handleOutLen_;
- Int64 dataOffset_;
- LobsRequest type_;
- LobsStorage storage_;
- Int64 operLen_;
- ExLobDesc desc_;
- Ex_Lob_Error error_;
- Int64 cliError_;
- Ex_Lob_Error status_;
- Int64 transId_;
- SB_Transid_Type transIdBig_;
- SB_Transseq_Type transStartId_;
- char descFileName_[MAX_LOB_FILE_NAME_LEN];
- char blackBox_[MAX_BLACK_BOX_LEN];
- Int64 blackBoxLen_;
+public:
+ ExLobInMemoryDescChunksEntry():
+ currentOffset_ (-1),
+ newOffset_( -1),
+ descPartnKey_(-1),
+ descSyskey_(-1),
+ chunkLen_(-1),
+ chunkNum_(-1)
+ {}
+ ExLobInMemoryDescChunksEntry(Int64 co,Int64 partnKey,Int64 syskey,Int64 chunkLen,Int32 chunkNum)
+ {
+ currentOffset_ = co; newOffset_ = -1;descPartnKey_ = partnKey; descSyskey_ = syskey; chunkLen_ = chunkLen; chunkNum_ =chunkNum;
+ }
+
+ void setCurrentOffset(Int64 co) { currentOffset_ = co;}
+ Int64 getCurrentOffset(){return currentOffset_;}
+
+ void setNewOffset(Int64 no) { newOffset_ = no;}
+ Int64 getNewOffset(){return newOffset_;}
+
+ void setDescPartnKey(Int64 dk) { descPartnKey_ = dk;}
+ Int64 getDescPartnKey(){return descPartnKey_;}
+
+ void setSyskey(Int64 sk) { descSyskey_ = sk;}
+ Int64 getSyskey(){return descSyskey_;}
+
+ void setChunkLen(Int64 cl) { chunkLen_ = cl;}
+ Int64 getChunkLen(){return chunkLen_;}
+
+ void setChunkNum(Int32 cn) { chunkNum_ = cn;}
+ Int32 getChunkNum(){return chunkNum_;}
+private:
+ Int64 currentOffset_;
+ Int64 newOffset_;
+ Int64 descPartnKey_;
+ Int64 descSyskey_;
+ Int64 chunkLen_;
+ Int32 chunkNum_;
};
class ExLobCursorBuffer
@@ -369,105 +409,103 @@ class ExLob
Ex_Lob_Error initialize(char *lobFile, Ex_Lob_Mode mode, char *dir,
LobsStorage storage, char *hdfsServer, Int64 hdfsPort,
- int bufferSize = 0, short replication =0, int blocksize=0, Int64 lobMaxSize = 0, ExLobGlobals *lobGlobals = NULL);
+ char *lobLocation,
+ int bufferSize = 0, short replication =0,
+ int blocksize=0, Int64 lobMaxSize = 0,
+ ExLobGlobals *lobGlobals = NULL);
Ex_Lob_Error initialize(char *lobFile);
- Ex_Lob_Error writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOperation, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize);
+ Ex_Lob_Error writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOperation, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,Int64 lobGCLimit, char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
Ex_Lob_Error writeLobData(char *source, Int64 sourceLen,
LobsSubOper subOperation,
Int64 tgtOffset,Int64 &operLen,
Int64 lobMaxMemChunkLen);
Ex_Lob_Error writeDataSimple(char *data, Int64 size, LobsSubOper subOperation, Int64 &operLen,
int bufferSize = 0, short replication =0, int blocksize=0);
- Ex_Lob_Error readToMem(char *memAddr, Int64 size, Int64 &operLen);
- Ex_Lob_Error readToFile(char *fileName, Int64 tgtLen,Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags);
- Ex_Lob_Error readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int64 handleInLen, Int64 &operLen);
- Ex_Lob_Error readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen);
+ Ex_Lob_Error readToMem(char *memAddr, Int64 size, Int64 &operLen,char * handleIn, Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, Int64 transId);
+ Ex_Lob_Error readToFile(char *fileName, Int64 tgtLen,Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags, char *handleIn,Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, Int64 transId);
+ Ex_Lob_Error readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int32 handleInLen, Int64 &operLen,Int64 transId);
+ Ex_Lob_Error readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen,char *handleIn, Int32 handeLenIn,Int64 transId);
Ex_Lob_Error readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen);
Ex_Lob_Error readDataCursorSimple(char *fileName, char *tgt, Int64 tgtSize, Int64 &operLen, ExLobGlobals *lobGlobals);
bool hasNoOpenCursors() { return lobCursors_.empty(); }
- Ex_Lob_Error openCursor(char *handleIn, Int64 handleInLen);
+ Ex_Lob_Error openCursor(char *handleIn, Int32 handleInLen,Int64 transId);
Ex_Lob_Error openDataCursor(char *fileName, LobsCursorType type, Int64 range,
Int64 bytesLeft, Int64 bufMaxSize, Int64 prefetch, ExLobGlobals *lobGlobals);
Ex_Lob_Error deleteCursor(char *cursorName, ExLobGlobals *lobGlobals);
- Ex_Lob_Error fetchCursor();
- Ex_Lob_Error selectCursorDesc(ExLobRequest *request);
- Ex_Lob_Error fetchCursorDesc(ExLobRequest *request);
- Ex_Lob_Error insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize);
- Ex_Lob_Error append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemLen);
- Ex_Lob_Error append(ExLobRequest *request);
- Ex_Lob_Error update(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemLen);
- Ex_Lob_Error update(ExLobRequest *request);
+ Ex_Lob_Error fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOffset, Int64 &outSize,NABoolean &isEOD,Int64 transId);
+ Ex_Lob_Error insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,char *handleIn,Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
+ Ex_Lob_Error append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemLen,Int64 lobGCLimit, char *handleIn,Int32 handleInLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
+ Ex_Lob_Error update(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemLen,Int64 lobGCLimit,char *handleIn,Int32 handleInLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
Ex_Lob_Error readSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
Ex_Lob_Error readHdfsSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
Ex_Lob_Error readLocalSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
Ex_Lob_Error readExternalSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
Ex_Lob_Error statSourceFile(char *srcfile, Int64 &sourceEOF);
- Ex_Lob_Error delDesc();
- Ex_Lob_Error delDesc(ExLobRequest *request);
+ Ex_Lob_Error delDesc(char *handleIn, Int32 handleInLen, Int64 transId);
Ex_Lob_Error purgeLob();
Ex_Lob_Error closeFile();
LobInputOutputFileType fileType(char *ioFileName);
- Ex_Lob_Error closeCursor(char *handleIn, Int64 handleInLen);
+ Ex_Lob_Error closeCursor(char *handleIn, Int32 handleInLen);
Ex_Lob_Error closeDataCursorSimple(char *fileName, ExLobGlobals *lobGlobals);
- Ex_Lob_Error closeCursorDesc(ExLobRequest *request);
+
Ex_Lob_Error doSanityChecks(char *dir, LobsStorage storage,
- Int64 handleInLen, Int64 handleOutLen,
- Int64 blackBoxLen);
- Ex_Lob_Error allocateDesc(unsigned int size, Int64 &descNum, Int64 &dataOffset,Int64 lobMaxSize);
- Ex_Lob_Error getNextFreeOffset(Int32 size,Int64 &freeOffset, Int64 lobMaxSize);
+ Int32 handleInLen, Int32 handleOutLen,
+ Int32 blackBoxLen);
+ Ex_Lob_Error allocateDesc(unsigned int size, Int64 &descNum, Int64 &dataOffset,Int64 lobMaxSize,Int64 lobMaxChunkMemSize, char *handleIn, Int32 handleInLen,Int64 lobGCLimit, void *lobGlobals);
Ex_Lob_Error readStats(char *buffer);
Ex_Lob_Error initStats();
- Ex_Lob_Error insertDesc(ExLobRequest *request) ;
-
- ExLobRequest *getRequest() { return &request_; }
+ Ex_Lob_Error insertDesc(Int64 offset, Int64 size, char *handleIn, Int32 handleInLen, char *handleOut, Int32 &handleOutLen, char *blackBox, Int32 blackBoxLen,void *lobGlobals) ;
Ex_Lob_Error lockDesc();
Ex_Lob_Error unlockDesc();
- //int getfdDesc() { return fdDesc_; }
- hdfsFile *getfdDesc() { return &fdDesc_;}
- char *getDescFileName() { return lobDescFile_; }
char *getDataFileName() { return lobDataFile_; }
- Ex_Lob_Error getNextFreeOffset(Int64 size, Int64 &dataOffset, Int64 lobMaxSize);
- Ex_Lob_Error print();
+
int getErrNo();
- Ex_Lob_Error delDesc(Int64 descNum);
- Ex_Lob_Error getDesc(ExLobDesc &desc);
- Ex_Lob_Error putDesc(ExLobDesc &desc, Int64 descNum);
- Ex_Lob_Error getDesc(ExLobRequest *request);
+
+ Ex_Lob_Error getDesc(ExLobDesc &desc,char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char * handleOut, Int32 &handleOutLen, Int64 transId);
Ex_Lob_Error writeData(Int64 offset, char *data, Int32 size, Int64 &operLen);
- Ex_Lob_Error readDataToMem(char *memAddr, Int64 offset, Int64 size, Int64 &operLen);
+ Ex_Lob_Error readDataToMem(char *memAddr, Int64 offset, Int64 size,
+ Int64 &operLen,char *handleIn, Int32 handleLenIn,
+ NABoolean multipleChunks, Int64 transId);
- Ex_Lob_Error readDataToLocalFile(char *fileName, Int64 offset, Int64 size,Int64 &operLen,Int64 lobMaxChunkMemLen ,Int32 fileFlags);
- Ex_Lob_Error readCursorDataToLocalFile(char *fileName, Int64 offset, Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemSize, Int32 fileflags);
- Ex_Lob_Error readDataToHdfsFile(char *fileName, Int64 offset, Int64 size, Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags);
- Ex_Lob_Error readDataToExternalFile(char *tgtFileName, Int64 offset, Int64 size, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags);
- Ex_Lob_Error readDataFromFile(char *memAddr, Int64 len, Int64 &operLen);
+ Ex_Lob_Error readDataToLocalFile(char *fileName, Int64 offset, Int64 size,Int64 &operLen,Int64 lobMaxChunkMemLen ,Int32 fileFlags,char *handleIn,Int32 handleInLen, NABoolean multipleChunks,Int64 transId);
+ Ex_Lob_Error readDataToHdfsFile(char *fileName, Int64 offset, Int64 size, Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags,char *handleIn,Int32 handleInLen, NABoolean multipleChunks,Int64 transId);
+ Ex_Lob_Error readDataToExternalFile(char *tgtFileName, Int64 offset, Int64 size, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags,char *handleIn,Int32 handleInLen, NABoolean multipleChunks,Int64 transId);
+ Ex_Lob_Error readDataFromFile(char *memAddr, Int64 len, Int64 &operLen);
+ Ex_Lob_Error compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int32 numEntries);
+ Ex_Lob_Error restoreLobDataFile();
+ Ex_Lob_Error purgeBackupLobDataFile();
Ex_Lob_Error emptyDirectory();
ExLobStats *getStats() { return &stats_; }
NAHeap *getLobGlobalHeap() { return lobGlobalHeap_;}
+ ExLobRequest *getRequest() { return &request_; }
+
+ //The next 2 functions are not active at this point. They serve as an example
+ //on how to send requests across to the mxlobsrvr process from the master
+ //process
+ Ex_Lob_Error getDesc(ExLobRequest *request);
+ Ex_Lob_Error sendReqToLobServer() ;
public:
char lobDataFile_[MAX_LOB_FILE_NAME_LEN];
- char lobDescFile_[MAX_LOB_FILE_NAME_LEN];
- //int fdDesc_;
- hdfsFile fdDesc_;
lobCursors_t lobCursors_;
ExLobLock lobCursorLock_;
LobsStorage storage_;
string dir_; // lob data directory
char *hdfsServer_;
Int64 hdfsPort_;
+ char *lobLocation_;
hdfsFS fs_;
hdfsFile fdData_;
int openFlags_;
- ExLobRequest request_;
ExLobStats stats_;
bool prefetchQueued_;
NAHeap *lobGlobalHeap_;
+ ExLobRequest request_;
};
typedef map<string, ExLob *> lobMap_t;
@@ -537,8 +575,6 @@ class ExLobGlobals
~ExLobGlobals();
Ex_Lob_Error initialize();
- Ex_Lob_Error setServerPhandle();
- Ex_Lob_Error resetServerPhandle();
lobMap_t * getLobMap() { return lobMap_; }
Ex_Lob_Error getLobPtr(char *lobName, ExLob *& lobPtr);
Ex_Lob_Error delLobPtr(char *lobName);
@@ -598,4 +634,6 @@ class ExLobGlobals
NAHeap *heap_;
};
+
+
#endif
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBenums.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBenums.h b/core/sql/exp/ExpLOBenums.h
index 67f5f43..99e072d 100644
--- a/core/sql/exp/ExpLOBenums.h
+++ b/core/sql/exp/ExpLOBenums.h
@@ -237,7 +237,10 @@ typedef enum {
Lob_Empty_Directory,
- Lob_Cleanup // destroy everything under globals
+ Lob_Cleanup, // destroy everything under globals
+ Lob_PerformGC,
+ Lob_RestoreLobDataFile,
+ Lob_PurgeBackupLobDataFile
} LobsOper;
typedef enum {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBexternal.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBexternal.h b/core/sql/exp/ExpLOBexternal.h
index b21b97d..e8080f6 100644
--- a/core/sql/exp/ExpLOBexternal.h
+++ b/core/sql/exp/ExpLOBexternal.h
@@ -93,7 +93,11 @@ enum LOBcliQueryType
LOB_DATA_EXTRACT,
// returns length of lob given a lobhandle
- LOB_CLI_SELECT_LOBLENGTH
+ LOB_CLI_SELECT_LOBLENGTH,
+
+ // performs GC of lob file
+ LOB_CLI_PERFORM_LOB_GC
+
};
enum LOBcliQueryPhase
@@ -116,7 +120,7 @@ Lng32 SQL_EXEC_LOBcliInterface
/*IN*/ char * inLobHandle,
/*IN*/ Lng32 inLobHandleLen,
/*IN*/ char * blackBox,
- /*IN*/ Lng32* blackBoxLen,
+ /*IN*/ Int32* blackBoxLen,
/*OUT*/ char * outLobHandle,
/*OUT*/ Lng32 * outLobHandleLen,
/*IN*/ LOBcliQueryType qType,
@@ -135,6 +139,16 @@ Lng32 SQL_EXEC_LOBcliInterface
in */
/*IN*/ Int64 xnId /* xn id of the parent process, if non-zero */
);
+Lng32 SQL_EXEC_LOB_GC_Interface
+(
+ /*IN*/ void *lobGlobals, // can be passed or NULL
+ /*IN*/ char * handle,
+ /*IN*/ Lng32 handleLen,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
+ /*IN*/ char *lobLocation,
+ /*IN*/ Int64 lobMaxMemChunkLen // if passed in as 0, will use default value of 1G for the in memory buffer to do compaction.
+ );
Lng32 SQL_EXEC_LOBddlInterface
(
@@ -146,6 +160,8 @@ Lng32 SQL_EXEC_LOBddlInterface
/*IN*/ short *lobNumList,
/*IN*/ short *lobTypList,
/*IN*/ char* *lobLocList,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
/*IN*/ Int64 lobMaxSize
);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBinterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.cpp b/core/sql/exp/ExpLOBinterface.cpp
index d55bab9..188fed4 100644
--- a/core/sql/exp/ExpLOBinterface.cpp
+++ b/core/sql/exp/ExpLOBinterface.cpp
@@ -41,13 +41,13 @@ Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap,NABoolean isHive, Int6
Int64 dummyParam = 0;
Int64 cliError = -1;
Ex_Lob_Error status;
-
+ Int32 dummyParam2 = 0;
char dir[100];
- strcpy(dir, "/h/temp");
+ strcpy(dir, "/lobs");
err = ExLobsOper(dir,
NULL, 0,
NULL, 0,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
dir, Lob_HDFS_File,
NULL, 0,
@@ -76,22 +76,101 @@ Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap,NABoolean isHive, Int6
return 0;
}
-Lng32 ExpLOBinterfaceCleanup(void *& lobGlob, void * lobHeap)
+Lng32 ExpLOBinterfacePerformGC(void *& lobGlob, char *lobName,void *descChunksArray, Int32 numEntries, char *hdfsServer, Int32 hdfsPort,char *lobLoc,Int64 lobMaxChunkMemSize)
{
Ex_Lob_Error err;
+ Ex_Lob_Error status;
+ Int64 dummyParam = 0;
+ Int64 cliError = -1;
+ Int32 dummyParam2 = 0;
+ err = ExLobsOper(lobName,
+ NULL, 0,
+ hdfsServer, hdfsPort,
+ NULL, dummyParam2, 0, dummyParam,
+ dummyParam, 0, dummyParam, status, cliError,
+ lobLoc, Lob_HDFS_File,
+ (char *)descChunksArray, numEntries,
+ 0,NULL,
+ Lob_PerformGC, // Lob_GC
+ Lob_None,
+ 1, // waited op
+ lobGlob,
+ 0,
+ NULL, 0
+ );
+ if (err != LOB_OPER_OK)
+ return -1;
+ else
+ return 0;
+}
+Lng32 ExpLOBinterfaceRestoreLobDataFile(void *& lobGlob, char *hdfsServer, Int32 hdfsPort,char *lobLoc,char *lobName)
+{
+ Ex_Lob_Error err;
+ Ex_Lob_Error status;
Int64 dummyParam = 0;
Int64 cliError = -1;
+ Int32 dummyParam2 = 0;
+ err = ExLobsOper(lobName,
+ NULL, 0,
+ hdfsServer, hdfsPort,
+ NULL, dummyParam2, 0, dummyParam,
+ dummyParam, 0, dummyParam, status, cliError,
+ lobLoc, Lob_HDFS_File,
+ NULL, 0,
+ 0,NULL,
+ Lob_RestoreLobDataFile, // Lob_GC
+ Lob_None,
+ 1, // waited op
+ lobGlob,
+ 0,
+ NULL, 0
+ );
+ if (err != LOB_OPER_OK)
+ return -1;
+ else
+ return 0;
+}
+Lng32 ExpLOBinterfacePurgeBackupLobDataFile(void *& lobGlob, char *hdfsServer, Int32 hdfsPort,char *lobLoc,char *lobName)
+{
+ Ex_Lob_Error err;
Ex_Lob_Error status;
-
- char dir[100];
- strcpy(dir, "/h/temp");
- err = ExLobsOper(dir,
+ Int64 dummyParam = 0;
+ Int64 cliError = -1;
+ Int32 dummyParam2 = 0;
+ err = ExLobsOper(lobName,
+ NULL, 0,
+ hdfsServer, hdfsPort,
+ NULL, dummyParam2, 0, dummyParam,
+ dummyParam, 0, dummyParam, status, cliError,
+ lobLoc, Lob_HDFS_File,
+ NULL, 0,
+ 0,NULL,
+ Lob_PurgeBackupLobDataFile, // Lob_GC
+ Lob_None,
+ 1, // waited op
+ lobGlob,
+ 0,
+ NULL, 0
+ );
+ if (err != LOB_OPER_OK)
+ return -1;
+ else
+ return 0;
+}
+Lng32 ExpLOBinterfaceCleanup(void *& lobGlob, void * lobHeap)
+{
+ Ex_Lob_Error err;
+ Ex_Lob_Error status;
+ Int64 dummyParam = 0;
+ Int64 cliError = -1;
+ Int32 dummyParam2 = 0;
+ err = ExLobsOper((char *)"dummy",
NULL, 0,
NULL, 0,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
- dir, Lob_HDFS_File,
+ NULL, Lob_HDFS_File,
NULL, 0,
0,NULL,
Lob_Cleanup, // Lob_Cleanup
@@ -119,14 +198,15 @@ Lng32 ExpLOBinterfaceCreate(
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam = 0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, (LobsStorage)lobType, //Lob_HDFS_File,
NULL, 0,
@@ -148,17 +228,7 @@ Lng32 ExpLOBinterfaceCreate(
else
return 0;
}
-/*
- * Lng32 ExpLOBinterfaceEmptyDirectory(void * lobGlob,
- char * lobName,
- char * lobLoc,
- Lng32 lobType = (Lng32)Lob_Empty_Directory,
- char * lobHdfsServer = NULL,
- Lng32 lobHdfsPort = 0,
- int bufferSize = 0,
- short replication =0,
- int blocksize=0);
- */
+
Lng32 ExpLOBinterfaceEmptyDirectory(
void * lobGlob,
char * lobName,
@@ -172,14 +242,15 @@ Lng32 ExpLOBinterfaceEmptyDirectory(
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, (LobsStorage)lobType, //Lob_HDFS_File,
NULL, 0,
@@ -204,14 +275,15 @@ Lng32 ExpLOBinterfaceDrop(void * lobGlob, char * lobHdfsServer ,
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam = 0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, Lob_HDFS_File,
NULL, 0,
@@ -230,21 +302,20 @@ Lng32 ExpLOBinterfaceDrop(void * lobGlob, char * lobHdfsServer ,
}
Lng32 ExpLOBInterfacePurgedata(void * lobGlob,
- char * lobHdfsServer ,
- Lng32 lobHdfsPort ,char * lobName, char * lobLoc)
+ char * lobName, char * lobLoc)
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
- // char dir[100];
- // strcpy(dir, "/h/vshah");
+
err = ExLobsOper(lobName,
- lobHdfsServer, lobHdfsPort,
+ NULL,0,
NULL, 0,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, Lob_HDFS_File,
NULL, 0,
@@ -271,7 +342,8 @@ Lng32 ExpLOBinterfaceCloseFile(void * lobGlob,
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
@@ -280,7 +352,7 @@ Lng32 ExpLOBinterfaceCloseFile(void * lobGlob,
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, ls, //Lob_HDFS_File,
NULL, 0,
@@ -309,7 +381,7 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 blackBoxLen,
@@ -329,6 +401,7 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
Int64 srcLobLen,
Int64 lobMaxSize,
Int64 lobMaxChunkMemSize,
+ Int64 lobGCLimit,
int bufferSize ,
short replication ,
int blockSize)
@@ -373,22 +446,14 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
blackBox, blackBoxLen,
lobMaxSize,
lobMaxChunkMemSize,
+ lobGCLimit,
bufferSize,
replication,
blockSize
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
-
- // done with the request (success or error)
- requestTag = -1;
+
if (err != LOB_OPER_OK)
{
if ((cliError) &&
@@ -399,14 +464,9 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
}
descSyskey = outDescSyskey;
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
-
- // error. Status contains the error code.
- return -(short)status;
+
+
+ return 0;
}
Lng32 ExpLOBInterfaceInsertSelect(void * lobGlob,
@@ -434,13 +494,14 @@ Lng32 ExpLOBInterfaceInsertSelect(void * lobGlob,
LobsSubOper lso = Lob_Foreign_Lob;
Ex_Lob_Error status;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Int64 cliError = -1;
err = ExLobsOper(tgtLobName,
lobHandle, handleLen,
NULL, 0, // hdfs server/port
- NULL, dummyParam, -1, descSyskey, lobLen,
+ NULL, dummyParam2, -1, descSyskey, lobLen,
0, dummyParam, status, cliError,
lobStorageLocation, Lob_HDFS_File,
lobData, lobLen, //strlen(srcLobData),
@@ -465,7 +526,7 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
char * lobStorageLocation,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 &requestTag,
Int64 xnId,
@@ -482,7 +543,8 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
Int64 srcDescKey,
Int64 srcDescTS,
Int64 lobMaxSize,
- Int64 lobMaxChunkMemSize
+ Int64 lobMaxChunkMemSize,
+ Int64 lobGCLimit
)
{
Ex_Lob_Error err;
@@ -512,32 +574,19 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
lobGlob,
xnId, NULL, 0,
lobMaxSize,
- lobMaxChunkMemSize
+ lobMaxChunkMemSize,
+ lobGCLimit
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
+
- // done with the request (success or error)
- requestTag = -1;
if (err != LOB_OPER_OK)
{
return -(short)err;
}
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
- // error. Status contains the error code.
- return -(short)status;
+ return 0;
}
Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
@@ -548,7 +597,7 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
char * lobStorageLocation,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 &requestTag,
Int64 xnId,
@@ -565,7 +614,8 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
Int64 srcDescKey,
Int64 srcDescTS,
Int64 lobMaxSize ,
- Int64 lobMaxChunkMemSize )
+ Int64 lobMaxChunkMemSize,
+ Int64 lobGCLimit)
{
Ex_Lob_Error err;
@@ -596,32 +646,20 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
xnId,
NULL, 0,
lobMaxSize,
- lobMaxChunkMemSize
+ lobMaxChunkMemSize,
+ lobGCLimit
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
+
- // done with the request (success or error)
- requestTag = -1;
if (err != LOB_OPER_OK)
{
return -(short)err;
}
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
- // error. Status contains the error code.
- return -(short)status;
+
+ return 0;
}
Lng32 ExpLOBInterfaceDelete(void * lobGlob,
@@ -639,7 +677,8 @@ Lng32 ExpLOBInterfaceDelete(void * lobGlob,
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
@@ -655,7 +694,7 @@ Lng32 ExpLOBInterfaceDelete(void * lobGlob,
err = ExLobsOper(lobName,
lobHandle, handleLen,
lobHdfsServer, lobHdfsPort, // hdfs server/port
- NULL, dummyParam,
+ NULL, dummyParam2,
descSyskey, dummyParam,
dummyParam,
requestTag, requestTag,
@@ -671,29 +710,14 @@ Lng32 ExpLOBInterfaceDelete(void * lobGlob,
NULL, 0
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
- // done with the request (success or error)
- requestTag = -1;
if (err != LOB_OPER_OK)
{
return -(short)err;
}
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
- // error. Status contains the error code.
- return -(short)status;
+ return 0;
}
@@ -719,15 +743,14 @@ Lng32 ExpLOBInterfaceSelect(void * lobGlob,
)
{
Ex_Lob_Error err;
-
- Int64 dummyParam;
Ex_Lob_Error status;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Int64 cliError;
LobsOper lo;
- if (checkStatus)
- lo = Lob_Check_Status;
- else if (lobHandle == NULL)
+
+ if (lobHandle == NULL)
{
requestTag = -1;
lo = Lob_ReadDataSimple;
@@ -743,7 +766,7 @@ Lng32 ExpLOBInterfaceSelect(void * lobGlob,
err = ExLobsOper(lobName,
lobHandle, handleLen,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam,
+ NULL, dummyParam2,
srcOffset, dummyParam,
outLen,
requestTag, requestTag,
@@ -759,17 +782,10 @@ Lng32 ExpLOBInterfaceSelect(void * lobGlob,
NULL, 0,
0,
lobMaxMemChunkLen,
- 0,0,0,inputFlags
+ 0,0,0,0,inputFlags
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
-
+
// done with the request (success or error)
requestTag = -1;
if (err != LOB_OPER_OK)
@@ -777,14 +793,9 @@ Lng32 ExpLOBInterfaceSelect(void * lobGlob,
return -(short)err;
}
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
// error. Status contains the error code.
- return -(short)status;
+ return 0;
}
Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
@@ -794,7 +805,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
char * lobHdfsServer,
Lng32 lobHdfsPort,
- Int64 handleLen,
+ Int32 handleLen,
char * lobHandle,
Int64 cursorBytes,
@@ -815,6 +826,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
Ex_Lob_Error err;
Int64 dummyParam = 0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError;
@@ -853,7 +865,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
err = ExLobsOper(lobName,
lobHandle, handleLen,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam,
+ NULL, dummyParam2,
srcOffset, dummyParam,
outLen,
requestTag, requestTag,
@@ -866,7 +878,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
waitedOp,
lobGlob,
0,
- NULL, 0,0,0,0,0,0,
+ NULL, 0,0,0,0,0,0,0,
openType
);
@@ -888,14 +900,15 @@ Lng32 ExpLOBinterfaceStats(
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam= 0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, (LobsStorage)lobType,
(char*)lobStats, 0,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBinterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h
index 646d846..26f2ad4 100644
--- a/core/sql/exp/ExpLOBinterface.h
+++ b/core/sql/exp/ExpLOBinterface.h
@@ -98,9 +98,7 @@ Lng32 ExpLOBinterfaceDrop(void * lobGlob,
char * lobName,
char * lobLoc);
-Lng32 ExpLOBInterfacePurgedata(void * lobGlob,
- char * lobHdfsServer ,
- Lng32 lobHdfsPort ,
+Lng32 ExpLOBInterfacePurgedata(void * lobGlob,
char * lobName,
char * lobLoc);
@@ -135,7 +133,7 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
Lng32 handleLen,
char * lobHandle,
- Int64 * outHandleLen,
+ Int32 * outHandleLen,
char * outLobHandle,
Int64 blackBoxLen,
@@ -156,6 +154,7 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
Int64 srcLobLen = 0,
Int64 lobMaxSize = 0,
Int64 lobMaxChunkMemSize = 0,
+ Int64 lobGCLimit = 0,
int bufferSize = 0,
short replication =0,
int blocksize=0
@@ -168,7 +167,7 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
char * lobLocation,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 &requestTag,
Int64 xnId,
@@ -188,7 +187,8 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
Int64 srcDescKey,
Int64 srcDescTS,
Int64 lobMaxSize = 0,
- Int64 lobMaxChunkMemSize = 0);
+ Int64 lobMaxChunkMemSize = 0,
+ Int64 lobGCLimit = 0);
Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
char * lobHdfsServer ,
@@ -197,7 +197,7 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
char * lobLocation,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 &requestTag,
Int64 xnId,
@@ -217,7 +217,8 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
Int64 srcDescKey,
Int64 srcDescTS,
Int64 lobMaxSize = 0,
- Int64 lobMaxChunkMemSize = 0
+ Int64 lobMaxChunkMemSize = 0,
+ Int64 lobGCLimit = 0
);
Lng32 ExpLOBInterfaceDelete(void * lobGlob,
@@ -260,7 +261,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
char * lobHdfsServer,
Lng32 lobHdfsPort,
- Int64 handleLen,
+ Int32 handleLen,
char * lobHandle,
Int64 cusrorBytes,
char *cursorId,
@@ -286,6 +287,11 @@ Lng32 ExpLOBinterfaceStats(void * lobGlob,
char * getLobErrStr(Lng32 errEnum);
+Lng32 ExpLOBinterfacePerformGC(void *& lobGlob, char *lobName,void *descChunksArray, Int32 numEntries, char *hdfsServer, Int32 hdfsPort,char *LOBlOC,Int64 lobMaxChunkMemSize);
+Lng32 ExpLOBinterfaceRestoreLobDataFile(void *& lobGlob, char *hdfsServer, Int32 hdfsPort,char *lobLoc,char *lobName);
+Lng32 ExpLOBinterfacePurgeBackupLobDataFile(void *& lobGlob, char *hdfsServer, Int32 hdfsPort,char *lobLoc,char *lobName);
+
+
Lng32 ExpLOBinterfaceEmptyDirectory(void * lobGlob,
char * lobName,
char * lobLoc,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBprocess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBprocess.cpp b/core/sql/exp/ExpLOBprocess.cpp
index e121c1e..e520c4c 100644
--- a/core/sql/exp/ExpLOBprocess.cpp
+++ b/core/sql/exp/ExpLOBprocess.cpp
@@ -23,8 +23,8 @@
/* -*-C++-*-
*****************************************************************************
*
- * File: ex_lob.C
- * Description: class to store and retrieve LOB data.
+ * File: ExpLOBprocess.cpp
+ * Description: class to store and retrieve LOB info from mxlobsrvr process.
*
*
* Created: 10/29/2012
@@ -35,7 +35,13 @@
*
*****************************************************************************
*/
+/*** Note *** This file is currently compiled and creates the mxlobsrvr executable. But the functions in this file are not active or used at this point. Code
+ maybe added in the near future to offload any tasks like garbage collection, to this process .Hence we are retainign this file as part of the mxlobsrvr
+infrastructure .If any functionas are added and need to be executed in the
+mxlobsrvr process, the sqstart/sqstop need to modified to call lobstop and
+lostart**/
+/****************************************************************************/
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
@@ -351,85 +357,7 @@ void process_mon_msg(MS_Mon_Msg *msg) {
}
}
-Ex_Lob_Error ExLob::append(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 offset;
- Int64 dummyParam;
- Lng32 handleOutLen;
- Lng32 clierr;
-
- Int64 size = request->getDesc().getSize();
- offset = request->getDataOffset();
-
- /* This is done in the master
- // allocate a new lob desc
- err = allocateDesc(size, dummyParam, offset);
-
- if (err != LOB_OPER_OK)
- return err;
-
- request->getDesc().setOffset(offset);
- */
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- request->getHandleOut(), &handleOutLen,
- LOB_CLI_INSERT_APPEND, LOB_CLI_ExecImmed,
- &offset, &size,
- &dummyParam, &dummyParam,
- 0,
- request->getTransId());
-
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::update(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 offset;
- Int64 dummyParam;
- Lng32 handleOutLen;
- Lng32 clierr;
-
- Int64 size = request->getDesc().getSize();
- offset = request->getDataOffset();
- /* sss this is done in the master
- // allocate a new lob desc
- err = allocateDesc(size, dummyParam, offset);
-
- if (err != LOB_OPER_OK)
- return err;
-
- request->getDesc().setOffset(offset);
- */
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- request->getHandleOut(), &handleOutLen,
- LOB_CLI_UPDATE_UNIQUE, LOB_CLI_ExecImmed,
- &offset, &size,
- &dummyParam, &dummyParam,
- 0,
- request->getTransId());
-
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-Ex_Lob_Error ExLob::putDesc(ExLobDesc &desc, Int64 descNum)
-{
- /* Ex_Lob_Error err = LOB_OPER_OK;
-
- Lng32 offset = sizeof(ExLobDescHeader) + sizeof(ExLobDesc) * descNum;
- if (pwrite(fdDesc_, &desc, sizeof(ExLobDesc), offset) == -1) {
- return LOB_DESC_WRITE_ERROR;
- }
- */
- return LOB_OPER_OK;
-}
Ex_Lob_Error ExLob::getDesc(ExLobRequest *request)
{
@@ -452,255 +380,14 @@ Ex_Lob_Error ExLob::getDesc(ExLobRequest *request)
request->setHandleOutLen(handleOutLen);
request->setBlackBoxLen(blackBoxLen);
- request->getDesc().setOffset(offset);
- request->getDesc().setSize(size);
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::delDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 offset;
- Int64 dummyParam;
- Lng32 handleOutLen;
- Lng32 clierr;
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- (char *)&dummyParam, (Lng32 *)&dummyParam,
- LOB_CLI_DELETE, LOB_CLI_ExecImmed,
- &dummyParam, &dummyParam,
- &dummyParam, &dummyParam,
- 0,
- request->getTransId());
-
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-/*Ex_Lob_Error ExLob::purgeLob()
-{
- Ex_Lob_Error err = LOB_OPER_OK;
-
- close(fdDesc_);
- fdDesc_ = -1;
-
- if (remove(lobDescFile_) != 0) {
- return LOB_DESC_FILE_DELETE_ERROR;
- }
-
- return LOB_OPER_OK;
- } */
-
-Ex_Lob_Error ExLob::purgeLob()
-{
- Ex_Lob_Error err = LOB_OPER_OK;
-
- hdfsCloseFile(fs_,fdDesc_);
- fdDesc_ = NULL;
-
- if (hdfsDelete(fs_,lobDescFile_,0) != 0) {
- return LOB_DESC_FILE_DELETE_ERROR;
- }
-
- return LOB_OPER_OK;
-}
-
-
-Ex_Lob_Error ExLob::getNextFreeOffset(Int64 size, Int64 &dataOffset, Int64 lobMaxSize)
-{
- dataOffset = 0;
- // Retrieve the next available, best-fit free offset to write into.
-
- if (sourceLen <= 0 || sourceLen > lobMaxSize)
- {
- return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size
- }
-}
-
-
-/*Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset)
-{
- Ex_Lob_Error err = LOB_OPER_OK;
-
-
- err = lockDesc();
- if (err != LOB_OPER_OK)
- return err;
- ExLobDescHeader header;
-
- if (pread(fdDesc_, &header, sizeof(ExLobDescHeader), 0) == -1) {
- flock(fdDesc_, LOCK_UN);
- return LOB_DESC_HEADER_READ_ERROR;
- }
-
- if (header.getAvailSize() >= size) {
- descNum = header.getFreeDesc();
- dataOffset = header.getDataOffset();
- header.incFreeDesc();
- header.decAvailSize(size);
- header.incDataOffset(size);
-
- if (pwrite(fdDesc_, &header, sizeof(ExLobDescHeader), 0) == -1) {
- err = LOB_DESC_HEADER_WRITE_ERROR;
- }
-
- }
- else {
- err = LOB_DATA_FILE_FULL_ERROR;
- }
-
-
- ExLobDesc desc(dataOffset, size, descNum);
- if (pwrite(fdDesc_, &desc, sizeof(ExLobDesc),
- sizeof(ExLobDescHeader) + sizeof(ExLobDesc) * descNum) == -1) {
- return LOB_DESC_WRITE_ERROR;
- }
-
- err = unlockDesc();
- if (err != LOB_OPER_OK)
- return err;
-
- return err;
-} */
-
-
-
-Ex_Lob_Error ExLob::insertDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Lng32 clierr;
- Int64 dummyParam;
- Lng32 handleOutLen = 0;
- Lng32 blackBoxLen = 0;
- Int64 offset = request->getDataOffset();
- Int64 size = request->getDesc().getSize();
-
- blackBoxLen = request->getBlackBoxLen();
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- request->getBlackBox(), &blackBoxLen,
- request->getHandleOut(), &handleOutLen,
- LOB_CLI_INSERT, LOB_CLI_ExecImmed,
- &offset, &size,
- &dummyParam, &dummyParam,
- 0,
- request->getTransId());
-
- request->setHandleOutLen(handleOutLen);
- request->setBlackBoxLen(blackBoxLen);
request->setCliError(clierr);
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::selectCursorDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 dummyParam;
- void *cliInterface = NULL;
- cursor_t cursor;
- int clierr;
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- (char *)&dummyParam, (Lng32 *)&dummyParam,
- LOB_CLI_SELECT_CURSOR, LOB_CLI_ExecImmed,
- &dummyParam, &dummyParam,
- &dummyParam, &dummyParam,
- &cliInterface,
- request->getTransId());
-
- cursor.cliInterface_ = cliInterface;
- // these are only used in library, but initialize them here anyway.
- cursor.bytesRead_ = -1;
- cursor.descOffset_ = -1;
- cursor.descSize_ = -1;
- cursor.eod_ = false;
- lobCursors_it it = lobCursors_.find(string(request->getHandleIn(), request->getHandleInLen()));
- if (it == lobCursors_.end())
- {
- lobCursors_.insert(pair<string, cursor_t>
- (string(request->getHandleIn(), request->getHandleInLen()), cursor));
- }
- else
- {
- it->second = cursor;
- }
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::fetchCursorDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 dummyParam;
- Int64 offset = -1;
- Int64 size = -1;
- int clierr;
-
- lobCursors_it it = lobCursors_.find(string(request->getHandleIn(), request->getHandleInLen()));
-
- if (it == lobCursors_.end())
- {
- return LOB_CURSOR_NOT_OPEN;
- }
-
- void *cliInterface = it->second.cliInterface_;
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- (char *)&dummyParam, (Lng32 *)&dummyParam,
- LOB_CLI_SELECT_FETCH, LOB_CLI_ExecImmed,
- &offset, &size,
- &dummyParam, &dummyParam,
- &cliInterface,
- request->getTransId());
-
- request->getDesc().setOffset(offset);
- request->getDesc().setSize(size);
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::closeCursorDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 dummyParam;
- int clierr;
-
- lobCursors_it it = lobCursors_.find(string(request->getHandleIn(), request->getHandleInLen()));
-
- if (it == lobCursors_.end())
- {
- return LOB_CURSOR_NOT_OPEN;
- }
-
- void *cliInterface = it->second.cliInterface_;
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- NULL, NULL,
- (char *)&dummyParam, (Lng32 *)&dummyParam,
- LOB_CLI_SELECT_CLOSE, LOB_CLI_ExecImmed,
- &dummyParam, &dummyParam,
- &dummyParam, &dummyParam,
- &cliInterface,
- request->getTransId());
-
- // do not update cliErr in request because the fetch would have updated it.
-
- // remove cursor from the map.
- err = closeCursor(request->getHandleIn(), request->getHandleInLen());
-
- return LOB_OPER_OK;
-}
void processRequest(ExLobRequest *request)
{
@@ -712,7 +399,7 @@ void processRequest(ExLobRequest *request)
ExLob *lobPtr;
ExLobDesc desc;
ExLobDesc *descPtr;
- Lng32 clierr;\
+ Lng32 clierr;
Lng32 handleOutLen;
Int64 size;
@@ -741,58 +428,10 @@ void processRequest(ExLobRequest *request)
}
switch(request->getType())
{
- case Lob_Req_Allocate_Desc:
- /* sss no need for this since it's done in master
- err = lobPtr->allocateDesc(request->getDesc().getSize(), descNum, dataOffset);
-
- request->setDescNumOut(descNum);
- request->setDataOffset(dataOffset);
- */
- err = lobPtr->insertDesc(request);
- break;
-
- case Lob_Req_Append:
- err = lobPtr->append(request);
- break;
-
- case Lob_Req_Select_Cursor:
- err = lobPtr->selectCursorDesc(request);
- break;
-
- case Lob_Req_Fetch_Cursor:
- err = lobPtr->fetchCursorDesc(request);
- if (request->getCliError() == 100) {
- lobPtr->closeCursorDesc(request);
- }
- break;
-
- case Lob_Req_Update:
- err = lobPtr->update(request);
- break;
-
case Lob_Req_Get_Desc:
err = lobPtr->getDesc(request);
break;
- case Lob_Req_Put_Desc:
- err = lobPtr->putDesc(request->getDesc(), request->getDescNumIn());
- break;
-
- case Lob_Req_Del_Desc:
- err = lobPtr->delDesc(request);
- break;
-
- case Lob_Req_Purge_Desc:
- /* no op ..it is dropped in mastererr = lobPtr->purgeLob();
- if (err == LOB_OPER_OK) {
- err = lobGlobals->delLobPtr(request->getDescFileName());
- } */
- break;
-
- case Lob_Req_Print:
- lobPtr->print();
- break;
-
default:
err = LOB_REQUEST_UNDEFINED_ERROR;
printf("bad request = %d\n", request->getType());
@@ -801,23 +440,12 @@ void processRequest(ExLobRequest *request)
request->setError(err);
- request->log();
+
return;
}
-void ExLobRequest::log()
-{
- printf("req num = %ld\n", getReqNum());
- printf("req type = %d\n", getType());
- printf("trans id = %ld\n", getTransId());
- printf("file = %s\n", getDescFileName());
- printf("size = %d\n", getDesc().getSize());
- printf("offset = %d\n", getDesc().getOffset());
- printf("lob err = %d\n", getError());
- printf("cli err = %d\n", getCliError());
- printf("\n");
-}
+
void receive_message(ExLobRequest *request)
{
@@ -859,27 +487,9 @@ Ex_Lob_Error ExLobGlobals::initialize()
return LOB_OPER_OK;
}
-/*Ex_Lob_Error ExLob::initialize(char *lobFile)
-{
- mode_t filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
- Lng32 openFlags = O_RDWR;
-
- strcpy(lobDescFile_, lobFile);
- fdDesc_ = open(lobDescFile_, openFlags);
- if (fdDesc_ == -1) {
- return LOB_DESC_FILE_OPEN_ERROR;
- }
- return LOB_OPER_OK;
- }*/
-Ex_Lob_Error ExLob::initialize(char *lobDescFile)
-{
- // Made this obsolete
-
- return LOB_OPER_OK;
-}
Ex_Lob_Error ExLobGlobals::getLobPtr(char *lobName, ExLob *& lobPtr)
{
@@ -922,106 +532,11 @@ Ex_Lob_Error ExLobGlobals::delLobPtr(char *lobName)
return LOB_OPER_OK;
}
-/*
-Ex_Lob_Error ExLob::lockDesc()
-{
-
- // get exclusive lock because read and write has to be atomic
- if (flock(fdDesc_, LOCK_EX) == -1) {
- return LOB_DESC_FILE_LOCK_ERROR;
- }
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::unlockDesc()
-{
- // release the lock
-
- if (flock(fdDesc_, LOCK_UN) == -1) {
- return LOB_DESC_FILE_LOCK_ERROR;
- }
-
- return LOB_OPER_OK;
-}
-*/
-
-
-Ex_Lob_Error ExLobDescHeader::print(char *descFileName)
-{
- printf("Header: \n");
- printf("freeDesc = %d\n", freeDesc_);
- printf("dataOffset = %d\n", dataOffset_);
- printf("avail Size = %d\n", availSize_);
- return LOB_OPER_OK;
-}
-/*Ex_Lob_Error ExLob::print()
-{
- // get exclusive lock because read and write has to be atomic
- if (flock(fdDesc_, LOCK_EX) == -1) {
- return LOB_DESC_FILE_LOCK_ERROR;
- }
-
- ExLobDescHeader header;
- if (pread(fdDesc_, &header, sizeof(ExLobDescHeader), 0) == -1) {
- return LOB_DESC_HEADER_READ_ERROR;
- }
- header.print(NULL);
- printf("Descriptors:\n");
- printf("dNum size state tail prev next nextfree offset\n");
- ExLobDesc desc;
- for (Lng32 numDescs = 0; numDescs < header.getFreeDesc(); numDescs++) {
- if (pread(fdDesc_, &desc, sizeof(ExLobDesc),
- sizeof(ExLobDescHeader) + sizeof(ExLobDesc) * numDescs) == -1) {
- flock(fdDesc_, LOCK_UN);
- return LOB_DESC_READ_ERROR;
- }
- printf("%4d ", numDescs);
- desc.print();
- }
- flock(fdDesc_, LOCK_UN);
- return LOB_OPER_OK;
-}
-*/
-
-Ex_Lob_Error ExLob::print()
-{
- // get exclusive lock because read and write has to be atomic
- Int32 retval = 0;
- //get a mutex to prevent multiple updates
- retval = pthread_mutex_lock(&g_mutex);
- if (retval != 0 )
- return LOB_ALLOC_ERROR;
- long openFlags = O_RDONLY ;
- ExLobDescHeader header;
- if (hdfsPread(fs_,fdDesc_,0, &header, sizeof(ExLobDescHeader)) <= 0) {
- return LOB_DESC_HEADER_READ_ERROR;
- }
-
- header.print(NULL);
-
- printf("Descriptors:\n");
-
- printf("dNum size state tail prev next nextfree offset\n");
- ExLobDesc desc;
- for (Lng32 numDescs = 0; numDescs < header.getFreeDesc(); numDescs++) {
- if (hdfsPread(fs_,fdDesc_, sizeof(ExLobDescHeader) + sizeof(ExLobDesc) * numDescs, &desc, sizeof(ExLobDesc) ) <= 0 ) {
- pthread_mutex_unlock(&g_mutex);
- return LOB_DESC_READ_ERROR;
- }
- printf("%4d ", numDescs);
- desc.print();
- }
-
-
- pthread_mutex_unlock(&g_mutex);
- return LOB_OPER_OK;
-}
Lng32 main(Lng32 argc, char *argv[])
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBprocess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBprocess.h b/core/sql/exp/ExpLOBprocess.h
index b7112d5..2e7ffb8 100644
--- a/core/sql/exp/ExpLOBprocess.h
+++ b/core/sql/exp/ExpLOBprocess.h
@@ -23,7 +23,7 @@
/* -*-C++-*-
*****************************************************************************
*
- * File: ex_lob.h
+ * File: ExLOBprocess.h
* Description: header file for LOB storage and retrieval
*
*
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/generator/GenItemFunc.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenItemFunc.cpp b/core/sql/generator/GenItemFunc.cpp
index c7e6748..a41c45a 100644
--- a/core/sql/generator/GenItemFunc.cpp
+++ b/core/sql/generator/GenItemFunc.cpp
@@ -2839,6 +2839,9 @@ short LOBinsert::codeGen(Generator * generator)
li->setLobStorageLocation((char*)lobStorageLocation().data());
li->setLobMaxSize(getLobMaxSize());
li->setLobMaxChunkMemSize(getLobMaxChunkMemSize());
+ li->setLobGCLimit(getLobGCLimit());
+ li->setLobHdfsServer((char *)getLobHdfsServer().data());
+ li->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, li);
return 0;
@@ -2864,7 +2867,8 @@ short LOBdelete::codeGen(Generator * generator)
ld->lobNum() = lobNum();
ld->setLobStorageType(lobStorageType());
ld->setLobStorageLocation((char*)lobStorageLocation().data());
-
+ ld->setLobHdfsServer((char *)getLobHdfsServer().data());
+ ld->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, ld);
return 0;
@@ -2908,6 +2912,9 @@ short LOBupdate::codeGen(Generator * generator)
lu->setLobStorageLocation((char*)lobStorageLocation().data());
lu->setLobMaxSize(getLobMaxSize());
lu->setLobMaxChunkMemSize(getLobMaxChunkMemSize());
+ lu->setLobGCLimit(getLobGCLimit());
+ lu->setLobHdfsServer((char *)getLobHdfsServer().data());
+ lu->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, lu);
return 0;
@@ -2929,7 +2936,8 @@ short LOBselect::codeGen(Generator * generator)
ls->lobNum() = lobNum();
ls->setLobStorageType(lobStorageType());
ls->setLobStorageLocation((char*)lobStorageLocation().data());
-
+ ls->setLobHdfsServer((char *)getLobHdfsServer().data());
+ ls->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, ls);
return 0;
@@ -2957,7 +2965,8 @@ short LOBconvertHandle::codeGen(Generator * generator)
lu->lobNum() = lobNum();
lu->setLobStorageType(lobStorageType());
lu->setLobStorageLocation((char*)lobStorageLocation().data());
-
+ lu->setLobHdfsServer((char *)getLobHdfsServer().data());
+ lu->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, lu);
return 0;
@@ -2987,6 +2996,8 @@ short LOBconvert::codeGen(Generator * generator)
lc->setLobStorageLocation((char*)lobStorageLocation().data());
generator->getExpGenerator()->linkClause(this, lc);
lc->setConvertSize(getTgtSize());
+ lc->setLobHdfsServer((char *)getLobHdfsServer().data());
+ lc->setLobHdfsPort(getLobHdfsPort());
return 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/optimizer/ItemFunc.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h
index 3bb7238..29776f0 100644
--- a/core/sql/optimizer/ItemFunc.h
+++ b/core/sql/optimizer/ItemFunc.h
@@ -2708,8 +2708,12 @@ public:
lobNum_(-1),
lobStorageType_(Lob_Invalid_Storage),
lobMaxSize_(CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)),
- lobMaxChunkMemSize_(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE))
- {}
+ lobMaxChunkMemSize_(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)),
+ lobGCLimit_(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)),
+ hdfsPort_((Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT)),
+ hdfsServer_( CmpCommon::getDefaultString(LOB_HDFS_SERVER))
+ {
+ }
// copyTopNode method
virtual ItemExpr * copyTopNode(ItemExpr *derivedNode = NULL,
@@ -2736,8 +2740,10 @@ public:
LobsStorage &lobStorageType() { return lobStorageType_; }
NAString &lobStorageLocation() { return lobStorageLocation_; }
Int64 getLobMaxSize() {return lobMaxSize_*1024*1024; }
- Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_;}
-
+ Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_*1024*1024;}
+ Int64 getLobGCLimit() { return lobGCLimit_*1025*1024;}
+ Int32 getLobHdfsPort() { return hdfsPort_;}
+ NAString &getLobHdfsServer(){return hdfsServer_;}
protected:
ObjectType obj_;
@@ -2746,6 +2752,9 @@ public:
NAString lobStorageLocation_;
Int32 lobMaxSize_; // In MB units
Int32 lobMaxChunkMemSize_; //In MB Units
+ Int32 lobGCLimit_ ;//In MB Units
+ Int32 hdfsPort_;
+ NAString hdfsServer_;
}; // LOBoper
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index 3e21c79..ef46a4d 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -5506,7 +5506,8 @@ NATable::NATable(BindWA *bindWA,
if(postCreateNATableWarnings != preCreateNATableWarnings)
tableConstructionHadWarnings_=TRUE;
-
+ const char *lobHdfsServer = CmpCommon::getDefaultString(LOB_HDFS_SERVER);
+ Int32 lobHdfsPort = (Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT);
if (hasLobColumn())
{
// read lob related information from lob metadata
@@ -5546,7 +5547,7 @@ NATable::NATable(BindWA *bindWA,
LOB_CLI_SELECT_CURSOR,
lobNumList,
lobTypList,
- lobLocList,0);
+ lobLocList,(char *)lobHdfsServer,lobHdfsPort,0);
if (cliRC == 0)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
index 28cd8d7..174cb6f 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
@@ -812,7 +812,8 @@ short CmpSeabaseMDcleanup::dropLOBs(ExeCliInterface *cliInterface)
return 0;
NAString newSchName = "\"" + catName_ + "\"" + "." + "\"" + schName_ + "\"";
-
+ const char *lobHdfsServer = CmpCommon::getDefaultString(LOB_HDFS_SERVER);
+ Int32 lobHdfsPort = (Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT);
cliRC = SQL_EXEC_LOBddlInterface((char*)newSchName.data(),
newSchName.length(),
objUID_,
@@ -820,7 +821,9 @@ short CmpSeabaseMDcleanup::dropLOBs(ExeCliInterface *cliInterface)
LOB_CLI_CLEANUP,
lobNumList_,
lobTypList_,
- lobLocList_,0);
+ lobLocList_,
+ (char *)lobHdfsServer,
+ lobHdfsPort,0);
return 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
index d5a0efa..b3e59f8 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
@@ -2190,6 +2190,8 @@ short CmpSeabaseDDL::createSeabaseTable2(
}
Int64 lobMaxSize = CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024;
+ const char *lobHdfsServer = CmpCommon::getDefaultString(LOB_HDFS_SERVER);
+ Int32 lobHdfsPort = (Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT);
if (j > 0)
{
//if the table is a volatile table return an error
@@ -2221,6 +2223,8 @@ short CmpSeabaseDDL::createSeabaseTable2(
lobNumList,
lobTypList,
lobLocList,
+ (char *)lobHdfsServer,
+ lobHdfsPort,
lobMaxSize);
if (rc < 0)
@@ -3519,6 +3523,8 @@ short CmpSeabaseDDL::dropSeabaseTable2(
short *lobNumList = new (STMTHEAP) short[numCols];
short *lobTypList = new (STMTHEAP) short[numCols];
char **lobLocList = new (STMTHEAP) char*[numCols];
+ const char *lobHdfsServer = CmpCommon::getDefaultString(LOB_HDFS_SERVER);
+ Int32 lobHdfsPort = (Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT);
Lng32 j = 0;
for (Int32 i = 0; i < nacolArr.entries(); i++)
{
@@ -3539,7 +3545,7 @@ short CmpSeabaseDDL::dropSeabaseTable2(
const char* f = ActiveSchemaDB()->getDefaults().
getValue(LOB_STORAGE_FILE_DIR);
-
+
strcpy(loc, f);
lobLocList[j] = loc;
@@ -3565,7 +3571,7 @@ short CmpSeabaseDDL::dropSeabaseTable2(
LOB_CLI_DROP,
lobNumList,
lobTypList,
- lobLocList,0);
+ lobLocList,(char *)lobHdfsServer, lobHdfsPort,0);
if (rc < 0)
{
*CmpCommon::diags() << DgSqlCode(-CAT_UNABLE_TO_DROP_OBJECT)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h
index 1a988b4..c704567 100644
--- a/core/sql/sqlcomp/DefaultConstants.h
+++ b/core/sql/sqlcomp/DefaultConstants.h
@@ -3296,6 +3296,7 @@ enum DefaultConstants
LOB_HDFS_SERVER,
LOB_HDFS_PORT,
+ LOB_GC_LIMIT_SIZE,
// Should the DISK POOL be turned on when replicating the DDL using COPY DDL
REPLICATE_DISK_POOL,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index b4aaa78..2ecbe9e 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -2068,12 +2068,16 @@ SDDkwd__(ISO_MAPPING, (char *)SQLCHARSETSTRING_ISO88591),
// precision but degraded performance.
SDDkwd__(LIMIT_MAX_NUMERIC_PRECISION, "SYSTEM"),
+ // Size in bytes used to perform garbage collection to lob data file
+ // default size is 5GB . Change to adjust disk usage.
+ DDint__(LOB_GC_LIMIT_SIZE, "5000"),
+
DDint__(LOB_HDFS_PORT, "0"),
DD_____(LOB_HDFS_SERVER, "default"),
-
+
// Size of memoryin bytes used to perform I/O to lob data file
// default size is 512MB . Change to adjust memory usage.
- DDint__(LOB_MAX_CHUNK_MEM_SIZE, "536870912"),
+ DDint__(LOB_MAX_CHUNK_MEM_SIZE, "512"),
// default size is 10 G (10000 M)
DDint__(LOB_MAX_SIZE, "10000"),
// default size is 32000. Change this to extract more data into memory.
@@ -6920,6 +6924,10 @@ DefaultToken NADefaults::token(Int32 attrEnum,
isValid = TRUE;
break;
+ case LOB_GC_LIMIT_SIZE:
+ if (tok >= 0 )
+ isValid=TRUE;
+
case TRAF_TRANS_TYPE:
if (tok == DF_MVCC || tok == DF_SSCC)
isValid = TRUE;
[07/11] incubator-trafodion git commit: Adding charset info for
lobhandle. New regression test for execrcising GC.
Posted by sa...@apache.org.
Adding charset info for lobhandle. New regression test for execrcising GC.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/95035198
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/95035198
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/95035198
Branch: refs/heads/master
Commit: 950351981c99e2876de50ad24c3a48bbb29e11e8
Parents: 70e7a09
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Thu Mar 3 17:31:15 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Thu Mar 3 17:31:15 2016 +0000
----------------------------------------------------------------------
.../sqf/conf/log4cxx.trafodion.masterexe.config | 1 +
core/sqf/conf/log4j.dtm.config | 2 +-
core/sqf/conf/log4j.hdfs.config | 4 +-
core/sqf/sql/scripts/gensq.pl | 18 ++++
core/sql/cli/Cli.cpp | 4 +-
core/sql/cli/CliExpExchange.cpp | 4 +-
core/sql/cli/Descriptor.cpp | 6 +-
core/sql/common/CharType.cpp | 4 +-
core/sql/exp/ExpLOBaccess.cpp | 2 +-
core/sql/generator/GenExpGenerator.cpp | 3 +-
core/sql/regress/executor/EXPECTED130 | 106 ++++++++++++++++---
core/sql/regress/executor/TEST130 | 32 ++++++
12 files changed, 160 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sqf/conf/log4cxx.trafodion.masterexe.config
----------------------------------------------------------------------
diff --git a/core/sqf/conf/log4cxx.trafodion.masterexe.config b/core/sqf/conf/log4cxx.trafodion.masterexe.config
index 5a8e96d..5d037ce 100644
--- a/core/sqf/conf/log4cxx.trafodion.masterexe.config
+++ b/core/sqf/conf/log4cxx.trafodion.masterexe.config
@@ -47,6 +47,7 @@ log4j.appender.mxoAppender.layout.ConversionPattern=%d, %p, %c, %m%n
log4j.logger.MXOSRVR=ERROR, mxoAppender
# SQL
+#log4j.logger.SQL.EXE=DEBUG
log4j.logger.SQL=INFO
log4j.logger.SQL.COMP=ERROR
log4j.logger.SQL.EXE.Memory=INFO
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sqf/conf/log4j.dtm.config
----------------------------------------------------------------------
diff --git a/core/sqf/conf/log4j.dtm.config b/core/sqf/conf/log4j.dtm.config
index b7fdb16..69bea7e 100644
--- a/core/sqf/conf/log4j.dtm.config
+++ b/core/sqf/conf/log4j.dtm.config
@@ -53,5 +53,5 @@ log4j.appender.hbaseclient.Append=true
log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.hadoop.hbase=ERROR
log4j.logger.org.trafodion.dtm=INFO
-log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
+#log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sqf/conf/log4j.hdfs.config
----------------------------------------------------------------------
diff --git a/core/sqf/conf/log4j.hdfs.config b/core/sqf/conf/log4j.hdfs.config
index 0a0ac49..9505331 100644
--- a/core/sqf/conf/log4j.hdfs.config
+++ b/core/sqf/conf/log4j.hdfs.config
@@ -50,5 +50,5 @@ log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.hadoop.hbase=ERROR
log4j.logger.org.trafodion.sql=ERROR
-log4j.logger.org.apache.hadoop.hbase=DEBUG
-log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
+#log4j.logger.org.apache.hadoop.hbase=DEBUG
+#log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sqf/sql/scripts/gensq.pl
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/gensq.pl b/core/sqf/sql/scripts/gensq.pl
index ea18311..d3892e5 100755
--- a/core/sqf/sql/scripts/gensq.pl
+++ b/core/sqf/sql/scripts/gensq.pl
@@ -558,6 +558,24 @@ sub genDTM {
printScript(1, "done\n");
printScript(1, "echo \"The Transaction Service is Ready.\"\n");
}
+sub genLOBConfig {
+
+ # Generate sqconfig.db config for LOB.
+ # This allows the process startup daemon (pstartd)
+ # to start it up after a node failure.
+ for ($i=0; $i < $gdNumNodes; $i++) {
+ my $l_progname="mxlobsrvr";
+ my $l_procargs="";
+ my $l_procname="\$ZLOBSRV$i";
+ my $l_procname_config = sprintf('$ZLOBSRV%d', $i);
+ my $l_stdout="stdout_\$ZLOBSRV_$i";
+ addDbProcData($l_procname_config, "PERSIST_RETRIES", "10,60");
+ addDbProcData($l_procname_config, "PERSIST_ZONES", $i);
+ addDbPersistProc($l_procname_config, $i, 1);
+ addDbProcDef( $ProcessType_Generic, $l_procname_config, $i, $l_progname, $l_stdout, $l_procargs);
+ }
+
+}
sub genSSMPCommand {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index e762714..3ed4e06 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -10482,7 +10482,7 @@ Lng32 SQLCLI_LOB_GC_Interface
{
cliInterface->retrieveSQLDiagnostics(myDiags);
- //tbd Give warning and rollback transaction and return with warning. For now return error and abort the iud operation itself.
+ //tbd Give warning and rollback just these updates and return with warning. For now return error and abort the iud operation itself since there is no support for nested transactions or SUSPEND and RESUME.
goto error_return;
}
i++;
@@ -10509,7 +10509,7 @@ Lng32 SQLCLI_LOB_GC_Interface
// Restore original data file.
Int32 rc2=ExpLOBoper::restoreLobDataFile(lobGlobals,tgtLobName, (void *)currContext.exHeap(),hdfsServer,hdfsPort,lobLocation);
- // if error restoring, mark this lob as corrupt.
+ // if error restoring, this lob could become corrupt.
goto error_return;
}
else
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sql/cli/CliExpExchange.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/CliExpExchange.cpp b/core/sql/cli/CliExpExchange.cpp
index 7105151..871985d 100644
--- a/core/sql/cli/CliExpExchange.cpp
+++ b/core/sql/cli/CliExpExchange.cpp
@@ -207,7 +207,9 @@ ex_expr::exp_return_type InputOutputExpr::describeOutput(void * output_desc_,
// Use SQLDESC_CHAR_SET_NAM (one-part name) for charset
- if ( DFS2REC::isAnyCharacter(operand->getDatatype()) )
+ if ( DFS2REC::isAnyCharacter(operand->getDatatype()) ||
+ (operand->getDatatype() == REC_BLOB) ||
+ (operand->getDatatype() == REC_CLOB ))
{
output_desc->setDescItem(entry, SQLDESC_CHAR_SET_NAM, 0,
(char*)CharInfo::getCharSetName(operand->getCharSet()));
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sql/cli/Descriptor.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Descriptor.cpp b/core/sql/cli/Descriptor.cpp
index 7cf666a..e369ee6 100644
--- a/core/sql/cli/Descriptor.cpp
+++ b/core/sql/cli/Descriptor.cpp
@@ -2931,9 +2931,9 @@ RETCODE Descriptor::setDescItem(Lng32 entry, Lng32 what_to_set,
if ( charset_name == NULL )
charset_name = (char*)CharInfo::getCharSetName(newCharSet);
- if ( !DFS2REC::isAnyCharacter(descItem.datatype) ||
- !CharInfo::isCharSetSupported(newCharSet)
- )
+ if (( !DFS2REC::isAnyCharacter(descItem.datatype) ||
+ !CharInfo::isCharSetSupported(newCharSet)
+ ) && !((descItem.datatype != REC_BLOB) ||(descItem.datatype != REC_CLOB)))
{
//Error 8895: An invalid character set name for the descriptor item SQLDESC_CHAR_SET.
diags << DgSqlCode(-CLI_INVALID_CHARSET_FOR_DESCRIPTOR) << DgString0(charset_name);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sql/common/CharType.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/CharType.cpp b/core/sql/common/CharType.cpp
index 1b21fd1..6f9e8a3 100644
--- a/core/sql/common/CharType.cpp
+++ b/core/sql/common/CharType.cpp
@@ -1536,7 +1536,7 @@ SQLBlob::SQLBlob(
externalFormat,
extFormatLen)
{
- setCharSet(CharInfo::DefaultCharSet);
+ setCharSet(CharInfo::ISO88591);//lobhandle can only be in ISO format
}
NAType *SQLBlob::newCopy(NAMemory* h) const
@@ -1589,7 +1589,7 @@ SQLClob::SQLClob(
externalFormat,
extFormatLen)
{
- setCharSet(CharInfo::DefaultCharSet);
+ setCharSet(CharInfo::ISO88591); //lob handle can only be in this format
}
NAType *SQLClob::newCopy(NAMemory* h) const
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index e4a0c90..217f96a 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -122,7 +122,7 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
struct timespec startTime;
struct timespec endTime;
Int64 secs, nsecs, totalnsecs;
-
+
if (dir)
{
if (dir_.empty())
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sql/generator/GenExpGenerator.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenExpGenerator.cpp b/core/sql/generator/GenExpGenerator.cpp
index 5629bda..c03e369 100644
--- a/core/sql/generator/GenExpGenerator.cpp
+++ b/core/sql/generator/GenExpGenerator.cpp
@@ -294,7 +294,7 @@ Attributes * ExpGenerator::convertNATypeToAttributes
attr->setIsoMapping((CharInfo::CharSet)SqlParser_ISO_MAPPING);
- if (naType->getTypeQualifier() == NA_CHARACTER_TYPE)
+ if (naType->getTypeQualifier() == NA_CHARACTER_TYPE)
{
const CharType *charType = (CharType *)naType;
@@ -378,6 +378,7 @@ Attributes * ExpGenerator::convertNATypeToAttributes
Int16 scale = lobLen & 0xFFFF;
attr->setPrecision(precision);
attr->setScale(scale);
+ attr->setCharSet(lobType->getCharSet());
}
else
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sql/regress/executor/EXPECTED130
----------------------------------------------------------------------
diff --git a/core/sql/regress/executor/EXPECTED130 b/core/sql/regress/executor/EXPECTED130
index 342c076..741fcaa 100644
--- a/core/sql/regress/executor/EXPECTED130
+++ b/core/sql/regress/executor/EXPECTED130
@@ -53,9 +53,9 @@ C1
C1 C2
----------- ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------
- 1 LOBH00000200010520117997292580853919884121437179977294818212317485757218627020"TRAFODION"."LOB130"
- 2 LOBH00000200010520117997292580853919884121437180341874318212317485760955694020"TRAFODION"."LOB130"
- 3 LOBH00000200010520117997292580853919884121437180704085518212317485764542640020"TRAFODION"."LOB130"
+ 1 LOBH00000200010592176192134729966219592176192139496647918212323493795172702020"TRAFODION"."LOB130"
+ 2 LOBH00000200010592176192134729966219592176192140030953518212323493800597686020"TRAFODION"."LOB130"
+ 3 LOBH00000200010592176192134729966219592176192140527060218212323493805564311020"TRAFODION"."LOB130"
--- 3 row(s) selected.
>>
@@ -333,7 +333,7 @@ And the dish ran away with the fork !
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_txt1.txt');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH00000200010520117997292583625519884121437206093184618212317486018305654020"TRAFODION"."LOB130" ' , 'tlob130_txt1.txt');
+>>extract lobtofile(LOB 'LOBH00000200010592176192134733245719592176192171050485618212323494110664949020"TRAFODION"."LOB130"
' , 'tlob130_txt1.txt');
Success. Targetfile :tlob130_txt1.txt Length : 19
--- SQL operation complete.
@@ -349,7 +349,7 @@ Success. Targetfile :tlob130_txt1.txt Length : 19
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_deep.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH00000200010520117997292583681719884121437210516812518212317486062586654020"TRAFODION"."LOB130"
' , 'tlob130_deep.jpg');
+>>extract lobtofile(LOB 'LOBH00000200010592176192134733309019592176192176652813118212323494166743790020"TRAFODION"."LOB130"
' , 'tlob130_deep.jpg');
Success. Targetfile :tlob130_deep.jpg Length : 159018
--- SQL operation complete.
@@ -365,7 +365,7 @@ Success. Targetfile :tlob130_deep.jpg Length : 159018
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_anoush.jpg');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH00000200010520117997292583681719884121437210516812518212317486062586654020"TRAFODION"."LOB130"
' , 'tlob130_anoush.jpg');
+>>extract lobtofile(LOB 'LOBH00000200010592176192134733309019592176192176652813118212323494166743790020"TRAFODION"."LOB130"
' , 'tlob130_anoush.jpg');
Success. Targetfile :tlob130_anoush.jpg Length : 230150
--- SQL operation complete.
@@ -484,7 +484,7 @@ And the dish ran away with the fork !
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'hdfs:\/\/\/lobs\/tlob130_txt2.txt');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH00000200010520117997292584465719884121437216347907218212317486120465668020"TRAFODION"."LOB130" ' , 'hdfs:///lobs/tlob130_txt2.txt');
+>>extract lobtofile(LOB 'LOBH00000200010592176192134734270219592176192182731401318212323494227492370020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_txt2.txt');
Success. Targetfile :hdfs:///lobs/tlob130_txt2.txt Length : 19
--- SQL operation complete.
@@ -500,7 +500,7 @@ Success. Targetfile :hdfs:///lobs/tlob130_txt2.txt Length : 19
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'hdfs:\/\/\/lobs\/tlob130_deep.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH00000200010520117997292584531219884121437220235037418212317486159808683020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_deep.jpg');
+>>extract lobtofile(LOB 'LOBH00000200010592176192134734319619592176192188535869918212323494285449446020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_deep.jpg');
Success. Targetfile :hdfs:///lobs/tlob130_deep.jpg Length : 159018
--- SQL operation complete.
@@ -516,7 +516,7 @@ Success. Targetfile :hdfs:///lobs/tlob130_deep.jpg Length : 159018
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'hdfs:\/\/\/lobs\/tlob130_anoush.jpg');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH00000200010520117997292583681719884121437210516812518212317486062586654020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_anoush.jpg');
+>>extract lobtofile(LOB 'LOBH00000200010592176192134733309019592176192176652813118212323494166743790020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_anoush.jpg');
Success. Targetfile :hdfs:///lobs/tlob130_anoush.jpg Length : 230150
--- SQL operation complete.
@@ -535,7 +535,7 @@ Column Name : c2
Input a filename to extract to :
Output File Name : lobc2out.jpg
Extracting lob handle for column c2...
-LOB handle for c2: LOBH00000200010520117997292583681719884121437210516812518212317486062586654020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH00000200010592176192134733309019592176192176652813118212323494166743790020"TRAFODION"."LOB130"
Extracting LOB data length for the above handle...
LOB data length :230150
Extracting lob data into file in chunks of 1000 ...
@@ -813,7 +813,7 @@ And the dish ran away with the spoon.
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_deep2.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH00000200020520117997292585982119884121437229332169918212317486250825302020"TRAFODION"."LOB130"
' , 'tlob130_deep2.jpg');
+>>extract lobtofile(LOB 'LOBH00000200020592176192134736068419592176192198951613118212323494389654681020"TRAFODION"."LOB130"
' , 'tlob130_deep2.jpg');
Success. Targetfile :tlob130_deep2.jpg Length : 159018
--- SQL operation complete.
@@ -822,7 +822,7 @@ Success. Targetfile :tlob130_deep2.jpg Length : 159018
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'hdfs:\/\/\/lobs\/tlob130_anoush2.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH00000200030520117997292585982119884121437229667368718212317486254205249020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_anoush2.jpg');
+>>extract lobtofile(LOB 'LOBH00000200030592176192134736068419592176192199447013618212323494394650219020"TRAFODION"."LOB130"
' , 'hdfs:///lobs/tlob130_anoush2.jpg');
Success. Targetfile :hdfs:///lobs/tlob130_anoush2.jpg Length : 230150
--- SQL operation complete.
@@ -849,10 +849,90 @@ Hey diddle diddle,
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_anoush3.jpg',create,truncate);/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH00000200030520117997292586250119884121437232179272218212317486279303189020"TRAFODION"."LOB130" ' , 'tlob130_anoush3.jpg',create,truncate);
+>>extract lobtofile(LOB 'LOBH00000200030592176192134736435719592176192203041777118212323494430688273020"TRAFODION"."LOB130"
' , 'tlob130_anoush3.jpg',create,truncate);
Success. Targetfile :tlob130_anoush3.jpg Length : 230150
--- SQL operation complete.
>>
+>>obey TEST130(lob_gc_cleanup);
+>>log;
+>>create table tlob130gc (c1 int not null, c2 blob, c3 blob, primary key (c1));
+
+--- SQL operation complete.
+>>--insert a few rows
+>>insert into tlob130gc values (1, stringtolob('aaaa'), stringtolob('bbbbb'));
+
+--- 1 row(s) inserted.
+>>insert into tlob130gc values (2, stringtolob('aaaa'), stringtolob('bbbbb'));
+
+--- 1 row(s) inserted.
+>>insert into tlob130gc values (3, stringtolob('aaaa'), stringtolob('bbbbb'));
+
+--- 1 row(s) inserted.
+>>insert into tlob130gc values (4, stringtolob('aaaa'), stringtolob('bbbbb'));
+
+--- 1 row(s) inserted.
+>>insert into tlob130gc values (5, stringtolob('aaaa'), stringtolob('bbbbb'));
+
+--- 1 row(s) inserted.
+>>insert into tlob130gc values (6, stringtolob('aaaa'), stringtolob('bbbbb'));
+
+--- 1 row(s) inserted.
+>>insert into tlob130gc values (7, stringtolob('aaaa'), stringtolob('bbbbb'));
+
+--- 1 row(s) inserted.
+>>insert into tlob130gc values (8, stringtolob('aaaa'), stringtolob('bbbbb'));
+
+--- 1 row(s) inserted.
+>>--create holes in the lob data file
+>>delete from tlob130gc where c1=2;
+
+--- 1 row(s) deleted.
+>>delete from tlob130gc where c1=6;
+
+--- 1 row(s) deleted.
+>>update tlob130gc set c2=stringtolob('xxxx')where c1=7;
+
+--- 1 row(s) updated.
+>>--check contents of table
+>>select c1,lobtostring(c2,10),lobtostring(c3,10) from tlob130gc;
+
+C1 (EXPR) (EXPR)
+----------- ---------- ----------
+
+ 1 aaaa bbbbb
+ 3 aaaa bbbbb
+ 4 aaaa bbbbb
+ 5 aaaa bbbbb
+ 7 xxxx bbbbb
+ 8 aaaa bbbbb
+
+--- 6 row(s) selected.
+>>cqd LOB_GC_LIMIT_SIZE '0';
+
+--- SQL operation complete.
+>>--this next insert should trigger a GC
+>>insert into tlob130gc values(9,stringtolob('aaaa'), stringtolob('bbbbb'));
+
+--- 1 row(s) inserted.
+>>--check contents of table. Should have one extra row compared to above select
+>>select c1,lobtostring(c2,10),lobtostring(c3,10) from tlob130gc;
+
+C1 (EXPR) (EXPR)
+----------- ---------- ----------
+
+ 1 aaaa bbbbb
+ 3 aaaa bbbbb
+ 4 aaaa bbbbb
+ 5 aaaa bbbbb
+ 7 xxxx bbbbb
+ 8 aaaa bbbbb
+ 9 aaaa bbbbb
+
+--- 7 row(s) selected.
+>>cqd LOB_GC_LIMIT_SIZE reset;
+
+--- SQL operation complete.
+>>
>>obey TEST130(lob_general_cleanup);
>>log;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/95035198/core/sql/regress/executor/TEST130
----------------------------------------------------------------------
diff --git a/core/sql/regress/executor/TEST130 b/core/sql/regress/executor/TEST130
index 87ca365..108da27 100755
--- a/core/sql/regress/executor/TEST130
+++ b/core/sql/regress/executor/TEST130
@@ -404,6 +404,37 @@ sh rm t130_extract_command;
sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_anoush3.jpg',create,truncate);/g" >> t130_extract_command;
obey t130_extract_command;
+obey TEST130(lob_gc_cleanup);
+?section lob_gc_cleanup
+log;
+drop table tlob130gc;
+obey TEST130(lob_gc_test);
+
+?section lob_gc_test
+log LOG130;
+create table tlob130gc (c1 int not null, c2 blob, c3 blob, primary key (c1));
+--insert a few rows
+insert into tlob130gc values (1, stringtolob('aaaa'), stringtolob('bbbbb'));
+insert into tlob130gc values (2, stringtolob('aaaa'), stringtolob('bbbbb'));
+insert into tlob130gc values (3, stringtolob('aaaa'), stringtolob('bbbbb'));
+insert into tlob130gc values (4, stringtolob('aaaa'), stringtolob('bbbbb'));
+insert into tlob130gc values (5, stringtolob('aaaa'), stringtolob('bbbbb'));
+insert into tlob130gc values (6, stringtolob('aaaa'), stringtolob('bbbbb'));
+insert into tlob130gc values (7, stringtolob('aaaa'), stringtolob('bbbbb'));
+insert into tlob130gc values (8, stringtolob('aaaa'), stringtolob('bbbbb'));
+--create holes in the lob data file
+delete from tlob130gc where c1=2;
+delete from tlob130gc where c1=6;
+update tlob130gc set c2=stringtolob('xxxx')where c1=7;
+--check contents of table
+select c1,lobtostring(c2,10),lobtostring(c3,10) from tlob130gc;
+cqd LOB_GC_LIMIT_SIZE '0';
+--this next insert should trigger a GC
+insert into tlob130gc values(9,stringtolob('aaaa'), stringtolob('bbbbb'));
+--check contents of table. Should have one extra row compared to above select
+select c1,lobtostring(c2,10),lobtostring(c3,10) from tlob130gc;
+cqd LOB_GC_LIMIT_SIZE reset;
+
obey TEST130(lob_general_cleanup);
-- cleanup
?section lob_general_cleanup
@@ -429,6 +460,7 @@ sh rm TMP130;
sh rm tlob130txt2;
sh rm lobc2out.jpg;
sh rm TEST130_argfile;
+drop table tlob130gc;
drop schema trafodion.lob130 cascade;
[06/11] incubator-trafodion git commit: Merge remote branch
'origin/master' into lob_gc_work
Posted by sa...@apache.org.
Merge remote branch 'origin/master' into lob_gc_work
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/70e7a09a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/70e7a09a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/70e7a09a
Branch: refs/heads/master
Commit: 70e7a09ae1319ccbd7e07eeb5aff3388ae35985c
Parents: c29103f fabfc80
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Mon Feb 29 06:20:37 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Mon Feb 29 06:20:37 2016 +0000
----------------------------------------------------------------------
.gitattributes | 5 -
.gitignore | 3 +
.rat-excludes | 13 +-
DISCLAIMER | 12 +
DISCLAIMER.txt | 12 -
LICENSE.txt | 327 -
Makefile | 13 +-
NOTICE | 9 +
NOTICE.txt | 9 -
RAT_README | 192 +
RAT_README.txt | 181 -
README | 58 +
README.txt | 58 -
core/bldenvchk.sh | 2 +-
core/conn/Makefile | 12 +-
.../jdbc_type2/native/SQLMXDatabaseMetaData.cpp | 6 +-
core/conn/jdbc_type2/native/SrvrCommon.cpp | 167 +-
core/conn/jdbc_type2/native/SrvrSmd.cpp | 7 -
core/conn/jdbc_type4/.gitignore | 1 +
core/conn/jdbc_type4/build.xml | 9 +-
core/conn/odbc/src/odbc/nsksrvr/Makefile | 2 +-
core/conn/trafci/manualtests/README | 50 +
core/conn/trafci/manualtests/statcheck.sql | 47 +
.../src/org/trafodion/ci/DatabaseQuery.java | 3 +-
.../odbc/odbcclient/unixcli/cli/drvrglobal.h | 21 +-
.../odbc/odbcclient/unixcli/makefile.lnx | 2 +-
.../odbc/odbcclient/unixcli/package/LICENSE | 1 -
.../odbc/odbcclient/unixcli/package/license.txt | 18 -
.../unixcli/package/linux64_install.sh | 397 -
.../odbc/odbcclient/unixcli/package/mklnxpkg.sh | 6 +-
.../unixcli/package/trafodbclnx_install.sh | 22 +-
core/dbsecurity/auth/src/dbUserAuth.cpp | 1 +
core/rest/.gitignore | 2 +
core/rest/Makefile | 13 +-
core/rest/pom.xml | 63 +-
core/rest/src/assembly/all.xml | 8 +
core/rest/src/main/asciidoc/_chapters/apis.adoc | 40 +-
.../org/trafodion/rest/GetStatusResponse.java | 2 +-
.../java/org/trafodion/rest/RESTServlet.java | 9 +-
.../java/org/trafodion/rest/ServerResource.java | 151 +-
.../org/trafodion/rest/VersionResource.java | 25 +-
.../rest/provider/JAXBContextResolver.java | 1 -
.../producer/PlainTextMessageBodyProducer.java | 3 -
.../java/org/trafodion/rest/util/ConfTool.java | 2 +-
.../org/trafodion/rest/zookeeper/ZkClient.java | 1 -
core/sqf/.gitignore | 3 +
core/sqf/LocalSettingsTemplate.sh | 7 +-
core/sqf/Makefile | 18 +-
core/sqf/export/lib/Parse/RecDescent.pm | 6009 ---
core/sqf/monitor/linux/amqpmon.cxx | 231 -
core/sqf/monitor/linux/amqpmon.h | 71 -
core/sqf/monitor/linux/makefile | 2 +-
core/sqf/sqenvcom.sh | 15 +-
core/sqf/sql/scripts/cleanlogs | 92 +
core/sqf/sql/scripts/dcscheck | 77 +-
core/sqf/sql/scripts/dcsstart | 4 +-
core/sqf/sql/scripts/dcsstop | 4 +
core/sqf/sql/scripts/findPort.py | 40 +-
core/sqf/sql/scripts/genms | 3 +-
.../sql/scripts/install_hadoop_regr_test_env | 22 +-
core/sqf/sql/scripts/install_local_hadoop | 2 +-
core/sqf/sql/scripts/install_traf_components | 68 +-
core/sqf/sql/scripts/lsof.awk | 110 +
core/sqf/sql/scripts/restcheck | 70 +
core/sqf/sql/scripts/sqcheck | 58 +-
core/sqf/sql/scripts/sqgen | 19 +-
.../sqf/sql/scripts/traf_coprocessor.properties | 26 +
core/sqf/sql/scripts/uninstall_traf_components | 159 +
core/sqf/sqvers | 2 +-
.../transactional/TransactionManager.java | 59 +-
.../TransactionRegionLocation.java | 22 +-
.../client/transactional/TransactionState.java | 13 +-
.../transactional/TrxRegionEndpoint.java | 552 +-
.../transactional/TrxRegionObserver.java | 72 +-
.../generated/TrxRegionProtos.java | 2182 +-
.../hbase-trx/src/main/protobuf/TrxRegion.proto | 17 +
core/sqf/src/seatrans/tm/hbasetmlib2/Makefile | 4 +-
.../java/org/trafodion/dtm/TmAuditTlog.java | 350 +-
core/sqf/src/tm/Makefile | 8 +-
core/sqf/src/tm/dtm.events.pb.cc | 1185 -
core/sqf/src/tm/dtm.events.pb.h | 709 -
core/sqf/src/tm/dtm.perf_stats.pb.cc | 4081 --
core/sqf/src/tm/dtm.perf_stats.pb.h | 2442 --
core/sqf/src/tm/idtmsrv.cpp | 48 +-
core/sql/SqlCompilerDebugger/mk.sh | 3 +
core/sql/bin/SqlciErrors.txt | 6 +-
core/sql/cli/CliExpExchange.cpp | 2 +
core/sql/comexe/ComTdbHbaseAccess.cpp | 16 +-
core/sql/comexe/ComTdbHbaseAccess.h | 10 +-
core/sql/common/ComSmallDefs.h | 15 +-
core/sql/common/ComUser.h | 6 +-
core/sql/common/DateTimeType.cpp | 69 +-
core/sql/common/DatetimeType.h | 8 +-
core/sql/common/NAUserId.h | 24 +-
core/sql/common/csconvert.cpp | 48 +
core/sql/common/csconvert.h | 4 +
core/sql/common/str.cpp | 152 +
core/sql/common/str.h | 3 +
core/sql/executor/ExExeUtilGet.cpp | 32 +-
core/sql/executor/ExHbaseAccess.cpp | 75 +-
core/sql/executor/ExHbaseAccess.h | 4 +-
core/sql/executor/ExHbaseIUD.cpp | 86 +-
core/sql/executor/ExHbaseSelect.cpp | 4 +
core/sql/executor/ExSequence.cpp | 2 +
core/sql/executor/ExStats.cpp | 24 +-
core/sql/executor/HBaseClient_JNI.cpp | 7 +-
core/sql/executor/HBaseClient_JNI.h | 2 +-
core/sql/executor/MdamPoint.cpp | 152 +-
core/sql/executor/MdamPoint.h | 5 +-
core/sql/executor/hiveHook.cpp | 2 +-
core/sql/exp/ExpAtp.cpp | 71 +
core/sql/exp/ExpAtp.h | 9 +
core/sql/exp/ExpCriDesc.cpp | 18 +
core/sql/exp/ExpCriDesc.h | 3 +
core/sql/exp/ExpHbaseInterface.cpp | 49 +-
core/sql/exp/ExpHbaseInterface.h | 6 +-
core/sql/exp/ExpSqlTupp.cpp | 12 +
core/sql/exp/ExpSqlTupp.h | 4 +
core/sql/exp/exp_clause_derived.h | 12 +-
core/sql/exp/exp_conv.cpp | 81 +-
core/sql/exp/exp_datetime.cpp | 474 +-
core/sql/exp/exp_datetime.h | 43 +-
core/sql/exp/exp_function.cpp | 16 +-
core/sql/exp/exp_tuple_desc.cpp | 22 +
core/sql/exp/exp_tuple_desc.h | 3 +
core/sql/generator/GenExplain.cpp | 266 +
core/sql/generator/GenItemFunc.cpp | 3 +
core/sql/generator/GenPreCode.cpp | 481 +-
core/sql/generator/GenRelScan.cpp | 36 +-
core/sql/generator/Generator.cpp | 12 +
core/sql/generator/Generator.h | 4 +-
core/sql/nskgmake/Makerules.linux | 5 +-
core/sql/nskgmake/SqlCompilerDebugger/Makefile | 15 +-
core/sql/optimizer/BindItemExpr.cpp | 36 +-
core/sql/optimizer/BindRelExpr.cpp | 17 +-
core/sql/optimizer/BindWA.cpp | 31 +-
core/sql/optimizer/BindWA.h | 5 +-
core/sql/optimizer/ItemExpr.cpp | 9 +-
core/sql/optimizer/ItemFunc.h | 1 +
core/sql/optimizer/NATable.cpp | 129 +-
core/sql/optimizer/NodeMap.cpp | 15 +-
core/sql/optimizer/NormRelExpr.cpp | 85 +-
core/sql/optimizer/RelExeUtil.cpp | 9 +-
core/sql/optimizer/RelExeUtil.h | 15 +-
core/sql/optimizer/RelExpr.cpp | 1 +
core/sql/optimizer/RelJoin.h | 10 +
core/sql/optimizer/RelScan.h | 10 +
core/sql/optimizer/SynthType.cpp | 7 +
core/sql/optimizer/ValueDesc.cpp | 32 +-
core/sql/optimizer/ValueDesc.h | 5 +
core/sql/parser/SqlParserAux.cpp | 22 +-
core/sql/parser/SqlParserAux.h | 1 +
core/sql/parser/StmtDDLCreateRoutine.h | 4 +
core/sql/parser/StmtDDLDropRoutine.h | 5 +
core/sql/parser/sqlparser.y | 108 +-
core/sql/parser/ulexer.h | 21 +
core/sql/regress/catman1/DIFF133.KNOWN | 25 -
core/sql/regress/catman1/DIFF136.KNOWN | 11 -
core/sql/regress/catman1/DIFF139.KNOWN | 6 -
core/sql/regress/catman1/EXPECTED129 | 1237 -
core/sql/regress/catman1/EXPECTED131 | 491 -
core/sql/regress/catman1/EXPECTED132 | 2214 -
core/sql/regress/catman1/EXPECTED133 | 453 -
core/sql/regress/catman1/EXPECTED135 | 1354 -
core/sql/regress/catman1/EXPECTED136 | 870 -
core/sql/regress/catman1/EXPECTED137 | 2501 --
core/sql/regress/catman1/EXPECTED138 | 1692 -
core/sql/regress/catman1/EXPECTED139 | 585 -
core/sql/regress/catman1/EXPECTED140 | 926 -
core/sql/regress/catman1/EXPECTED141 | Bin 78469 -> 0 bytes
core/sql/regress/catman1/FILTER132 | 40 -
core/sql/regress/catman1/TEST129 | 442 -
core/sql/regress/catman1/TEST131 | 365 -
core/sql/regress/catman1/TEST132 | 447 -
core/sql/regress/catman1/TEST133 | 236 -
core/sql/regress/catman1/TEST135 | 465 -
core/sql/regress/catman1/TEST136 | 400 -
core/sql/regress/catman1/TEST137 | 608 -
core/sql/regress/catman1/TEST138 | 347 -
core/sql/regress/catman1/TEST139 | 325 -
core/sql/regress/catman1/TEST140 | 433 -
core/sql/regress/catman1/TEST141 | 394 -
core/sql/regress/catman1/cidefs | 116 -
core/sql/regress/catman1/etest140.cpp | 98 -
core/sql/regress/catman1/runregr | 31 -
core/sql/regress/catman1/udrtest135.cpp | 55 -
core/sql/regress/core/EXPECTED005.SB | 2 +-
core/sql/regress/core/EXPECTED116 | 301 +-
core/sql/regress/core/EXPECTED131 | 491 +
core/sql/regress/core/EXPECTEDRTS | 1393 +-
core/sql/regress/core/FILTER116 | 33 +
core/sql/regress/core/TEST000 | 14 +-
core/sql/regress/core/TEST116 | 55 +-
core/sql/regress/core/TEST131 | 365 +
core/sql/regress/executor/DIFF013.KNOWN | 2 +-
core/sql/regress/executor/EXPECTED013.SB | 29 +-
core/sql/regress/executor/EXPECTED015.SB | 222 +
core/sql/regress/executor/EXPECTED022.LINUX | 8881 ----
core/sql/regress/executor/EXPECTED022.SB | 9467 +++++
core/sql/regress/executor/EXPECTED131 | 472 +
core/sql/regress/executor/EXPECTED140 | 927 +
core/sql/regress/executor/FILTER131 | 31 +
core/sql/regress/executor/FILTER140 | 42 +
core/sql/regress/executor/TEST015 | 123 +
core/sql/regress/executor/TEST022 | 50 +
core/sql/regress/executor/TEST131 | 101 +
core/sql/regress/executor/TEST140 | 96 +
core/sql/regress/fullstack2/EXPECTED062 | 41 +-
core/sql/regress/fullstack2/EXPECTED062.RELEASE | 33 +-
core/sql/regress/hive/EXPECTED005 | 26 +
core/sql/regress/hive/EXPECTED009 | 28 +-
core/sql/regress/hive/TEST005 | 7 +
core/sql/regress/hive/TEST005_a.hive.sql | 9 +
core/sql/regress/hive/TEST009 | 14 +-
core/sql/regress/hive/tbl_gbk.data | Bin 0 -> 129 bytes
core/sql/regress/privs1/DIFF133.KNOWN | 25 +
core/sql/regress/privs1/DIFF136.KNOWN | 11 +
core/sql/regress/privs1/EXPECTED132 | 2233 +
core/sql/regress/privs1/EXPECTED133 | 459 +
core/sql/regress/privs1/EXPECTED136 | 870 +
core/sql/regress/privs1/EXPECTED137 | 1925 +
core/sql/regress/privs1/EXPECTED141 | Bin 0 -> 113222 bytes
core/sql/regress/privs1/FILTER132 | 40 +
core/sql/regress/privs1/TEST132 | 409 +
core/sql/regress/privs1/TEST133 | 236 +
core/sql/regress/privs1/TEST136 | 400 +
core/sql/regress/privs1/TEST137 | 608 +
core/sql/regress/privs1/TEST141 | 459 +
core/sql/regress/privs1/etest132.cpp | 55 +
core/sql/regress/privs1/etest141.cpp | 98 +
core/sql/regress/privs1/runregr | 31 +
core/sql/regress/privs2/DIFF139.KNOWN | 6 +
core/sql/regress/privs2/EXPECTED129 | 1237 +
core/sql/regress/privs2/EXPECTED135 | 1434 +
core/sql/regress/privs2/EXPECTED138 | 1772 +
core/sql/regress/privs2/EXPECTED139 | 585 +
core/sql/regress/privs2/EXPECTED140 | 970 +
core/sql/regress/privs2/FILTER140 | 33 +
core/sql/regress/privs2/TEST129 | 442 +
core/sql/regress/privs2/TEST135 | 493 +
core/sql/regress/privs2/TEST138 | 347 +
core/sql/regress/privs2/TEST139 | 325 +
core/sql/regress/privs2/TEST140 | 434 +
core/sql/regress/privs2/etest140.cpp | 98 +
core/sql/regress/privs2/runregr | 31 +
core/sql/regress/privs2/udrtest135.cpp | 55 +
core/sql/regress/seabase/EXPECTED002 | 146 +-
core/sql/regress/seabase/EXPECTED010 | 610 +-
core/sql/regress/seabase/EXPECTED011 | 24 +-
core/sql/regress/seabase/EXPECTED012 | 53 +-
core/sql/regress/seabase/EXPECTED013 | 40 +-
core/sql/regress/seabase/EXPECTED016 | 168 +-
core/sql/regress/seabase/EXPECTED026 | 34 +-
core/sql/regress/seabase/FILTER002 | 1 +
core/sql/regress/seabase/TEST012 | 18 +
core/sql/regress/seabase/TEST013 | 20 +-
core/sql/regress/tools/runallsb | 11 +-
core/sql/regress/tools/runregr | 227 +-
core/sql/regress/tools/runregr_core.ksh | 6 +-
core/sql/regress/tools/runregr_executor.ksh | 2 +-
core/sql/regress/tools/runregr_privs1.ksh | 489 +
core/sql/regress/tools/runregr_privs2.ksh | 489 +
core/sql/regress/tools/sbdefs | 3 +-
core/sql/regress/tools/setupenv | 52 +-
core/sql/regress/tools/setuplnxenv | 25 +-
core/sql/sqlcat/desc.h | 1 -
core/sql/sqlci/Param.cpp | 8 +-
core/sql/sqlci/SqlCmd.cpp | 37 +-
core/sql/sqlcomp/CmpDDLCatErrorCodes.h | 4 +-
core/sql/sqlcomp/CmpDescribe.cpp | 13 +-
core/sql/sqlcomp/CmpSeabaseDDL.h | 12 +-
core/sql/sqlcomp/CmpSeabaseDDLauth.cpp | 27 +-
core/sql/sqlcomp/CmpSeabaseDDLauth.h | 1 +
core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 33 +-
core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 65 +-
core/sql/sqlcomp/CmpSeabaseDDLindex.cpp | 6 +-
core/sql/sqlcomp/CmpSeabaseDDLmd.h | 11 +
core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp | 10 +-
core/sql/sqlcomp/CmpSeabaseDDLschema.cpp | 532 +-
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 66 +-
core/sql/sqlcomp/DefaultConstants.h | 15 +-
core/sql/sqlcomp/NADefaults.h | 2 +
core/sql/sqlcomp/PrivMgr.cpp | 12 +-
core/sql/sqlcomp/PrivMgr.h | 9 +-
core/sql/sqlcomp/PrivMgrCommands.cpp | 4 +-
core/sql/sqlcomp/PrivMgrCommands.h | 2 +-
core/sql/sqlcomp/PrivMgrComponentOperations.cpp | 2 +-
core/sql/sqlcomp/PrivMgrComponentPrivileges.cpp | 377 +-
core/sql/sqlcomp/PrivMgrDefs.h | 9 -
core/sql/sqlcomp/PrivMgrDesc.h | 51 +-
core/sql/sqlcomp/PrivMgrMD.cpp | 210 +-
core/sql/sqlcomp/PrivMgrMD.h | 15 +-
core/sql/sqlcomp/PrivMgrMDDefs.h | 4 +-
core/sql/sqlcomp/PrivMgrObjects.cpp | 52 +
core/sql/sqlcomp/PrivMgrObjects.h | 4 +
core/sql/sqlcomp/PrivMgrPrivileges.cpp | 711 +-
core/sql/sqlcomp/PrivMgrPrivileges.h | 39 +-
core/sql/sqlcomp/PrivMgrRoles.cpp | 46 +-
core/sql/sqlcomp/QCache.cpp | 30 +-
core/sql/sqlcomp/nadefaults.cpp | 168 +-
core/sql/sqlmxevents/logmxevent_sq.cpp | 2 +-
.../org/trafodion/sql/CoprocessorUtils.java | 104 +
.../java/org/trafodion/sql/HBaseClient.java | 19 +-
.../java/org/trafodion/sql/HTableClient.java | 574 +-
core/sql/udrserv/udrserv.cpp | 3 +-
core/sql/ustat/hs_cli.cpp | 169 +-
core/sql/ustat/hs_const.h | 4 +
core/sql/ustat/hs_globals.cpp | 93 +
core/sql/ustat/hs_lex.ll | 4 +
core/sql/ustat/hs_parser.cpp | 17 +-
core/sql/ustat/hs_update.cpp | 12 +-
core/sql/ustat/hs_yacc.y | 9 +-
dcs/.gitignore | 2 +
dcs/Makefile | 14 +-
dcs/bin/dcs-config.sh | 10 +
dcs/bin/scripts/dcsbind.sh | 137 +-
dcs/bin/scripts/dcsunbind.sh | 99 +
dcs/bin/scripts/parse_dcs_site.py | 46 +
dcs/bin/start-dcs.sh | 13 +-
dcs/bin/stop-dcs.sh | 7 +-
dcs/conf/dcs-env.sh | 8 +-
dcs/conf/master | 0
dcs/pom.xml | 80 +-
dcs/src/assembly/all.xml | 4 +-
.../main/asciidoc/_chapters/architecture.adoc | 15 +-
.../main/asciidoc/_chapters/configuration.adoc | 78 +-
.../main/asciidoc/_chapters/external_apis.adoc | 34 -
dcs/src/main/asciidoc/_chapters/ops_mgt.adoc | 30 +-
.../asciidoc/_chapters/troubleshooting.adoc | 4 +-
dcs/src/main/asciidoc/index.adoc | 1 -
.../dcs/master/listener/ConnectReply.java | 60 +-
.../org/trafodion/dcs/server/ServerManager.java | 15 +-
.../Descriptor2List.java | 2 +-
.../serverHandler/ServerApiSqlEndTransact.java | 14 +-
.../serverHandler/ServerApiSqlExecDirect.java | 2 +-
.../serverHandler/ServerApiSqlExecute.java | 1 +
.../serverHandler/ServerApiSqlFetch.java | 1 -
.../dcs/servermt/serverSql/TrafConnection.java | 3 +-
.../dcs/servermt/serverSql/TrafStatement.java | 5 +-
dcs/src/main/resources/dcs-default.xml | 45 -
.../java/org/trafodion/jdbc_test/BatchTest.java | 582 +
.../org/trafodion/jdbc_test/JdbcCommon.java | 224 +
docs/.gitignore | 1 +
.../ODBC_Sample_Programs/basicsql.cpp | 320 -
.../ODBC_Sample_Programs/build.bat | 25 -
.../client_install/ODBC_Sample_Programs/run.bat | 23 -
docs/client_install/pom.xml | 289 +
.../Trafodion_Client_Installation_Guide.xml | 3251 --
.../source/img/Add_Alias_SQuirreL.png | Bin 9719 -> 0 bytes
.../source/img/Add_Driver_SQuirreL.png | Bin 42544 -> 0 bytes
.../img/Database_Connection_in_DbVisualizer.png | Bin 55768 -> 0 bytes
.../source/img/DbVisualizer_Driver_Manager.png | Bin 102328 -> 0 bytes
.../source/img/Extracted_Files.png | Bin 15061 -> 0 bytes
.../source/img/InstallComplete.png | Bin 19467 -> 0 bytes
.../source/img/InstallerWizardPaths.png | Bin 25831 -> 0 bytes
.../source/img/InstallerWizardWelcome.png | Bin 24394 -> 0 bytes
.../client_install/source/img/JDBC_JAR_Path.png | Bin 4552 -> 0 bytes
.../source/img/OptionalComponents.png | Bin 31715 -> 0 bytes
.../source/img/PerlJavaServerURL.png | Bin 3510 -> 0 bytes
.../source/img/Physical_Connection.png | Bin 147371 -> 0 bytes
.../img/Properties_for_Alias_SQuirreL.png | Bin 95544 -> 0 bytes
.../client_install/source/img/ProxySettings.png | Bin 2828 -> 0 bytes
.../source/img/TQW_control_options.png | Bin 12363 -> 0 bytes
docs/client_install/source/img/TQW_login.png | Bin 126619 -> 0 bytes
docs/client_install/source/img/TQW_logout.png | Bin 3049 -> 0 bytes
.../source/img/TQW_query_plan.png | Bin 83528 -> 0 bytes
docs/client_install/source/img/TQW_session.png | Bin 1983 -> 0 bytes
.../source/img/TQW_whiteboard.png | Bin 70729 -> 0 bytes
docs/client_install/source/img/path2.png | Bin 32134 -> 0 bytes
docs/client_install/source/img/varval2.png | Bin 13296 -> 0 bytes
docs/client_install/source/xml-profile.xml | 41 -
.../src/asciidoc/_chapters/SQuirrel.adoc | 70 +
.../src/asciidoc/_chapters/about.adoc | 176 +
.../src/asciidoc/_chapters/dbviz.adoc | 84 +
.../src/asciidoc/_chapters/introduction.adoc | 49 +
.../src/asciidoc/_chapters/jdbct4.adoc | 325 +
.../src/asciidoc/_chapters/odb.adoc | 99 +
.../src/asciidoc/_chapters/odbc_linux.adoc | 310 +
.../src/asciidoc/_chapters/odbc_windows.adoc | 239 +
.../src/asciidoc/_chapters/sample_prog.adoc | 75 +
.../src/asciidoc/_chapters/trafci.adoc | 473 +
docs/client_install/src/asciidoc/index.adoc | 79 +
.../src/images/Add_Alias_SQuirreL.jpg | Bin 0 -> 48082 bytes
.../src/images/Add_Driver_SQuirreL.jpg | Bin 0 -> 58950 bytes
.../Database_Connection_in_DbVisualizer.jpg | Bin 0 -> 58043 bytes
.../src/images/DbVisualizer_Driver_Manager.jpg | Bin 0 -> 80198 bytes
.../src/images/Extracted_Files.jpg | Bin 0 -> 28327 bytes
.../src/images/InstallComplete.jpg | Bin 0 -> 47365 bytes
.../src/images/InstallerWizardPaths.jpg | Bin 0 -> 77163 bytes
.../src/images/InstallerWizardWelcome.jpg | Bin 0 -> 53537 bytes
.../client_install/src/images/JDBC_JAR_Path.jpg | Bin 0 -> 18215 bytes
.../src/images/OptionalComponents.jpg | Bin 0 -> 71744 bytes
.../src/images/PerlJavaServerURL.jpg | Bin 0 -> 27662 bytes
.../src/images/Physical_Connection.jpg | Bin 0 -> 185241 bytes
.../images/Properties_for_Alias_SQuirreL.jpg | Bin 0 -> 84197 bytes
.../client_install/src/images/ProxySettings.jpg | Bin 0 -> 20985 bytes
.../src/images/TQW_control_options.jpg | Bin 0 -> 14642 bytes
docs/client_install/src/images/TQW_login.jpg | Bin 0 -> 51058 bytes
docs/client_install/src/images/TQW_logout.jpg | Bin 0 -> 7602 bytes
.../src/images/TQW_query_plan.jpg | Bin 0 -> 45607 bytes
docs/client_install/src/images/TQW_session.jpg | Bin 0 -> 5008 bytes
.../src/images/TQW_whiteboard.jpg | Bin 0 -> 43936 bytes
docs/client_install/src/images/path2.jpg | Bin 0 -> 57338 bytes
docs/client_install/src/images/varval2.jpg | Bin 0 -> 19505 bytes
.../src/resources/source/basicsql.cpp | 394 +
.../src/resources/source/build.bat | 25 +
.../client_install/src/resources/source/run.bat | 23 +
docs/command_interface/pom.xml | 289 +
.../Trafodion_Command_Interface_Guide.xml | 9846 -----
docs/command_interface/source/img/VST101.png | Bin 52396 -> 0 bytes
docs/command_interface/source/img/VST401.png | Bin 39217 -> 0 bytes
docs/command_interface/source/img/VST601.png | Bin 2355 -> 0 bytes
docs/command_interface/source/img/dpnvslf.png | Bin 36808 -> 0 bytes
docs/command_interface/source/img/envvar.png | Bin 26344 -> 0 bytes
docs/command_interface/source/img/instlcm1.png | Bin 5237 -> 0 bytes
.../command_interface/source/img/instlcm2.3.png | Bin 36015 -> 0 bytes
docs/command_interface/source/img/instlcm2.png | Bin 6316 -> 0 bytes
docs/command_interface/source/img/iwiz1.png | Bin 6701 -> 0 bytes
docs/command_interface/source/img/iwiz2.3.png | Bin 7705 -> 0 bytes
docs/command_interface/source/img/iwiz2.3b.png | Bin 7987 -> 0 bytes
docs/command_interface/source/img/iwiz2.3c.png | Bin 35606 -> 0 bytes
.../source/img/iwiz2.3comp.png | Bin 10358 -> 0 bytes
docs/command_interface/source/img/iwiz2.3d.png | Bin 4998 -> 0 bytes
docs/command_interface/source/img/iwiz2.3e.png | Bin 5714 -> 0 bytes
docs/command_interface/source/img/iwiz2.3f.png | Bin 12093 -> 0 bytes
docs/command_interface/source/img/iwiz2.3g.png | Bin 8425 -> 0 bytes
docs/command_interface/source/img/iwiz2.3h.png | Bin 8425 -> 0 bytes
docs/command_interface/source/img/iwiz2.3n.png | Bin 11522 -> 0 bytes
docs/command_interface/source/img/iwiz2.png | Bin 1360 -> 0 bytes
docs/command_interface/source/img/iwiz3.png | Bin 4777 -> 0 bytes
docs/command_interface/source/img/iwiz4.png | Bin 4753 -> 0 bytes
docs/command_interface/source/img/iwiz5.png | Bin 150880 -> 0 bytes
docs/command_interface/source/img/iwiz7.png | Bin 6061 -> 0 bytes
docs/command_interface/source/img/iwiz8.png | Bin 7022 -> 0 bytes
docs/command_interface/source/img/iwiz9.png | Bin 3433 -> 0 bytes
.../source/img/iwizediturl.png | Bin 3510 -> 0 bytes
.../command_interface/source/img/iwizone2.3.png | Bin 87777 -> 0 bytes
.../source/img/iwizopensource.png | Bin 10283 -> 0 bytes
.../source/img/iwizproxyset.png | Bin 2828 -> 0 bytes
docs/command_interface/source/img/java-opt.png | Bin 127106 -> 0 bytes
docs/command_interface/source/img/launch02.png | Bin 18206 -> 0 bytes
docs/command_interface/source/img/launchs1.png | Bin 3068 -> 0 bytes
docs/command_interface/source/img/log-file.png | Bin 8302 -> 0 bytes
docs/command_interface/source/img/logvar.png | Bin 39647 -> 0 bytes
.../command_interface/source/img/logvar_new.png | Bin 13368 -> 0 bytes
docs/command_interface/source/img/mycomp.png | Bin 13033 -> 0 bytes
.../source/img/new_iwiz2.3.png | Bin 7407 -> 0 bytes
.../source/img/new_iwiz2.3f.png | Bin 8632 -> 0 bytes
docs/command_interface/source/img/p-errlog.png | Bin 7387 -> 0 bytes
docs/command_interface/source/img/p-scrlog.png | Bin 4205 -> 0 bytes
docs/command_interface/source/img/path.png | Bin 5592 -> 0 bytes
docs/command_interface/source/img/path2.png | Bin 32134 -> 0 bytes
docs/command_interface/source/img/script.png | Bin 7142 -> 0 bytes
docs/command_interface/source/img/shortct1.png | Bin 27091 -> 0 bytes
docs/command_interface/source/img/shortct2.png | Bin 33784 -> 0 bytes
docs/command_interface/source/img/shortct3.png | Bin 30072 -> 0 bytes
docs/command_interface/source/img/shortct4.png | Bin 17710 -> 0 bytes
docs/command_interface/source/img/shortct5.png | Bin 30622 -> 0 bytes
docs/command_interface/source/img/sysprop.png | Bin 27422 -> 0 bytes
docs/command_interface/source/img/varval.png | Bin 10013 -> 0 bytes
docs/command_interface/source/img/varval2.png | Bin 13296 -> 0 bytes
docs/command_interface/source/img/winbin.png | Bin 53558 -> 0 bytes
docs/command_interface/source/img/zip2.3.png | Bin 15342 -> 0 bytes
docs/command_interface/source/img/zip2.3a.png | Bin 14073 -> 0 bytes
docs/command_interface/source/xml-profile.xml | 41 -
.../src/asciidoc/_chapters/about.adoc | 177 +
.../src/asciidoc/_chapters/commands.adoc | 4315 ++
.../src/asciidoc/_chapters/install.adoc | 99 +
.../src/asciidoc/_chapters/interactive.adoc | 941 +
.../src/asciidoc/_chapters/introduction.adoc | 34 +
.../src/asciidoc/_chapters/launch.adoc | 637 +
.../src/asciidoc/_chapters/perlpython.adoc | 258 +
.../src/asciidoc/_chapters/scripts.adoc | 164 +
docs/command_interface/src/asciidoc/index.adoc | 76 +
docs/command_interface/src/images/VST101.jpg | Bin 0 -> 49552 bytes
docs/command_interface/src/images/VST401.jpg | Bin 0 -> 37869 bytes
docs/command_interface/src/images/VST601.jpg | Bin 0 -> 17388 bytes
docs/command_interface/src/images/dpnvslf.jpg | Bin 0 -> 17215 bytes
docs/command_interface/src/images/envvar.jpg | Bin 0 -> 55944 bytes
docs/command_interface/src/images/instlcm1.jpg | Bin 0 -> 66629 bytes
.../command_interface/src/images/instlcm2.3.jpg | Bin 0 -> 16721 bytes
docs/command_interface/src/images/instlcm2.jpg | Bin 0 -> 84657 bytes
docs/command_interface/src/images/iwiz1.jpg | Bin 0 -> 61823 bytes
docs/command_interface/src/images/iwiz2.3.jpg | Bin 0 -> 73496 bytes
docs/command_interface/src/images/iwiz2.3b.jpg | Bin 0 -> 76467 bytes
docs/command_interface/src/images/iwiz2.3c.jpg | Bin 0 -> 18174 bytes
.../src/images/iwiz2.3comp.jpg | Bin 0 -> 76540 bytes
docs/command_interface/src/images/iwiz2.3d.jpg | Bin 0 -> 42750 bytes
docs/command_interface/src/images/iwiz2.3e.jpg | Bin 0 -> 39508 bytes
docs/command_interface/src/images/iwiz2.3f.jpg | Bin 0 -> 93285 bytes
docs/command_interface/src/images/iwiz2.3h.jpg | Bin 0 -> 67094 bytes
docs/command_interface/src/images/iwiz2.3n.jpg | Bin 0 -> 83274 bytes
docs/command_interface/src/images/iwiz2.jpg | Bin 0 -> 15025 bytes
docs/command_interface/src/images/iwiz3.jpg | Bin 0 -> 36199 bytes
docs/command_interface/src/images/iwiz4.jpg | Bin 0 -> 35220 bytes
docs/command_interface/src/images/iwiz5.jpg | Bin 0 -> 76695 bytes
docs/command_interface/src/images/iwiz7.jpg | Bin 0 -> 51256 bytes
docs/command_interface/src/images/iwiz8.jpg | Bin 0 -> 65627 bytes
docs/command_interface/src/images/iwiz9.jpg | Bin 0 -> 18671 bytes
.../src/images/iwizediturl.jpg | Bin 0 -> 27662 bytes
.../command_interface/src/images/iwizone2.3.jpg | Bin 0 -> 61060 bytes
.../src/images/iwizopensource.jpg | Bin 0 -> 74346 bytes
.../src/images/iwizproxyset.jpg | Bin 0 -> 20985 bytes
docs/command_interface/src/images/java-opt.jpg | Bin 0 -> 62772 bytes
docs/command_interface/src/images/launch02.jpg | Bin 0 -> 29749 bytes
docs/command_interface/src/images/launchs1.jpg | Bin 0 -> 21801 bytes
docs/command_interface/src/images/log-file.jpg | Bin 0 -> 82163 bytes
docs/command_interface/src/images/logvar.jpg | Bin 0 -> 18400 bytes
.../command_interface/src/images/logvar_new.jpg | Bin 0 -> 20451 bytes
docs/command_interface/src/images/mycomp.jpg | Bin 0 -> 23996 bytes
.../src/images/new_iwiz2.3.jpg | Bin 0 -> 79083 bytes
docs/command_interface/src/images/p-errlog.jpg | Bin 0 -> 94633 bytes
docs/command_interface/src/images/p-scrlog.jpg | Bin 0 -> 34198 bytes
docs/command_interface/src/images/path.jpg | Bin 0 -> 50183 bytes
docs/command_interface/src/images/path2.jpg | Bin 0 -> 57338 bytes
docs/command_interface/src/images/script.jpg | Bin 0 -> 66933 bytes
docs/command_interface/src/images/shortct1.jpg | Bin 0 -> 61270 bytes
docs/command_interface/src/images/shortct2.jpg | Bin 0 -> 57414 bytes
docs/command_interface/src/images/shortct3.jpg | Bin 0 -> 39950 bytes
docs/command_interface/src/images/shortct4.jpg | Bin 0 -> 42194 bytes
docs/command_interface/src/images/shortct5.jpg | Bin 0 -> 63636 bytes
docs/command_interface/src/images/sysprop.jpg | Bin 0 -> 63444 bytes
docs/command_interface/src/images/varval.jpg | Bin 0 -> 20590 bytes
docs/command_interface/src/images/varval2.jpg | Bin 0 -> 19505 bytes
docs/command_interface/src/images/winbin.jpg | Bin 0 -> 60550 bytes
docs/command_interface/src/images/zip2.3.jpg | Bin 0 -> 64722 bytes
docs/command_interface/src/images/zip2.3a.jpg | Bin 0 -> 78101 bytes
docs/messages_guide/pom.xml | 288 +
.../src/asciidoc/_chapters/about.adoc | 181 +
.../src/asciidoc/_chapters/binder_msgs.adoc | 2886 ++
.../src/asciidoc/_chapters/compiler_msgs.adoc | 1549 +
.../src/asciidoc/_chapters/ddl_msgs.adoc | 3046 ++
.../src/asciidoc/_chapters/executor_msgs.adoc | 934 +
.../asciidoc/_chapters/file_system_errors.adoc | 529 +
.../src/asciidoc/_chapters/generator_msgs.adoc | 59 +
.../src/asciidoc/_chapters/introduction.adoc | 116 +
.../src/asciidoc/_chapters/optimizer_msgs.adoc | 118 +
.../src/asciidoc/_chapters/parser_msgs.adoc | 3273 ++
.../src/asciidoc/_chapters/sort_msgs.adoc | 504 +
.../src/asciidoc/_chapters/sqlstate.adoc | 2111 +
.../_chapters/udr_language_mgr_msgs.adoc | 112 +
docs/messages_guide/src/asciidoc/index.adoc | 74 +
docs/odb_manual/Trafodion_odb_User_Guide.xml | 181 -
docs/odb_user/pom.xml | 288 +
.../src/asciidoc/_chapters/a_warnings.adoc | 101 +
docs/odb_user/src/asciidoc/_chapters/about.adoc | 183 +
.../src/asciidoc/_chapters/b_develop.adoc | 50 +
.../src/asciidoc/_chapters/compare_tables.adoc | 202 +
.../src/asciidoc/_chapters/concepts.adoc | 941 +
.../src/asciidoc/_chapters/install.adoc | Bin 0 -> 13598 bytes
.../src/asciidoc/_chapters/introduction.adoc | 50 +
docs/odb_user/src/asciidoc/_chapters/load.adoc | 1229 +
.../src/asciidoc/_chapters/query_driver.adoc | 274 +
.../src/asciidoc/_chapters/sql_interpreter.adoc | 363 +
docs/odb_user/src/asciidoc/index.adoc | 76 +
docs/odb_user/src/images/sql_ansi_colors.jpg | Bin 0 -> 20035 bytes
docs/provisioning_guide/pom.xml | 289 +
.../src/asciidoc/_chapters/about.adoc | 197 +
.../src/asciidoc/_chapters/activate.adoc | 147 +
.../src/asciidoc/_chapters/enable_security.adoc | 491 +
.../src/asciidoc/_chapters/introduction.adoc | 466 +
.../src/asciidoc/_chapters/prepare.adoc | 356 +
.../src/asciidoc/_chapters/recipe_install.adoc | 29 +
.../src/asciidoc/_chapters/recipe_upgrade.adoc | 30 +
.../src/asciidoc/_chapters/requirements.adoc | 510 +
.../src/asciidoc/_chapters/script_install.adoc | 511 +
.../src/asciidoc/_chapters/script_remove.adoc | 118 +
.../src/asciidoc/_chapters/script_upgrade.adoc | 442 +
docs/provisioning_guide/src/asciidoc/index.adoc | 74 +
docs/shared/google-analytics-postprocessor.rb | 43 +
docs/shared/trafodion-logo.jpg | Bin 0 -> 28850 bytes
docs/shared/trafodion-manuals.css | 431 +
docs/shared/trafodion-theme.yml | 76 +
docs/sql_reference/pom.xml | 292 +
.../source/Trafodion_SQL_Reference_Manual.xml | 37555 -----------------
docs/sql_reference/source/xml-profile.xml | 42 -
.../src/asciidoc/_chapters/about.adoc | 241 +
.../src/asciidoc/_chapters/cqds.adoc | 349 +
.../src/asciidoc/_chapters/introduction.adoc | 515 +
.../src/asciidoc/_chapters/limits.adoc | 37 +
.../src/asciidoc/_chapters/olap_functions.adoc | 1079 +
.../src/asciidoc/_chapters/reserved_words.adoc | 287 +
.../src/asciidoc/_chapters/runtime_stats.adoc | 1354 +
.../src/asciidoc/_chapters/sql_clauses.adoc | 1432 +
.../sql_functions_and_expressions.adoc | 7885 ++++
.../_chapters/sql_language_elements.adoc | 4088 ++
.../src/asciidoc/_chapters/sql_statements.adoc | 8465 ++++
.../src/asciidoc/_chapters/sql_utilities.adoc | 1028 +
docs/sql_reference/src/asciidoc/index.adoc | 81 +
docs/src/site/markdown/advocate.md | 15 -
docs/src/site/markdown/build-tools-manual.md | 197 -
docs/src/site/markdown/build.md | 80 -
docs/src/site/markdown/code-organization.md | 27 -
docs/src/site/markdown/code.md | 32 -
docs/src/site/markdown/contribute.md | 69 -
docs/src/site/markdown/contributing-redirect.md | 18 +
.../markdown/cplusplus-coding-guidelines.md | 310 -
.../src/site/markdown/create-dev-environment.md | 153 -
docs/src/site/markdown/develop.md | 245 -
docs/src/site/markdown/document.md | 124 -
docs/src/site/markdown/documentation.md | 40 +-
docs/src/site/markdown/download.md | 155 +-
.../site/markdown/enable-secure-trafodion.md | 210 +-
docs/src/site/markdown/index.md | 125 +-
docs/src/site/markdown/install-preparation.md | 138 -
docs/src/site/markdown/install-troubleshoot.md | 21 -
docs/src/site/markdown/install.md | 201 -
docs/src/site/markdown/ldapcheck.md | 28 +-
docs/src/site/markdown/ldapconfigcheck.md | 36 +-
.../src/site/markdown/manage-dev-environment.md | 51 -
docs/src/site/markdown/merge.md | 140 -
docs/src/site/markdown/passwordless-ssh.md | 4 +-
docs/src/site/markdown/port-assignment.md | 134 -
docs/src/site/markdown/quickstart.md | 95 +-
docs/src/site/markdown/release.md | 225 -
.../site/markdown/setup-build-environment.md | 160 -
docs/src/site/markdown/team-redirect.md | 18 +
docs/src/site/markdown/testing.md | 179 -
docs/src/site/markdown/tests.md | 71 -
docs/src/site/markdown/uninstall.md | 176 -
docs/src/site/markdown/website.md | 108 -
docs/src/site/markdown/wiki.md | 23 -
.../site/resources/images/logos/egg-logo.gif | Bin 3296 -> 0 bytes
.../site/resources/images/logos/egg-logo.png | Bin 0 -> 7241 bytes
.../resources/images/logos/trafodion-logo.jpg | Bin 0 -> 18953 bytes
docs/src/site/site.xml | 220 +-
install/.gitignore | 2 +
install/Makefile | 11 +-
install/installer/cloud_cli_setup | 89 +
install/installer/dcs_installer | 43 +-
install/installer/rest_installer | 10 +-
install/installer/setup_ssh_key | 27 +
.../tools/traf_cloudera_uninstall_suse | 5 +
install/installer/traf_add_sudoAccess | 53 +-
install/installer/traf_add_user | 37 +-
install/installer/traf_cloudera_mods98 | 119 +-
install/installer/traf_config | 92 +-
install/installer/traf_config_check | 216 +-
install/installer/traf_config_setup | 186 +-
install/installer/traf_create_systemdefaults | 203 -
install/installer/traf_getHadoopNodes | 13 +-
install/installer/traf_hortonworks_mods98 | 143 +-
install/installer/traf_package_setup | 29 +-
install/installer/traf_setup | 41 +-
install/installer/traf_sqgen | 98 +
install/installer/traf_start | 11 +-
install/installer/traf_user_prompt | 28 +
install/installer/traf_user_prompt_check | 27 +
install/installer/trafodion_config_default | 20 +-
install/installer/trafodion_install | 133 +-
install/installer/trafodion_uninstaller | 15 +-
install/traf_tools_setup.sh | 225 +-
licenses/Apache | 203 +
licenses/LICENSE-bsd2 | 14 -
licenses/LICENSE-bsd4 | 31 -
licenses/LICENSE-facebook | 14 -
licenses/LICENSE-js | 18 -
licenses/LICENSE-salesforce | 23 -
licenses/Makefile | 46 +
licenses/lic-clients-bin | 177 +
licenses/lic-components-src | 9 +
licenses/lic-dcs-bin | 9 +
licenses/lic-dcs-src | 69 +
licenses/lic-rest-bin | 9 +
licenses/lic-server-bin | 120 +
licenses/lic-server-src | 76 +
licenses/lic-test-src | 43 +
pom.xml | 56 +-
.../phoenix/end2end/AlterTableTest.java | 22 +
.../phoenix/end2end/ArithmeticQueryTest.java | 22 +
.../phoenix/end2end/AutoCommitTest.java | 22 +
.../org/trafodion/phoenix/end2end/BaseTest.java | 27 +-
.../trafodion/phoenix/end2end/BatchTest.java | 250 -
.../phoenix/end2end/BinaryRowKeyTest.java | 22 +
.../phoenix/end2end/CoalesceFunctionTest.java | 22 +
.../end2end/CompareDecimalToLongTest.java | 22 +
.../phoenix/end2end/CreateTableTest.java | 22 +
.../phoenix/end2end/CustomEntityDataTest.java | 22 +
.../phoenix/end2end/DeleteRangeTest.java | 22 +
.../end2end/DescColumnSortOrderTest.java | 24 +-
.../phoenix/end2end/DistinctCountTest.java | 22 +
.../phoenix/end2end/ExecuteStatementsTest.java | 22 +
.../phoenix/end2end/ExtendedQueryExecTest.java | 22 +
.../phoenix/end2end/FunkyNamesTest.java | 22 +
.../phoenix/end2end/GroupByCaseTest.java | 22 +
.../trafodion/phoenix/end2end/IndexTest.java | 22 +
.../trafodion/phoenix/end2end/IsNullTest.java | 22 +
.../trafodion/phoenix/end2end/KeyOnlyTest.java | 22 +
.../phoenix/end2end/MultiCfQueryExecTest.java | 22 +
.../trafodion/phoenix/end2end/OrderByTest.java | 22 +
.../phoenix/end2end/ProductMetricsTest.java | 24 +-
.../phoenix/end2end/QueryExecTest.java | 22 +
.../end2end/QueryExecWithoutSCNTest.java | 22 +
.../phoenix/end2end/QueryPlanTest.java | 22 +
.../phoenix/end2end/ReadIsolationLevelTest.java | 22 +
.../phoenix/end2end/SaltedTableTest.java | 22 +
.../end2end/SaltedTableUpsertSelectTest.java | 22 +
.../end2end/SaltedTableVarLengthRowKeyTest.java | 22 +
.../phoenix/end2end/ServerExceptionTest.java | 22 +
.../phoenix/end2end/SkipScanQueryTest.java | 22 +
.../phoenix/end2end/StatementHintsTest.java | 22 +
.../trafodion/phoenix/end2end/StddevTest.java | 22 +
.../phoenix/end2end/ToCharFunctionTest.java | 22 +
.../phoenix/end2end/ToNumberFunctionTest.java | 22 +
.../org/trafodion/phoenix/end2end/TopNTest.java | 22 +
.../phoenix/end2end/UpsertBigValuesTest.java | 22 +
.../end2end/UpsertSelectAutoCommitTest.java | 22 +
.../phoenix/end2end/UpsertSelectTest.java | 22 +
.../phoenix/end2end/UpsertValuesTest.java | 22 +
.../phoenix/end2end/VariableLengthPKTest.java | 22 +
wms/bin/local-servers.sh | 4 +-
wms/bin/servers.sh | 4 +-
wms/bin/start-wms.sh | 6 +-
wms/bin/stop-wms.sh | 8 +-
wms/bin/wms | 7 +-
wms/bin/wms-config.sh | 7 +-
wms/bin/wms-daemon.sh | 7 +-
wms/bin/wms-daemons.sh | 7 +-
wms/bin/zookeepers.sh | 7 +-
wms/conf/wms-env.sh | 31 +-
.../trafodion/wms/rest/ServerListResource.java | 2 +-
.../org/trafodion/wms/server/ServerManager.java | 1 -
wms/src/saveVersion.sh | 2 +-
723 files changed, 105964 insertions(+), 101738 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/cli/CliExpExchange.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/comexe/ComTdbHbaseAccess.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/comexe/ComTdbHbaseAccess.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/executor/ExHbaseAccess.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/executor/ExHbaseIUD.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/generator/GenItemFunc.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/generator/GenPreCode.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/optimizer/ItemFunc.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/sqlcomp/CmpSeabaseDDLindex.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/70e7a09a/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
[09/11] incubator-trafodion git commit: Fixes for review comments
Posted by sa...@apache.org.
Fixes for review comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/f3cbc3cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/f3cbc3cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/f3cbc3cd
Branch: refs/heads/master
Commit: f3cbc3cdd54770ad8677f985e78d8e5143e5fa95
Parents: 69fbcc9
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Tue Mar 8 21:19:40 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Tue Mar 8 21:19:40 2016 +0000
----------------------------------------------------------------------
core/sql/cli/Cli.cpp | 4 +---
core/sql/exp/ExpLOB.cpp | 4 ++--
2 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f3cbc3cd/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index 3ed4e06..145b42f 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -9622,7 +9622,6 @@ Lng32 SQLCLI_LOBcliInterface
if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(myDiags);
- cliInterface->retrieveSQLDiagnostics(myDiags);
if (myDiags->containsError(-CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION))
{
cliRC = 0;
@@ -9673,8 +9672,7 @@ Lng32 SQLCLI_LOBcliInterface
if (cliRC < 0)
{
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
+ cliInterface->retrieveSQLDiagnostics(myDiags);
goto error_return;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f3cbc3cd/core/sql/exp/ExpLOB.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.cpp b/core/sql/exp/ExpLOB.cpp
index 2b534ee..eba2655 100644
--- a/core/sql/exp/ExpLOB.cpp
+++ b/core/sql/exp/ExpLOB.cpp
@@ -248,7 +248,7 @@ Int32 ExpLOBoper::restoreLobDataFile(void *lobGlob, char *lobName, void *lobHeap
}
else
lobGlobL = lobGlob;
- rc = initLOBglobal(lobGlobL, lobHeap);
+
if (rc)
return -1;
@@ -269,7 +269,7 @@ Int32 ExpLOBoper::purgeBackupLobDataFile(void *lobGlob,char *lobName, void *lobH
}
else
lobGlobL = lobGlob;
- rc = initLOBglobal(lobGlobL, lobHeap);
+
if (rc)
return -1;
[04/11] incubator-trafodion git commit: Changes for performing GC.
Changes to pass in hdfsServer and port via CQD and remove hardcoding. Changes
to pass in lobLocation via CQD and remove hardcoding. Changes to remove all
mxlobsrvr process functionality
Posted by sa...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 9c66eed..e4a0c90 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -23,7 +23,7 @@
/* -*-C++-*-
*****************************************************************************
*
- * File: ex_lob.C
+ * File: EXLOBaccess.cpp
* Description: class to store and retrieve LOB data.
*
*
@@ -47,25 +47,25 @@
#include <sys/stat.h>
#include <sys/time.h>
-#define SQ_USE_HDFS 1
-#ifdef SQ_USE_HDFS
+
+
#include "hdfs.h"
#include "jni.h"
-#endif
+
#include "ExpLOBstats.h"
#include "ExpLOBaccess.h"
#include "ExpLOBinterface.h"
-
+#include "ExpLOBexternal.h"
#include "NAVersionedObject.h"
#include "ComQueue.h"
#include "NAMemory.h"
#include <seabed/ms.h>
-#include <../../sqf/src/seabed/src/trans.h>
#include <seabed/fserr.h>
#include <curl/curl.h>
+#include <../../sqf/src/seabed/src/trans.h>
extern int ms_transid_get(bool pv_supp,
bool pv_trace,
MS_Mon_Transid_Type *pp_transid,
@@ -79,15 +79,12 @@ ExLob::ExLob() :
storage_(Lob_Invalid_Storage),
dir_(string()),
lobGlobalHeap_(NULL),
-
- // fdDesc_(-1),
- fdDesc_(NULL),
fs_(NULL),
fdData_(NULL),
openFlags_(0)
{
lobDataFile_[0] = '\0';
- lobDescFile_[0] = '\0';
+
}
ExLob::~ExLob()
@@ -97,10 +94,8 @@ ExLob::~ExLob()
hdfsCloseFile(fs_, fdData_);
fdData_ = NULL;
}
- if (fdDesc_) {
- hdfsCloseFile(fs_, fdDesc_);
- fdDesc_ = NULL;
- }
+
+
/*
Commenting this out. It is causing cores during hive access.
@@ -118,6 +113,7 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
char *dir,
LobsStorage storage,
char *hdfsServer, Int64 hdfsPort,
+ char *lobLocation,
int bufferSize , short replication ,
int blockSize, Int64 lobMaxSize, ExLobGlobals *lobGlobals)
{
@@ -136,12 +132,12 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s/%s", dir_.c_str(), lobFile);
- snprintf(lobDescFile_, MAX_LOB_FILE_NAME_LEN, "%s/%s.desc", dir_.c_str(), lobFile);
+
}
else
{
snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s", lobFile);
- snprintf(lobDescFile_, MAX_LOB_FILE_NAME_LEN, "%s.desc", lobFile);
+
}
if (storage_ != Lob_Invalid_Storage)
@@ -156,7 +152,7 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
hdfsServer_ = hdfsServer;
hdfsPort_ = hdfsPort;
-
+ lobLocation_ = lobLocation;
clock_gettime(CLOCK_MONOTONIC, &startTime);
if (lobGlobals->getHdfsFs() == NULL)
@@ -200,110 +196,95 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
}
hdfsCloseFile(fs_, fdData_);
fdData_ = NULL;
- if (!lobGlobals->isHive())
- {
- //Create the desc header file that holds info about the
- //lob data file offsets etc.
- fdDesc_ = hdfsOpenFile(fs_, lobDescFile_, O_WRONLY, bufferSize, replication, blockSize);
- if (!fdDesc_)
- {
- return LOB_DESC_FILE_CREATE_ERROR;
- }
- //write empty header info into it.
- ExLobDescHeader header(lobMaxSize);
-
- Int64 numWritten = 0;
- numWritten = hdfsWrite(fs_, fdDesc_, (void *)&header, sizeof(ExLobDescHeader));
- if (numWritten <=0)
- return LOB_DATA_WRITE_ERROR;
-
-
- if (hdfsFlush(fs_, fdDesc_))
- return LOB_DATA_FLUSH_ERROR;
-
- hdfsCloseFile(fs_, fdDesc_);
- fdDesc_ = NULL;
- }
+
}
lobGlobalHeap_ = lobGlobals->getHeap();
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::fetchCursor()
-{
- Ex_Lob_Error err;
-
- request_.setType(Lob_Req_Fetch_Cursor);
-
- err = request_.send();
-
- if (err != LOB_OPER_OK) {
- return err;
- }
-
- err = request_.getError();
-
- return err;
-}
-
-Ex_Lob_Error ExLob::delDesc(Int64 descNum)
+Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOffset, Int64 &outSize,NABoolean &isEOD, Int64 transId)
{
- Ex_Lob_Error err;
-
- request_.setType(Lob_Req_Del_Desc);
- request_.setDescNumIn(descNum);
+ Ex_Lob_Error err = LOB_OPER_OK;
+ Int64 dummyParam;
+ int cliErr=0;
+ Int64 offset = 0;
+ Int64 size = 0;
+ lobCursors_it it = lobCursors_.find(string(handleIn, handleLenIn));
- err = request_.send();
+ if (it == lobCursors_.end())
+ {
+ return LOB_CURSOR_NOT_OPEN;
+ }
- if (err != LOB_OPER_OK) {
+ void *cliInterface = it->second.cliInterface_;
+
+
+ cliErr = SQL_EXEC_LOBcliInterface(handleIn, handleLenIn,
+ 0, 0,
+ (char *)&dummyParam, (Lng32 *)&dummyParam,
+ LOB_CLI_SELECT_FETCH, LOB_CLI_ExecImmed,
+ &offset, &size,
+ &dummyParam, &dummyParam,
+ &cliInterface,
+ transId);
+ if (err != LOB_OPER_OK)
+ {
return err;
- }
+ }
+ if (cliErr == 100 )
+ {
+ isEOD= TRUE;
+ cliErr = SQL_EXEC_LOBcliInterface(handleIn, handleLenIn,
+ NULL, NULL,
+ (char *)&dummyParam, (Lng32 *)&dummyParam,
+ LOB_CLI_SELECT_CLOSE, LOB_CLI_ExecImmed,
+ &dummyParam, &dummyParam,
+ &dummyParam, &dummyParam,
+ &cliInterface,
+ transId);
+
+ }
+ else
+ {
+ outOffset = offset;
+ outSize = size;
+ }
- err = request_.getError();
+
return err;
}
-Ex_Lob_Error ExLob::getDesc(ExLobDesc &desc)
-{
- Ex_Lob_Error err;
-
- request_.setType(Lob_Req_Get_Desc);
-
- err = request_.send();
-
- if (err != LOB_OPER_OK) {
- return err;
- }
-
- request_.getDescOut(desc);
- err = request_.getError();
- return err;
-}
-Ex_Lob_Error ExLob::putDesc(ExLobDesc &desc, Int64 descNum)
+Ex_Lob_Error ExLob::getDesc(ExLobDesc &desc,char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char *handleOut, Int32 &handleOutLen, Int64 transId)
{
- Ex_Lob_Error err;
-
- request_.setType(Lob_Req_Put_Desc);
- request_.setDescNumIn(descNum);
- request_.putDescIn(desc);
-
- err = request_.send();
-
- if (err != LOB_OPER_OK) {
- return err;
- }
-
- err = request_.getError();
+ Ex_Lob_Error err = LOB_OPER_OK;
+ NABoolean multipleChunks = FALSE;
+ Int32 clierr = 0;
+ Int64 size,offset,dummyParam = 0;
+
+ clierr = SQL_EXEC_LOBcliInterface(handleIn,
+ handleInLen,
+ blackBox, blackBoxLen,
+ handleOut, &handleOutLen,
+ LOB_CLI_SELECT_UNIQUE, LOB_CLI_ExecImmed,
+ &offset, &size,
+ &dummyParam, &dummyParam,
+ 0,
+ transId);
+
+ if (clierr < 0)
+ return LOB_DESC_READ_ERROR;
+ desc.setOffset(offset);
+ desc.setSize(size);
+
return err;
}
-
Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operLen)
{
Ex_Lob_Error err;
@@ -644,10 +625,14 @@ Ex_Lob_Error ExLob::readExternalSourceFile(char *srcfile, char *&fileData, Int32
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOper, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize)
+Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOper, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemSize,Int64 lobGCLimit, char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char *handleOut, Int32 &handleOutLen, void *lobGlobals)
{
- Ex_Lob_Error err;
+ Ex_Lob_Error err=LOB_OPER_OK;
Int64 dataOffset = 0;
+ Int64 outDescPartnKey = 0;
+ Int64 outDescSyskey = 0;
+ Int32 clierr = 0;
+
// Calculate sourceLen for each subOper.
if (subOper == Lob_File)
{
@@ -658,25 +643,59 @@ Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOpe
if (sourceLen <= 0 || sourceLen > lobMaxSize)
{
return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size
+ //TBD trigger compaction
}
- err = allocateDesc((unsigned int)sourceLen, descNumOut, dataOffset, lobMaxSize);
+ err = allocateDesc((unsigned int)sourceLen, descNumOut, dataOffset, lobMaxSize, lobMaxChunkMemSize,handleIn, handleInLen, lobGCLimit,lobGlobals);
operLen = 0;
if (err != LOB_OPER_OK)
return err;
-
- //send a message to mxlobsrvr to insert into the descriptor tables
- request_.setType(Lob_Req_Allocate_Desc);
- request_.getDesc().setSize(sourceLen);
- request_.setDescNumOut(descNumOut);
- request_.setDataOffset(dataOffset);
- err = request_.send();
- if (err != LOB_OPER_OK) {
- return err;
+
+ clierr = SQL_EXEC_LOBcliInterface(handleIn,
+ handleInLen,
+ NULL, blackBoxLen,
+ handleOut, &handleOutLen,
+ LOB_CLI_INSERT, LOB_CLI_ExecImmed,
+ &dataOffset, &sourceLen,
+ &outDescPartnKey, &outDescSyskey,
+ 0,
+ 0);
+ if (clierr < 0 ) {
+ return LOB_DESC_WRITE_ERROR;
}
return err;
}
+
+Ex_Lob_Error ExLob::insertDesc(Int64 offset, Int64 size, char *handleIn, Int32 handleInLen, char *handleOut, Int32 &handleOutLen, char *blackBox, Int32 blackBoxLen,void *lobGlobals)
+{
+
+ Lng32 clierr;
+ Int64 dummyParam;
+ Int64 outDescSyskey = 0;
+ Int64 outDescPartnKey = 0;
+ handleOutLen = 0;
+ Int32 chunkNum = 1;
+
+ NABoolean foundUnused = FALSE;
+
+ clierr = SQL_EXEC_LOBcliInterface(handleIn,
+ handleInLen,
+ NULL, &chunkNum,
+ handleOut, &handleOutLen,
+ LOB_CLI_INSERT, LOB_CLI_ExecImmed,
+ &offset, &size,
+ &outDescPartnKey, &outDescSyskey,
+ 0,
+ 0);
+
+ if (clierr < 0 ) {
+ return LOB_DESC_WRITE_ERROR;
+ }
+ return LOB_OPER_OK;
+}
+
+
Ex_Lob_Error ExLob::writeLobData(char *source, Int64 sourceLen, LobsSubOper subOperation, Int64 tgtOffset,Int64 &operLen, Int64 lobMaxChunkMemSize)
{
Ex_Lob_Error err;
@@ -694,8 +713,7 @@ Ex_Lob_Error ExLob::writeLobData(char *source, Int64 sourceLen, LobsSubOper subO
err = readSourceFile(source, inputAddr, allocMemSize, readOffset);
if (err != LOB_OPER_OK)
return err;
- }
-
+ }
else
{ // in memory
@@ -725,10 +743,10 @@ Ex_Lob_Error ExLob::writeLobData(char *source, Int64 sourceLen, LobsSubOper subO
return err;
}
-Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size, Int64 &operLen)
+Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size, Int64 &operLen,char * handleIn, Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, Int64 transId)
{
Ex_Lob_Error err = LOB_OPER_OK;
-
+ NABoolean multipleChunks = FALSE;
int cliErr;
@@ -737,12 +755,19 @@ Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size, Int64 &operLen)
Int64 sizeToRead = 0;
- err = getDesc(desc);
+ err = getDesc(desc,handleIn,handleInLen,blackBox, &blackBoxLen,handleOut,handleOutLen,transId);
+ if (err != LOB_OPER_OK)
+ {
+ return err;
+ }
sizeToRead = MINOF(size,desc.getSize());
- if (getRequest()->getBlackBoxLen() == -1)
- sizeToRead = size;
- err = readDataToMem(memAddr, desc.getOffset(),sizeToRead, operLen);
+ if (blackBoxLen == -1)
+ {
+ sizeToRead = size;
+ multipleChunks = TRUE;
+ }
+ err = readDataToMem(memAddr, desc.getOffset(),sizeToRead, operLen, handleIn,handleInLen, multipleChunks,transId);
return err;
}
@@ -774,18 +799,20 @@ LobInputOutputFileType ExLob::fileType(char *ioFileName)
else
return LOCAL_FILE;
}
-Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 tgtLength, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags)
+Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 tgtLength, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags,char *handleIn,Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, Int64 transId)
{
Ex_Lob_Error err = LOB_OPER_OK;
Int64 srcOffset = 0;
Int64 srcLength = 0;
LobInputOutputFileType tgtType = fileType(tgtFileName);
ExLobDesc desc;
- err = getDesc(desc);
+ NABoolean multipleChunks = FALSE;
+ err = getDesc(desc,handleIn,handleInLen,blackBox, &blackBoxLen,handleOut,handleOutLen,transId);
if (err != LOB_OPER_OK)
return err;
- if (getRequest()->getBlackBoxLen() == -1) // mxlobsrvr returned -1 indicating multiple chunks for this particular lob handle
+ if (blackBoxLen == -1) // mxlobsrvr returned -1 indicating multiple chunks for this particular lob handle
{
+ multipleChunks = TRUE;
//the data retrieval in chunks is handled in readDataToMem.
}
else if (tgtLength <=0 )
@@ -799,19 +826,19 @@ Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 tgtLength, Int64 &operLe
}
if (tgtType == HDFS_FILE)
{
- err = readDataToHdfsFile(tgtFileName, srcOffset , tgtLength,operLen, lobMaxChunkMemLen, fileflags);
+ err = readDataToHdfsFile(tgtFileName, srcOffset , tgtLength,operLen, lobMaxChunkMemLen, fileflags,handleIn,handleInLen,multipleChunks,transId);
if (err != LOB_OPER_OK)
return err;
}
else if(tgtType == CURL_FILE)
{
- err = readDataToExternalFile(tgtFileName, srcOffset, tgtLength, operLen, lobMaxChunkMemLen, fileflags);
+ err = readDataToExternalFile(tgtFileName, srcOffset, tgtLength, operLen, lobMaxChunkMemLen, fileflags,handleIn, handleInLen,multipleChunks,transId);
if (err != LOB_OPER_OK)
return err;
}
else if (tgtType == LOCAL_FILE)
{
- err = readDataToLocalFile(tgtFileName,srcOffset, tgtLength,operLen, lobMaxChunkMemLen, fileflags);
+ err = readDataToLocalFile(tgtFileName,srcOffset, tgtLength,operLen, lobMaxChunkMemLen, fileflags,handleIn,handleInLen,multipleChunks,transId);
if (err != LOB_OPER_OK)
return err;
}
@@ -821,12 +848,15 @@ Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 tgtLength, Int64 &operLe
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemSize)
+Ex_Lob_Error ExLob::append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemSize, Int64 lobGCLimit, char *handleIn, Int32 handleInLen, char * handleOut, Int32 &handleOutLen,void *lobGlobals)
{
Ex_Lob_Error err = LOB_OPER_OK;
Int64 dummyParam;
Int64 dataOffset=0;
Int64 sourceLen = size;
+ Int32 clierr = 0;
+ Int32 chunkNum = 0;
+ Int64 outDescPartnKey, outDescSyskey = 0;
if (so == Lob_File)
{
err = statSourceFile(data, sourceLen);
@@ -837,22 +867,21 @@ Ex_Lob_Error ExLob::append(char *data, Int64 size, LobsSubOper so, Int64 headDes
{
return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size
}
- err = allocateDesc((unsigned int)sourceLen, dummyParam, dataOffset, lobMaxSize);
+ err = allocateDesc((unsigned int)sourceLen, dummyParam, dataOffset, lobMaxSize,lobMaxChunkMemSize,handleIn, handleInLen,lobGCLimit,lobGlobals);
if (err != LOB_OPER_OK)
return err;
- request_.setType(Lob_Req_Append);
- request_.getDesc().setSize(sourceLen);
- request_.setDataOffset(dataOffset);
- request_.send();
-
- err = request_.getError();
-
- if (err != LOB_OPER_OK) {
- return err;
- }
-
- int cliErr = request_.getCliError();
- if (cliErr < 0 || cliErr == 100) { // some error or EOD.
+
+ clierr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen,
+ 0, &chunkNum,
+ handleOut, &handleOutLen,
+ LOB_CLI_INSERT_APPEND, LOB_CLI_ExecImmed,
+ &dataOffset, &sourceLen,
+ &outDescPartnKey, &outDescSyskey,
+ 0,
+ 0);
+
+
+ if (clierr < 0 || clierr == 100) { // some error or EOD.
return LOB_DESC_APPEND_ERROR;
}
@@ -866,20 +895,20 @@ Ex_Lob_Error ExLob::append(char *data, Int64 size, LobsSubOper so, Int64 headDes
return err;
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize)
+Ex_Lob_Error ExLob::insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,char * handleIn, Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals)
{
- Ex_Lob_Error err;
+ Ex_Lob_Error err=LOB_OPER_OK;
ExLobDesc desc;
-
+ int clierr = 0;
operLen = 0;
// get offset and input size from desc (the one that was just inserted into the descriptor handle table)
- err = getDesc(desc);
- if (err != LOB_OPER_OK)
- return err;
+
+ err = getDesc(desc,handleIn,handleInLen,blackBox, &blackBoxLen,handleOut,handleOutLen,0);
+
- int cliErr = request_.getCliError();
- if (cliErr < 0 || cliErr == 100) { // some error or EOD.
+
+ if (err !=LOB_OPER_OK) { // some error or EOD.
return LOB_DESC_READ_ERROR;
}
@@ -900,12 +929,15 @@ Ex_Lob_Error ExLob::insertData(char *data, Int64 size, LobsSubOper so,Int64 head
return err;
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::update(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize)
+Ex_Lob_Error ExLob::update(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize, Int64 lobGCLimit, char *handleIn, Int32 handleInLen, char *handleOut, Int32 &handleOutLen, void *lobGlobals)
{
Ex_Lob_Error err = LOB_OPER_OK;
Int64 dummyParam;
Int64 dataOffset = 0;
Int64 sourceLen = size;
+ Int32 clierr = 0;
+ Int64 outDescPartnKey,outDescSyskey = 0;
+ Int32 chunkNum = 0;
if (so == Lob_File)
{
err = statSourceFile(data, sourceLen);
@@ -916,24 +948,21 @@ Ex_Lob_Error ExLob::update(char *data, Int64 size, LobsSubOper so,Int64 headDesc
{
return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size
}
- err = allocateDesc((unsigned int)sourceLen, dummyParam, dataOffset, lobMaxSize);
+ err = allocateDesc((unsigned int)sourceLen, dummyParam, dataOffset, lobMaxSize, lobMaxChunkMemSize, handleIn, handleInLen, lobGCLimit,lobGlobals);
if (err != LOB_OPER_OK)
return err;
- // send a message to mxlobsrvr to do an update into descriptor tables
- request_.setType(Lob_Req_Update);
- request_.getDesc().setSize(sourceLen);
- request_.setDataOffset(dataOffset);
-
- request_.send();
-
- err = request_.getError();
-
- if (err != LOB_OPER_OK) {
- return err;
- }
-
- int cliErr = request_.getCliError();
- if (cliErr < 0 || cliErr == 100) { // some error or EOD.
+
+ clierr = SQL_EXEC_LOBcliInterface(handleIn,
+ handleInLen,
+ 0, &chunkNum,
+ handleOut, &handleOutLen,
+ LOB_CLI_UPDATE_UNIQUE, LOB_CLI_ExecImmed,
+ &dataOffset, &sourceLen,
+ &outDescPartnKey, &outDescSyskey,
+ 0,
+ 0);
+
+ if (clierr < 0 || clierr == 100) { // some error or EOD.
return LOB_DESC_UPDATE_ERROR;
}
char *inputAddr = data;
@@ -948,18 +977,27 @@ Ex_Lob_Error ExLob::update(char *data, Int64 size, LobsSubOper so,Int64 headDesc
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::delDesc()
+Ex_Lob_Error ExLob::delDesc(char *handleIn, Int32 handleInLen, Int64 transId)
{
- Ex_Lob_Error err;
- Int64 dummyParam;
-
- request_.setType(Lob_Req_Del_Desc);
+ Ex_Lob_Error err;
+ Int64 offset=0;
+ Int64 dummyParam=0;
+ Lng32 clierr=0;
- request_.send();
-
- err = request_.getError();
+ clierr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen,
+ 0, 0,
+ (char *)&dummyParam, (Lng32 *)&dummyParam,
+ LOB_CLI_DELETE, LOB_CLI_ExecImmed,
+ &dummyParam, &dummyParam,
+ &dummyParam, &dummyParam,
+ 0,
+ transId);
- return err;
+ if (clierr < 0)
+ return LOB_DESC_FILE_DELETE_ERROR;
+
+ return LOB_OPER_OK;
+
}
Ex_Lob_Error ExLob::purgeLob()
@@ -969,38 +1007,37 @@ Ex_Lob_Error ExLob::purgeLob()
{
return LOB_DATA_FILE_DELETE_ERROR;
}
- if (hdfsDelete(fs_, lobDescFile_, 0) != 0)
- {
- return LOB_DESC_FILE_DELETE_ERROR;
- }
-
-
+
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::openCursor(char *handleIn, Int64 handleInLen)
+Ex_Lob_Error ExLob::openCursor(char *handleIn, Int32 handleInLen,Int64 transId)
{
Ex_Lob_Error err;
cursor_t cursor;
+ Int32 clierr;
+ Int64 dummyParam = 0;
+ void *cliInterface = NULL;
+
+ clierr = SQL_EXEC_LOBcliInterface(handleIn,
+ handleInLen,
+ 0, 0,
+ (char *)&dummyParam, (Lng32 *)&dummyParam,
+ LOB_CLI_SELECT_CURSOR, LOB_CLI_ExecImmed,
+ &dummyParam, &dummyParam,
+ &dummyParam, &dummyParam,
+ &cliInterface,
+ transId);
- request_.setType(Lob_Req_Select_Cursor);
-
- err = request_.send();
-
- if (err != LOB_OPER_OK) {
- return err;
+ if (clierr <0 ) {
+ return LOB_DESC_READ_ERROR;
}
- err = request_.getError();
-
- if (err != LOB_OPER_OK) {
- return err;
- }
cursor.bytesRead_ = -1;
cursor.descOffset_ = -1;
cursor.descSize_ = -1;
- cursor.cliInterface_ = NULL; // used only in lob process
+ cursor.cliInterface_ = cliInterface; // used only in lob process
cursor.eod_ = false;
cursor.eor_ = false;
cursor.eol_ = false;
@@ -1103,7 +1140,7 @@ Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type, Int64 range,
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int64 handleInLen, Int64 &operLen)
+Ex_Lob_Error ExLob::readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int32 handleInLen, Int64 &operLen,Int64 transId)
{
int dataOffset;
Ex_Lob_Error result;
@@ -1129,7 +1166,7 @@ Ex_Lob_Error ExLob::readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int64 h
return LOB_OPER_OK;
}
- result = readCursorData(tgt, tgtSize, cursor, operLen); // increments cursor
+ result = readCursorData(tgt, tgtSize, cursor, operLen, handleIn,handleInLen,transId); // increments cursor
if (result != LOB_OPER_OK)
return result;
@@ -1139,221 +1176,38 @@ Ex_Lob_Error ExLob::readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int64 h
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::readDataCursorSimple(char *file, char *tgt, Int64 tgtSize,
- Int64 &operLen, ExLobGlobals *lobGlobals)
-{
- int dataOffset;
- Ex_Lob_Error result = LOB_OPER_OK;
- cursor_t *cursor;
- ExLobCursor::bufferList_t::iterator c_it;
- ExLobCursorBuffer *buf = NULL;
- Int64 bytesToCopy = 0;
- operLen = 0;
- Int64 len;
- char *target = tgt;
- bool done = false;
-
- struct timespec startTime;
- struct timespec endTime;
- lobCursorLock_.lock();
- lobCursors_it it = lobCursors_.find(string(file, strlen(file)));
- if (it == lobCursors_.end())
+Ex_Lob_Error ExLob::closeCursor(char *handleIn, Int32 handleInLen)
+{
+ lobCursors_it it = lobCursors_.find(string(handleIn, handleInLen));
+ if (it != lobCursors_.end())
{
- lobCursorLock_.unlock();
- return LOB_CURSOR_NOT_OPEN;
+ lobCursors_.erase(it);
}
- else
- {
- cursor = &(it->second);
- }
+ return LOB_OPER_OK;
+}
- lobCursorLock_.unlock();
- while ((operLen < tgtSize) && !done && !cursor->eol_)
- {
- lobGlobals->traceMessage("locking cursor",cursor,__LINE__);
- cursor->lock_.lock();
+Ex_Lob_Error ExLob::doSanityChecks(char *dir, LobsStorage storage,
+ Int32 handleInLen, Int32 handleOutLen,
+ Int32 blackBoxLen)
+{
- // if no buffers to read and is eor or eod, we are done.
- // else wait for prefetch thread to wake us up.
- if (cursor->prefetchBufList_.size() == 0) {
- if (cursor->eor_ || cursor->eod_) {
- done = true;
- } else {
- cursor->bufferMisses_++;
- lobGlobals->traceMessage("wait on condition cursor",cursor,__LINE__);
- cursor->lock_.wait();
- }
- lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- continue;
- }
+#ifdef SQ_USE_HDFS
+ if (!fs_)
+ return LOB_HDFS_CONNECT_ERROR;
+#else
+ if (fdData_ == -1)
+ return LOB_DATA_FILE_OPEN_ERROR;
+#endif
- // a buffer is available
- c_it = cursor->prefetchBufList_.begin();
- buf = *c_it;
- lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
+ if (dir_.compare(dir) != 0)
+ return LOB_DIR_NAME_ERROR;
- bytesToCopy = min(buf->bytesRemaining_, tgtSize - operLen);
- memcpy(target, buf->data_ + buf->bytesUsed_, bytesToCopy);
- target += bytesToCopy;
- if (bytesToCopy == buf->bytesRemaining_) { // buffer is now empty
- buf->bytesRemaining_ = -1;
- buf->bytesUsed_ = -1;
- lobGlobals->postfetchBufListLock_.lock();
- lobGlobals->postfetchBufList_.push_back(buf);
- lobGlobals->postfetchBufListLock_.unlock();
- lobGlobals->traceMessage("locking cursor",cursor,__LINE__);
- cursor->lock_.lock();
- c_it = cursor->prefetchBufList_.erase(c_it);
- lobGlobals->traceMessage("signal condition cursor",cursor,__LINE__);
- cursor->lock_.wakeOne(); // wake up prefetch thread if it was waiting for an empty buffer.
- lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- } else {
- buf->bytesUsed_ += bytesToCopy;
- buf->bytesRemaining_ -= bytesToCopy;
- }
- stats_.bytesPrefetched += bytesToCopy;
- operLen += bytesToCopy;
- }
-
- // update stats
- stats_.bytesRead += operLen;
- stats_.bytesToRead += tgtSize;
- stats_.numReadReqs++;
-
- return LOB_OPER_OK;
-}
-
-void ExLobCursor::emptyPrefetchList(ExLobGlobals *lobGlobals)
-{
- ExLobCursor::bufferList_t::iterator c_it;
- ExLobCursorBuffer *buf = NULL;
-
- c_it = prefetchBufList_.begin();
- while (c_it != prefetchBufList_.end())
- {
- buf = *c_it;
- lobGlobals->postfetchBufListLock_.lock();
- lobGlobals->postfetchBufList_.push_back(buf);
- lobGlobals->postfetchBufListLock_.unlock();
- c_it = prefetchBufList_.erase(c_it);
- }
-}
-
-// Seems like this is currently unused.
-// closeDataCusrorSimple takes care of destroying the cursor.But addign code
-// similar to closeDataCursorSimple for correctness in case it is used in future
-Ex_Lob_Error ExLob::deleteCursor(char *cursorName, ExLobGlobals *lobGlobals)
-{
- cursor_t *cursor = NULL;
-
- lobCursorLock_.lock();
-
- lobCursors_it it = lobCursors_.find(string(cursorName, strlen(cursorName)));
- if (it != lobCursors_.end())
- {
- cursor = &(it->second);
- lobGlobals->traceMessage("locking cursor",cursor,__LINE__);
- cursor->lock_.lock();
- cursor->emptyPrefetchList(lobGlobals);
- lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- lobCursors_.erase(it);
- }
-
- lobCursorLock_.unlock();
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::closeCursor(char *handleIn, Int64 handleInLen)
-{
- lobCursors_it it = lobCursors_.find(string(handleIn, handleInLen));
- if (it != lobCursors_.end())
- {
- lobCursors_.erase(it);
- }
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::closeDataCursorSimple(char *fileName, ExLobGlobals *lobGlobals)
-{
- cursor_t *cursor = NULL;
- Int64 secs = 0;
- Int64 nsecs = 0;
-
- lobCursorLock_.lock();
-
- lobCursors_it it = lobCursors_.find(string(fileName, strlen(fileName)));
- if (it != lobCursors_.end())
- {
- cursor = &(it->second);
- lobGlobals->traceMessage("locking cursor",cursor,__LINE__);
- cursor->lock_.lock();
-
- clock_gettime(CLOCK_MONOTONIC, &cursor->closeTime_);
- secs = cursor->closeTime_.tv_sec - cursor->openTime_.tv_sec;
- nsecs = cursor->closeTime_.tv_nsec - cursor->openTime_.tv_nsec;
-
- if (cursor->eod_ || cursor->eor_) { // prefetch thread already done,
- cursor->emptyPrefetchList(lobGlobals);
- lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- lobCursors_.erase(it); // so erase it here.
- // no need to unlock as cursor object is gone.
- } else {
- cursor->eol_ = true; // prefetch thread will do the eol rituals
- lobGlobals->traceMessage("signal condition cursor",cursor,__LINE__);
- cursor->lock_.wakeOne(); // wakeup prefetch thread
- lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- }
- }
-
- lobCursorLock_.unlock();
-
- if (nsecs < 0) {
- secs--;
- nsecs += NUM_NSECS_IN_SEC;
- }
- Int64 totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
- stats_.cursorElapsedTime += totalnsecs;
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::print()
-{
- Ex_Lob_Error err;
- request_.setType(Lob_Req_Print);
- err = request_.send();
- return err;
-}
-
-Ex_Lob_Error ExLob::doSanityChecks(char *dir, LobsStorage storage,
- Int64 handleInLen, Int64 handleOutLen,
- Int64 blackBoxLen)
-{
-
-#ifdef SQ_USE_HDFS
- if (!fs_)
- return LOB_HDFS_CONNECT_ERROR;
-#else
- if (fdData_ == -1)
- return LOB_DATA_FILE_OPEN_ERROR;
-#endif
-
- if (dir_.compare(dir) != 0)
- return LOB_DIR_NAME_ERROR;
-
- if (storage_ != storage)
- return LOB_STORAGE_TYPE_ERROR;
+ if (storage_ != storage)
+ return LOB_STORAGE_TYPE_ERROR;
if (handleInLen > MAX_HANDLE_IN_LEN) {
return LOB_HANDLE_IN_LEN_ERROR;
@@ -1370,84 +1224,199 @@ Ex_Lob_Error ExLob::doSanityChecks(char *dir, LobsStorage storage,
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::getNextFreeOffset(Int32 size,Int64 &freeOffset, Int64 lobMaxSize)
-{
- Ex_Lob_Error err = LOB_OPER_OK;
- Lng32 retval = 0;
- Int32 openFlags = O_RDONLY ;
- Int64 currentOffset = 0;
-
- return LOB_OPER_OK;
-
-}
-
-Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, Int64 lobMaxSize)
+Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, Int64 lobMaxSize, Int64 lobMaxChunkMemLen, char *handleIn, Int32 handleInLen, Int64 lobGCLimit, void *lobGlobals)
{
+ NABoolean GCDone = FALSE;
Ex_Lob_Error err = LOB_OPER_OK;
Lng32 retval = 0;
Int64 numRead = 0;
Int64 numWritten = 0;
-
- // TBD need a way to lock access to this file.
+ dataOffset = 0;
+ Int64 dummyParam = 0;
+ if (size > lobMaxSize)
+ return LOB_MAX_LIMIT_ERROR;
Int32 openFlags = O_RDONLY ;
- fdDesc_ = hdfsOpenFile(fs_, lobDescFile_, O_RDONLY, 0, 0,0);
- if (!fdDesc_) {
- hdfsCloseFile(fs_,fdDesc_);
- fdDesc_ = NULL;
- return LOB_DESC_FILE_OPEN_ERROR;
- }
- ExLobDescHeader header(lobMaxSize);
- numRead = hdfsPread(fs_,fdDesc_, 0, (void *)&header, sizeof(ExLobDescHeader) );
- if (numRead <=0)
- {
- return LOB_DESC_HEADER_READ_ERROR;
- }
- if (header.getAvailSize() >= size) {
- descNum = header.getFreeDesc();
-
- dataOffset = header.getDataOffset();
- header.incFreeDesc();
- header.decAvailSize(size);
- header.incDataOffset(size);
-
- hdfsCloseFile(fs_,fdDesc_);
- fdDesc_ = NULL;
- openFlags = O_WRONLY;
- fdDesc_ = hdfsOpenFile(fs_,lobDescFile_,openFlags,0,0,0);
- if (!fdDesc_) {
-
- return LOB_DESC_FILE_OPEN_ERROR;
- }
- numWritten = hdfsWrite(fs_,fdDesc_, (void *)&header, sizeof(ExLobDescHeader)) ;
- if (numWritten <= 0)
- {
- return LOB_DESC_HEADER_WRITE_ERROR;
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_, O_RDONLY, 0, 0,0);
+ if (!fdData_) {
+ hdfsCloseFile(fs_,fdData_);
+ fdData_ = NULL;
+ return LOB_DATA_FILE_OPEN_ERROR;
}
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
+ if (fInfo)
+ dataOffset = fInfo->mSize;
+
+ if (dataOffset > lobGCLimit) // 5 GB default
+ {
+ GCDone = TRUE;
+ /* Int32 rc = SQL_EXEC_LOBcliInterface(handleIn, handleInLen,
+ 0,0,
+ (char *)dummyParam, (Lng32 *)&dummyParam,
+ LOB_CLI_PERFORM_LOB_GC, LOB_CLI_ExecImmed,
+ &dummyParam, &dummyParam,
+ &dummyParam, &dummyParam,
+ NULL,
+ 0); //don't pass transid */
+ Int32 rc = SQL_EXEC_LOB_GC_Interface(lobGlobals,handleIn,handleInLen,
+ hdfsServer_,hdfsPort_,
+ lobLocation_,
+ lobMaxChunkMemLen);
+
+ }
+ if (GCDone) // recalculate the new offset
+ fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
+ if (fInfo)
+ dataOffset = fInfo->mSize;
-
+
+ //Find the last offset in the file
+ // dataOffset = hdfsTell(fs_,fdData_); //commenting out.hdfsTell always returns 0 !!
+
+ return LOB_OPER_OK;
+}
+Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int32 numEntries)
+{
+ Ex_Lob_Error rc = LOB_OPER_OK;
+ Int64 maxMemChunk = 1024*1024*1024; //1GB limit for intermediate buffer for transfering data
+ char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
+ str_sprintf(saveLobDataFile, "%s_save",lobDataFile_);
+ char * tmpLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+5];
+ str_sprintf(tmpLobDataFile, "%s_tmp",lobDataFile_);
+
+ hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_);
+ if (fs == NULL)
+ return LOB_DATA_FILE_OPEN_ERROR;
+
+
+ hdfsFile fdData = hdfsOpenFile(fs, lobDataFile_, O_RDONLY, 0, 0,0);
+ if (!fdData)
+ {
+ hdfsCloseFile(fs,fdData);
+ fdData = NULL;
+ return LOB_DATA_FILE_OPEN_ERROR;
}
- else {
- return LOB_DATA_FILE_FULL_ERROR;
+
+ hdfsFile fdTemp = hdfsOpenFile(fs, tmpLobDataFile,O_WRONLY|O_CREAT,0,0,0);
+ if (!fdTemp)
+ {
+ hdfsCloseFile(fs,fdTemp);
+ fdTemp = NULL;
+ return LOB_DATA_FILE_OPEN_ERROR;
}
- ExLobDesc desc(dataOffset, size, descNum);
- hdfsCloseFile(fs_,fdDesc_);
- fdDesc_=NULL;
- openFlags = O_WRONLY| O_APPEND;
- fdDesc_ = hdfsOpenFile(fs_,lobDescFile_,openFlags,0,0,0);
- numWritten = hdfsWrite(fs_,fdDesc_, (void *)&desc, sizeof(ExLobDesc));
- if (numWritten <= 0)
- {
- err = LOB_DESC_WRITE_ERROR;
- }
- hdfsCloseFile(fs_,fdDesc_);
- fdDesc_=NULL;
-
- // TBD need a way to unlock this hdfs file.
- return err;
+ Int32 i = 0;
+ Int64 bytesRead = 0;
+ Int64 bytesWritten = 0;
+ Int64 size = 0;
+ Int64 chunkLen = 0;
+ char * tgt = NULL;
+ while (i < numEntries)
+ {
+ chunkLen = dcArray[i].getChunkLen();
+ if (chunkLen > maxMemChunk)
+ {
+ tgt = (char *)(getLobGlobalHeap())->allocateMemory(maxMemChunk);
+ while (chunkLen > maxMemChunk)
+ {
+ bytesRead = hdfsPread(fs,fdData,dcArray[i].getCurrentOffset(),tgt,maxMemChunk);
+ if (bytesRead != maxMemChunk)
+ {
+ getLobGlobalHeap()->deallocateMemory(tgt);
+ return LOB_DATA_READ_ERROR;
+ }
+ bytesWritten = hdfsWrite(fs,fdTemp, tgt,maxMemChunk);
+ if (bytesWritten != size)
+ {
+ getLobGlobalHeap()->deallocateMemory(tgt);
+ return LOB_TARGET_FILE_WRITE_ERROR;
+ }
+ chunkLen -= maxMemChunk;
+ }
+
+ }
+ else
+ {
+ tgt = (char *)(getLobGlobalHeap())->allocateMemory(chunkLen);
+ bytesRead = hdfsPread(fs,fdData,dcArray[i].getCurrentOffset(),tgt,chunkLen);
+ if (bytesRead != chunkLen)
+ {
+ getLobGlobalHeap()->deallocateMemory(tgt);
+ return LOB_DATA_READ_ERROR;
+ }
+ bytesWritten = hdfsWrite(fs,fdTemp, tgt,chunkLen);
+ if (bytesWritten != chunkLen)
+ {
+ getLobGlobalHeap()->deallocateMemory(tgt);
+ return LOB_TARGET_FILE_WRITE_ERROR;
+ }
+ }
+ if (hdfsFlush(fs, fdTemp)) {
+ return LOB_DATA_FLUSH_ERROR;
+ }
+ getLobGlobalHeap()->deallocateMemory(tgt);
+ i++;
+ }
+ hdfsCloseFile(fs,fdTemp);
+ hdfsCloseFile(fs,fdData);
+
+ //Now save the data file and rename the tempfile to the original datafile
+
+ Int32 rc2 = hdfsRename(fs,lobDataFile_,saveLobDataFile);
+ if (rc2 == -1)
+ {
+ NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
+ NADELETEBASIC(tmpLobDataFile,getLobGlobalHeap());
+ return LOB_DATA_FILE_WRITE_ERROR;
+ }
+ rc2 = hdfsRename(fs,tmpLobDataFile, lobDataFile_);
+ if (rc2 == -1)
+ {
+ NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
+ NADELETEBASIC(tmpLobDataFile,getLobGlobalHeap());
+ return LOB_DATA_FILE_WRITE_ERROR;
+ }
+ NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
+ NADELETEBASIC(tmpLobDataFile,getLobGlobalHeap());
+ return LOB_OPER_OK;
}
+Ex_Lob_Error ExLob::restoreLobDataFile()
+{
+ Ex_Lob_Error rc = LOB_OPER_OK;
+
+
+ hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_);
+ if (fs == NULL)
+ return LOB_DATA_FILE_OPEN_ERROR;
+ char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
+ str_sprintf(saveLobDataFile, "%s_save",lobDataFile_);
+ Int32 rc2 = hdfsDelete(fs,lobDataFile_,FALSE);//ok to ignore error.
+ rc2 = hdfsRename(fs,saveLobDataFile, lobDataFile_);
+ if (rc2)
+ {
+ NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
+ return LOB_OPER_ERROR;
+ }
+ NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
+ return rc;
+
+}
+
+Ex_Lob_Error ExLob::purgeBackupLobDataFile()
+{
+ Ex_Lob_Error rc = LOB_OPER_OK;
+
+ hdfsFS fs = hdfsConnect(hdfsServer_,hdfsPort_);
+ if (fs == NULL)
+ return LOB_DATA_FILE_OPEN_ERROR;
+ char * saveLobDataFile = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
+ str_sprintf(saveLobDataFile, "%s_save",lobDataFile_);
+ Int32 rc2 = hdfsDelete(fs,saveLobDataFile,FALSE);//ok to ignore error.
+
+ NADELETEBASIC(saveLobDataFile,getLobGlobalHeap());
+ return rc;
+}
///////////////////////////////////////////////////////////////////////////////
// ExLobDescHeader definitions
///////////////////////////////////////////////////////////////////////////////
@@ -1482,7 +1451,7 @@ ExLobDesc::~ExLobDesc()
{
}
-Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen)
+Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen, char *handleIn, Int32 handleLenIn, Int64 transId)
{
ExLobDesc desc;
Ex_Lob_Error err;
@@ -1493,23 +1462,26 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
tOffset offset;
struct timespec startTime;
struct timespec endTime;
+ NABoolean isEOD=FALSE;
+ Int64 outOffset = 0;
+ Int64 outSize = 0;
while ( (operLen < tgtSize) && !cursor.eod_ )
{
if (cursor.bytesRead_ == cursor.descSize_) // time to read next chunck
{
- err = fetchCursor();
+ err = fetchCursor(handleIn, handleLenIn,outOffset, outSize,isEOD,transId);
if (err != LOB_OPER_OK) {
return err;
}
- if (request_.getCliError() == 100) {
+ if (isEOD) {
cursor.eod_ = true; // subsequent call will return 100 and close the cursor
continue;
} else {
- cursor.descSize_ = request_.getDesc().getSize();
- cursor.descOffset_ = request_.getDesc().getOffset();
+ cursor.descSize_ = outSize;
+ cursor.descOffset_ = outOffset;
cursor.bytesRead_ = 0;
}
}
@@ -1549,7 +1521,7 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
if (bytesRead == -1) {
return LOB_DATA_READ_ERROR;
} else if (bytesRead == 0) {
- cursor.eod_ = true;
+ cursor.eod_ = true;
continue;
}
@@ -1563,92 +1535,29 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen)
-{
- ExLobDesc desc;
- Ex_Lob_Error err;
- Int64 bytesAvailable = 0;
- Int64 bytesToCopy = 0;
- Int64 bytesRead = 0;
- operLen = 0;
- tOffset offset;
- struct timespec startTime;
- struct timespec endTime;
- bool done = false;
-
- if (!fdData_) {
- return LOB_CURSOR_NOT_OPEN_ERROR;
- }
-
- if (cursor.bytesRead_ == -1) { // starting
- cursor.bytesRead_ = 0;
- }
-
- clock_gettime(CLOCK_MONOTONIC, &startTime);
-
- while ( (operLen < tgtSize) && !done )
- {
- //offset = cursor.descOffset_ + cursor.bytesRead_;
- bytesToCopy = tgtSize - operLen;
- offset = cursor.descOffset_ + cursor.bytesRead_;
-
- // gets chunks of 64KB. Uses readDirect internally.
- // bytesRead = hdfsPread(fs_, fdData_, offset, tgt, bytesToCopy);
- bytesRead = hdfsRead(fs_, fdData_, tgt, bytesToCopy);
-
- stats_.numHdfsReqs++;
-
- if (bytesRead == -1) {
- return LOB_DATA_READ_ERROR;
- } else if (bytesRead == 0) {
- done = true;
- }
-
- cursor.bytesRead_ += bytesRead;
- operLen += bytesRead;
- tgt += bytesRead;
- }
-
- clock_gettime(CLOCK_MONOTONIC, &endTime);
-
- Int64 secs = endTime.tv_sec - startTime.tv_sec;
- Int64 nsecs = endTime.tv_nsec - startTime.tv_nsec;
- if (nsecs < 0) {
- secs--;
- nsecs += NUM_NSECS_IN_SEC;
- }
-
- Int64 totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
- stats_.CumulativeReadTime += totalnsecs;
- return LOB_OPER_OK;
-}
Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
- Int64 offset, Int64 size, Int64 &operLen)
+ Int64 offset, Int64 size, Int64 &operLen,
+ char *handleIn, Int32 handleLenIn,
+ NABoolean multipleChunks, Int64 transId)
{
Ex_Lob_Error err = LOB_OPER_OK;
operLen = 0;
Int64 bytesRead = 0;
- NABoolean multipleChunks = FALSE;
- if (getRequest()->getBlackBoxLen() == -1) // mxlobsrvr returned -1 indicating multiple chunks for this particular lob handle
+ if (multipleChunks)
{
- multipleChunks = TRUE;
- err = openCursor(getRequest()->getHandleIn(),
- getRequest()->getHandleInLen());
+ err = openCursor(handleIn,
+ handleLenIn,transId);
//now we can fetch the descriptors for each chunk
}
- else
- if (err != LOB_OPER_OK)
- return err;
- int cliErr = request_.getCliError();
- if (cliErr < 0 || cliErr == 100) {
- return LOB_DESC_READ_ERROR;
- }
+ if (err != LOB_OPER_OK)
+ return err;
+
if (fdData_)// we may have a stale handle. close and open to refresh
{
@@ -1676,22 +1585,19 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
memAddr, size)) == -1) {
return LOB_DATA_READ_ERROR;
- }
-
-
+ }
operLen = bytesRead;
return LOB_OPER_OK;
}
else
{
//handle reading the multiple chunks like a cursor
- err = readCursor(memAddr,size, getRequest()->getHandleIn(),
- getRequest()->getHandleInLen(), operLen);
-
-
+ err = readCursor(memAddr,size, handleIn,
+ handleLenIn, operLen, transId);
+
if (err==LOB_OPER_OK)
- closeCursor(getRequest()->getHandleIn(),
- getRequest()->getHandleInLen());
+ closeCursor(handleIn,
+ handleLenIn);
else
return err;
}
@@ -1700,7 +1606,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
-Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemSize, Int32 fileflags)
+Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemSize, Int32 fileflags,char *handleIn,Int32 handleInLen, NABoolean multipleChunks,Int64 transId)
{
Ex_Lob_Error err;
Int64 operLen = 0;
@@ -1710,10 +1616,10 @@ Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 siz
Int64 tgtOffset = 0;
char *lobData = 0;
Int64 chunkSize = 0;
-
+
if (srcLen <=0)
return LOB_SOURCE_DATA_ALLOC_ERROR;
- // open the targte file for writing
+ // open the target file for writing
int filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
int openFlags = O_RDWR ; // O_DIRECT needs mem alignment
if (((LobTgtFileFlags)fileflags == Lob_Append_Or_Error ) ||
@@ -1742,15 +1648,16 @@ Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 siz
return LOB_TARGET_FILE_OPEN_ERROR;
}
}
- if ((srcLen < lobMaxChunkMemSize) && (getRequest()->getBlackBoxLen() != -1)) // simple single I/O case
+ if ((srcLen < lobMaxChunkMemSize) && (multipleChunks ==FALSE)) // simple single I/O case
{
+
lobData = (char *) (getLobGlobalHeap())->allocateMemory(srcLen);
if (lobData == NULL)
{
return LOB_SOURCE_DATA_ALLOC_ERROR;
}
- err = readDataToMem(lobData, srcOffset,srcLen,operLen);
+ err = readDataToMem(lobData, srcOffset,srcLen,operLen, handleIn,handleInLen, multipleChunks,transId);
if (err != LOB_OPER_OK)
{
getLobGlobalHeap()->deallocateMemory(lobData);
@@ -1767,8 +1674,8 @@ Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 siz
}
else // multiple chunks to read
{
- err = openCursor(getRequest()->getHandleIn(),
- getRequest()->getHandleInLen());
+ err = openCursor(handleIn,
+ handleInLen,transId);
if (err != LOB_OPER_OK)
return err;
while ( srcLen > 0)
@@ -1782,8 +1689,8 @@ Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 siz
return LOB_SOURCE_DATA_ALLOC_ERROR;
}
//handle reading the multiple chunks like a cursor
- err = readCursor(lobData,chunkSize, getRequest()->getHandleIn(),
- getRequest()->getHandleInLen(), operLen);
+ err = readCursor(lobData,chunkSize, handleIn,
+ handleInLen, operLen, transId);
if ((err != LOB_OPER_OK) || (operLen != chunkSize))
{
@@ -1801,15 +1708,15 @@ Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 siz
srcLen -= chunkSize;
tgtOffset += chunkSize;
}
- closeCursor(getRequest()->getHandleIn(),
- getRequest()->getHandleInLen());
+ closeCursor(handleIn,
+ handleInLen);
}
close(fdDestFile);
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemLen, Int32 fileflags)
+Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemLen, Int32 fileflags,char *handleIn, Int32 handleInLen, NABoolean multipleChunks,Int64 transId)
{
Ex_Lob_Error err;
Int64 operLen = 0;
@@ -1820,6 +1727,7 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s
char *lobData = 0;
Int64 chunkSize = 0;
hdfsFile fdTgtFile;
+
// open and write to the target file
int openFlags = O_WRONLY;
if ((LobTgtFileFlags)fileflags == Lob_Append_Or_Error )
@@ -1855,7 +1763,7 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s
}
}
- if ((srcLen < lobMaxChunkMemLen) && (getRequest()->getBlackBoxLen() != -1)) // simple single I/O case
+ if ((srcLen < lobMaxChunkMemLen) && (multipleChunks ==FALSE)) // simple single I/O case
{
lobData = (char *) (getLobGlobalHeap())->allocateMemory(srcLen);
@@ -1863,7 +1771,7 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s
{
return LOB_SOURCE_DATA_ALLOC_ERROR;
}
- err = readDataToMem(lobData, srcOffset,srcLen,operLen);
+ err = readDataToMem(lobData, srcOffset,srcLen,operLen,handleIn,handleInLen, multipleChunks,transId);
if (err != LOB_OPER_OK)
{
getLobGlobalHeap()->deallocateMemory(lobData);
@@ -1885,8 +1793,9 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s
}
else
{// multiple chunks to read
- err = openCursor(getRequest()->getHandleIn(),
- getRequest()->getHandleInLen());
+ err = openCursor(handleIn,
+ handleInLen,
+ transId);
if (err != LOB_OPER_OK)
return err;
while ( srcLen > 0)
@@ -1899,13 +1808,13 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s
return LOB_SOURCE_DATA_ALLOC_ERROR;
}
//handle reading the multiple chunks like a cursor
- err = readCursor(lobData,chunkSize, getRequest()->getHandleIn(),
- getRequest()->getHandleInLen(), operLen);
+ err = readCursor(lobData,chunkSize, handleIn,
+ handleInLen, operLen, transId);
if ((err != LOB_OPER_OK) || (operLen != chunkSize))
{
getLobGlobalHeap()->deallocateMemory(lobData);
- return err;
+ return LOB_DATA_READ_ERROR;
}
writeOperLen += hdfsWrite(fs_,fdTgtFile,lobData, chunkSize);
if (writeOperLen <= 0)
@@ -1922,8 +1831,8 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s
srcLen -= chunkSize;
}
- closeCursor(getRequest()->getHandleIn(),
- getRequest()->getHandleInLen());
+ closeCursor(handleIn,
+ handleInLen);
}
hdfsCloseFile(fs_, fdTgtFile);
fdTgtFile=NULL;
@@ -1936,7 +1845,7 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s
-Ex_Lob_Error ExLob::readDataToExternalFile(char *tgtFileName, Int64 offset, Int64 size, Int64 &operLen,Int64 lobMaxChunkMemLen,Int32 fileflags)
+Ex_Lob_Error ExLob::readDataToExternalFile(char *tgtFileName, Int64 offset, Int64 size, Int64 &operLen,Int64 lobMaxChunkMemLen,Int32 fileflags,char *handleIn,Int32 handleInLen, NABoolean multipleChunks,Int64 transId)
{
//TBD
return LOB_OPER_OK;
@@ -1965,975 +1874,1210 @@ Ex_Lob_Error ExLob::initStats()
stats_.init();
return LOB_OPER_OK;
}
-void ExLobGlobals::traceMessage(const char *logMessage, ExLobCursor *cursor,
- int line)
-{
- if ( threadTraceFile_ && logMessage)
- {
- fprintf(threadTraceFile_,
- "Thread: 0x%lx Line: %d %s 0x%lx\n" ,
- (unsigned long)pthread_self(), line, logMessage,
- (unsigned long) cursor);
- fflush(threadTraceFile_);
- }
-
-}
-Ex_Lob_Error ExLobGlobals::performRequest(ExLobHdfsRequest *request)
-{
+//Main driver of any LOB related operation
+
+Ex_Lob_Error ExLobsOper (
+ char *lobName, // lob name
+ char *handleIn, // input handle (for cli calls)
+ Int32 handleInLen, // input handle len
+ char *hdfsServer, // server where hdfs fs resides
+ Int64 hdfsPort, // port number to access hdfs server
+ char *handleOut, // output handle (for cli calls)
+ Int32 &handleOutLen, // output handle len
+ Int64 descNumIn, // input desc Num (for flat files only)
+ Int64 &descNumOut, // output desc Num (for flat files only)
+ Int64 &retOperLen, // length of data involved in this operation
+ Int64 requestTagIn, // only for checking status
+ Int64 &requestTagOut, // returned with every request other than check status
+ Ex_Lob_Error &requestStatus, // returned req status
+ Int64 &cliError, // err returned by cli call
+ char *dir, // directory in the storage
+ LobsStorage storage, // storage type
+ char *source, // source (memory addr, filename, foreign lob etc)
+ Int64 sourceLen, // source len (memory len, foreign desc offset etc)
+ Int64 cursorBytes,
+ char *cursorId,
+ LobsOper operation, // LOB operation
+ LobsSubOper subOperation, // LOB sub operation
+ Int64 waited, // waited or nowaited
+ void *&globPtr, // ptr to the Lob objects.
+ Int64 transId,
+ void *blackBox, // black box to be sent to cli
+ Int32 blackBoxLen, // length of black box
+ Int64 lobMaxSize,
+ Int64 lobMaxChunkMemSize,
+ Int64 lobGCLimit,
+ int bufferSize ,
+ short replication ,
+ int blockSize,
+ Lng32 openType)
+{
Ex_Lob_Error err = LOB_OPER_OK;
- ExLob *lobPtr;
- ExLobCursorBuffer *buf;
- ExLobCursor *cursor;
- Int64 size;
- NABoolean seenEOR = false;
- NABoolean seenEOD = false;
- ExLobCursor::bufferList_t::iterator c_it;
- Int64 totalBufSize;
+ ExLob *lobPtr = NULL;
+ char fn[MAX_LOB_FILE_NAME_LEN];
+ struct timespec startTime;
+ struct timespec endTime;
+ Int64 secs, nsecs, totalnsecs;
+ ExLobPreOpen *preOpenObj;
+ ExLobGlobals *lobGlobals = NULL;
+ transId = 0;
+ retOperLen = 0;
+ ExLobDesc desc;
+
+ lobMap_t *lobMap = NULL;
+ lobMap_it it;
- switch (request->reqType_)
- {
- case Lob_Hdfs_Cursor_Prefetch :
- lobPtr = request->lobPtr_;
- cursor = request->cursor_;
- traceMessage("locking cursor",cursor,__LINE__);
- cursor->lock_.lock();
- while (!cursor->eod_ && !cursor->eor_ && !cursor->eol_)
- {
- postfetchBufListLock_.lock();
- c_it = postfetchBufList_.begin();
- if (c_it != postfetchBufList_.end()) {
- buf = *c_it;
- postfetchBufList_.erase(c_it);
- postfetchBufListLock_.unlock();
- traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- } else {
- postfetchBufListLock_.unlock();
- // there are no empty buffers.
- // if prefetch list already has the max, wait for one to free up.
- totalBufSize = cursor->prefetchBufList_.size() * cursor->bufMaxSize_;
- if (totalBufSize > LOB_CURSOR_PREFETCH_BYTES_MAX) {
- traceMessage("wait on condition cursor",cursor,__LINE__);
- cursor->lock_.wait();
- char buffer2[2048];
- sprintf(buffer2, "cursor->eod_ %d cursor->eor_ %d "
- "cursor->eol_ %d", cursor->eod_,
- cursor->eor_, cursor->eol_);
- traceMessage(buffer2, cursor, __LINE__);
- continue;
- }
- // create a new buffer
- traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- buf = new (getHeap()) ExLobCursorBuffer();
- buf->data_ = (char *) (getHeap())->allocateMemory( cursor->bufMaxSize_);
- lobPtr->stats_.buffersUsed++;
- }
- size = min(cursor->bufMaxSize_, (cursor->maxBytes_ - cursor->bytesRead_ + (16 * 1024)));
- if (buf->data_) {
- lobPtr->readCursorDataSimple(buf->data_, size, *cursor, buf->bytesRemaining_);
- buf->bytesUsed_ = 0;
- traceMessage("locking cursor",cursor,__LINE__);
- cursor->lock_.lock();
- if (size < (cursor->bufMaxSize_)) {
- cursor->eor_ = true;
- seenEOR = true;
- }
- if (buf->bytesRemaining_) {
- cursor->prefetchBufList_.push_back(buf);
- traceMessage("signal condition cursor",cursor,__LINE__);
- cursor->lock_.wakeOne();
- traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- } else {
- cursor->eod_ = true;
- seenEOD = true;
- traceMessage("signal condition cursor",cursor,__LINE__);
- cursor->lock_.wakeOne();
- traceMessage("unlocking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- postfetchBufListLock_.lock();
- postfetchBufList_.push_back(buf);
- postfetchBufListLock_.unlock();
- }
- } else {
- assert("data_ is null");
- }
- // Important! Break and do not access cursor object if we have reached
- // end of data or range.
- // The main thread could have destroyed the cursor
- // in ::closeDataCursorSimple
- if (seenEOD || seenEOR)
- {
- char buffer2[2048];
- sprintf(buffer2, "seenEOD %d seenEOR %d",
- seenEOD, seenEOR);
- traceMessage(buffer2, cursor, __LINE__);
- break;
- }
- traceMessage("locking cursor",cursor,__LINE__);
- cursor->lock_.lock();
- } // while
+ clock_gettime(CLOCK_MONOTONIC, &startTime);
- if (!seenEOD && !seenEOR)
+ char *fileName = lobName;
+
+ if (globPtr == NULL)
+ {
+ if (operation == Lob_Init)
{
- traceMessage("locking cursor",cursor,__LINE__);
- cursor->lock_.unlock();
- if (cursor->eol_) { // never reaches here ??
- lobPtr->deleteCursor(cursor->name_, this);
- }
+ globPtr = (void *) new ExLobGlobals();
+ if (globPtr == NULL)
+ return LOB_INIT_ERROR;
+
+ lobGlobals = (ExLobGlobals *)globPtr;
+
+ err = lobGlobals->initialize();
+ return err;
}
- processPreOpens();
- break;
+ else
+ {
+ return LOB_GLOB_PTR_ERROR;
+ }
+ }
+ else
+ {
+ lobGlobals = (ExLobGlobals *)globPtr;
- default:
- request->error_ = LOB_HDFS_REQUEST_UNKNOWN;
- }
+ lobMap = lobGlobals->getLobMap();
- return LOB_OPER_OK;
-}
+ it = lobMap->find(string(fileName));
-Ex_Lob_Error ExLobDesc::print()
-{
- printf("%4d %4d %4d %4d %4d %4d %8d\n",
- dataSize_, dataState_, tail_, prev_, next_, nextFree_, dataOffset_);
- return LOB_OPER_OK;
-}
+ if (it == lobMap->end())
+ {
+ //lobPtr = new (lobGlobals->getHeap())ExLob();
+ lobPtr = new ExLob();
+ if (lobPtr == NULL)
+ return LOB_ALLOC_ERROR;
-///////////////////////////////////////////////////////////////////////////////
-// ExLobGlobals definitions
-///////////////////////////////////////////////////////////////////////////////
+ err = lobPtr->initialize(fileName, (operation == Lob_Create) ? EX_LOB_CREATE : EX_LOB_RW, dir, storage, hdfsServer, hdfsPort, dir,bufferSize, replication, blockSize,lobMaxSize,lobGlobals);
+ if (err != LOB_OPER_OK)
+ return err;
-ExLobGlobals::ExLobGlobals() :
- lobMap_(NULL),
- fs_(NULL),
- isCliInitialized_(FALSE),
- isHive_(FALSE),
- threadTraceFile_(NULL),
- heap_(NULL)
-{
- //initialize the log file
- if (getenv("TRACE_HDFS_THREAD_ACTIONS"))
+ lobMap->insert(pair<string, ExLob*>(string(fileName), lobPtr));
+ }
+ else
+ {
+ lobPtr = it->second;
+
+ }
+ }
+ /*
+// **Note** This is code that needs to get called before sneding a request to the
+//mxlobsrvr process. It's inactive code currently
+ MS_Mon_Transid_Type transIdBig;
+ MS_Mon_Transseq_Type transStartId;
+ if (!lobGlobals->isHive())
{
- char logFileName[50]= "";
- sprintf(logFileName,"trace_threads.%d",getpid());
- threadTraceFile_ = fopen(logFileName,"a");
+ // get current transaction
+
+ int transIdErr = ms_transid_get(false, false, &transIdBig, &transStartId);
+ // set the pass thru request object values in the lob
+
+ lobPtr->getRequest()->setValues(lobPtr->getDescFileName(),
+ descNumIn, handleInLen, handleIn, storage,
+ transId, transIdBig, transStartId,
+ (char *)blackBox, blackBoxLen);
}
-}
+ */
+ switch(operation)
+ {
+ case Lob_Create:
+ break;
-ExLobGlobals::~ExLobGlobals()
-{
- ExLobCursor::bufferList_t::iterator c_it;
- ExLobCursorBuffer *buf = NULL;
+ case Lob_InsertDesc:
+ err = lobPtr->writeDesc(sourceLen, source, subOperation, descNumOut, retOperLen, lobMaxSize, lobMaxChunkMemSize,lobGCLimit,handleIn,handleInLen,(char *)blackBox, &blackBoxLen,handleOut,handleOutLen,lobGlobals);
+ break;
- preOpenListLock_.lock();
- preOpenList_.clear();
- preOpenListLock_.unlock();
+ case Lob_InsertData:
+ err = lobPtr->insertData(source, sourceLen, subOperation, descNumIn, retOperLen, lobMaxSize,lobMaxChunkMemSize,handleIn,handleInLen,(char *)blackBox, blackBoxLen,handleOut,handleOutLen,lobGlobals);
+ break;
-
- if (lobMap_)
- delete lobMap_;
+ case Lob_InsertDataSimple:
+ err = lobPtr->writeDataSimple(source, sourceLen, subOperation, retOperLen,
+ bufferSize , replication , blockSize);
+ break;
- for (int i=0; i<NUM_WORKER_THREADS; i++) {
- enqueueShutdownRequest();
- }
+ case Lob_Read:
+ if (subOperation == Lob_Memory)
+ err = lobPtr->readToMem(source,sourceLen,retOperLen,handleIn,handleInLen,(char *)blackBox, blackBoxLen,handleOut,handleOutLen,transId);
+ else if (subOperation == Lob_File)
+ err = lobPtr->readToFile(source, sourceLen, retOperLen, lobMaxChunkMemSize, openType,handleIn,handleInLen,(char *)blackBox, blackBoxLen,handleOut,handleOutLen,transId);
+ else
+ err = LOB_SUBOPER_ERROR;
+ break;
- for (int i=0; i<NUM_WORKER_THREADS; i++) {
- pthread_join(threadId_[i], NULL);
- }
- // Free the post fetch bugf list AFTER the worker threads have left to
- // avoid slow worker thread being stuck and master deallocating these
- // buffers and not consuming the buffers which could cause a lock.
-
- postfetchBufListLock_.lock();
- c_it = postfetchBufList_.begin();
- while (c_it != postfetchBufList_.end()) {
- buf = *c_it;
- if (buf->data_) {
- heap_->deallocateMemory( buf->data_);
- }
- c_it = postfetchBufList_.erase(c_it);
- }
- postfetchBufListLock_.unlock();
-
- //msg_mon_close_process(&serverPhandle);
- if (threadTraceFile_)
- fclose(threadTraceFile_);
- threadTraceFile_ = NULL;
-}
+ case Lob_ReadDesc: // read desc only. Needed for pass thru.
+ err = lobPtr->getDesc(desc,handleIn,handleInLen,(char *)blackBox, &blackBoxLen,handleOut,handleOutLen,transId);
+ retOperLen = 0;
+ break;
+ case Lob_OpenCursor:
+ err = lobPtr->openCursor(handleIn, handleInLen,transId);
+ break;
-Ex_Lob_Error ExLobGlobals::setServerPhandle()
-{
- int nid;
-
- int err = msg_mon_get_my_info(&nid, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
- char server[12];
- sprintf(server, "%s%d", "$ZLOBSRV", nid);
+ case Lob_OpenDataCursorSimple:
+ if (openType == 1) { // preopen
+ sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
+ preOpenObj = new (lobGlobals->getHeap()) ExLobPreOpen(lobPtr, fn, descNumIn, sourceLen, cursorBytes, waited);
+ lobGlobals->addToPreOpenList(preOpenObj);
+ } else if (openType == 2) { // must open
+ sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
+ fileName = fn;
+ err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals);
+ } else
+ err = LOB_SUBOPER_ERROR;
+ break;
- int oid;
- err = msg_mon_open_process(server, &serverPhandle, &oid);
+ case Lob_ReadCursor:
+ if ((subOperation == Lob_Memory) || (subOperation == Lob_Buffer))
+ err = lobPtr->readCursor(source, sourceLen, handleIn, handleInLen, retOperLen,transId);
+ else if (subOperation == Lob_File)
+ err = lobPtr->readCursor(source, -1, handleIn, handleInLen, retOperLen,transId);
+ else
+ err = LOB_SUBOPER_ERROR;
+ break;
- if (err != XZFIL_ERR_OK)
- return LOB_SERVER_OPEN_ERROR;
+ case Lob_ReadDataCursorSimple:
+ sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
+ fileName = fn;
+ err = lobPtr->readDataCursorSimple(fileName, source, sourceLen, retOperLen, lobGlobals);
+ break;
- return LOB_OPER_OK;
-}
+ case Lob_CloseFile:
+ if (lobPtr->hasNoOpenCursors()) {
+ lobGlobals->traceMessage("Lob_CloseFile",NULL,__LINE__);
+ err = lobPtr->closeFile();
+ it = lobMap->find(string(lobName));
+ lobMap->erase(it);
+ delete lobPtr;
+ lobPtr = NULL;
+ }
+ break;
-Ex_Lob_Error ExLobGlobals::resetServerPhandle()
-{
- Ex_Lob_Error err;
+ case Lob_CloseCursor:
+ err = lobPtr->closeCursor(handleIn, handleInLen);
+ break;
- msg_mon_close_process(&serverPhandle);
+ case Lob_CloseDataCursorSimple:
+ sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
+ fileName = fn;
+ err = lobPtr->closeDataCursorSimple(fileName, lobGlobals);
+ break;
- err = setServerPhandle();
+ case Lob_Append:
+ if ((subOperation == Lob_Memory) ||(subOperation == Lob_Buffer))
+ err = lobPtr->append(source, sourceLen, subOperation, descNumIn, retOperLen,lobMaxSize, lobMaxChunkMemSize,lobGCLimit,handleIn,handleInLen,handleOut,handleOutLen,lobGlobals);
+ else if (subOperation == Lob_File)
+ err = lobPtr->append(source, -1, subOperation, descNumIn, retOperLen,lobMaxSize, lobMaxChunkMemSize,lobGCLimit,handleIn,handleInLen,handleOut,handleOutLen,lobGlobals);
+ else
+ err = LOB_SUBOPER_ERROR;
+ break;
- return err;
-}
+ case Lob_Update:
+ if ((subOperation == Lob_Memory)||(subOperation == Lob_Buffer))
+ err = lobPtr->update(source, sourceLen, subOperation, descNumIn, retOperLen, lobMaxSize, lobMaxChunkMemSize,lobGCLimit,handleIn,handleInLen,handleOut,handleOutLen,lobGlobals);
+ else if (subOperation == Lob_File)
+ err = lobPtr->update(source, -1, subOperation,descNumIn, retOperLen,lobMaxSize, lobMaxChunkMemSize,lobGCLimit,handleIn,handleInLen,handleOut,handleOutLen,lobGlobals);
+ else
+ err = LOB_SUBOPER_ERROR;
+ break;
-// called once per process
-Ex_Lob_Error ExLobGlobals::initialize()
-{
- Ex_Lob_Error err = LOB_OPER_OK;
+ case Lob_Delete:
+ err = lobPtr->delDesc(handleIn, handleInLen,transId);
+ break;
- lobMap_ = (lobMap_t *) new (getHeap())lobMap_t; // Leaving this allocated from system heap. Since this class contains hdfsFS unable to derive from LOB heap
- if (lobMap_ == NULL)
- return LOB_INIT_ERROR;
+ case Lob_Drop:
+ err = lobPtr->purgeLob();
+ it = lobMap->find(string(lobName));
+ lobMap->erase(it);
+ delete lobPtr;
+ lobPtr = NULL;
+ break;
-
- err = setServerPhandle();
-
+ case Lob_Purge:
+ err = lobPtr->purgeLob();
+ it = lobMap->find(string(lobName));
+ lobMap->erase(it);
+ delete lobPtr;
+ lobPtr = NULL;
+ break;
- // start the worker threads
- startWorkerThreads();
- return err;
-}
+ case Lob_Stats:
+ err = lobPtr->readStats(source);
+ lobPtr->initStats(); // because file may remain open across cursors
+ break;
-static void *workerThreadMain(void *arg)
-{
- // parameter passed to the thread is an instance of the ExLobHdfs object
- ExLobGlobals *glob = (ExLobGlobals *)arg;
+ case Lob_Empty_Directory:
+ lobPtr->initialize(fileName, EX_LOB_RW,
+ dir, storage, hdfsServer, hdfsPort, dir,bufferSize, replication, blockSize);
+ err = lobPtr->emptyDirectory();
+ break;
- glob->doWorkInThread();
+ case Lob_Cleanup:
+ delete lobGlobals;
+ break;
+ case Lob_PerformGC:
+ err = lobPtr->compactLobDataFile((ExLobInMemoryDescChunksEntry *)source,sourceLen);
+ break;
+ case Lob_RestoreLobDataFile:
+ err = lobPtr->restoreLobDataFile();
+ break;
+ case Lob_PurgeBackupLobDataFile:
+ err = lobPtr->purgeBackupLobDataFile();
+ break;
+ default:
+ err = LOB_OPER_ERROR;
+ break;
+ }
+ /*
+//**Note ** This code is needed to reinstate the master transaction after
+// returning from the mxlobsrvr process. This is inactive code for now
+if (!lobGlobals->isHive() )
+ {
+ if (lobPtr)
+ // set the pass thru request object values from the lob
+ lobPtr->getRequest()->getValues(descNumOut, handleOutLen, handleOut,
+ requestStatus, cliError,
+ (char *)blackBox, blackBoxLen); // reinstate the transaction
+ if (TRANSID_IS_VALID(transIdBig)) {
+ ms_transid_reinstate(transIdBig, transStartId);
+ }
+ }
- return NULL;
+ */
+ clock_gettime(CLOCK_MONOTONIC, &endTime);
+
+ secs = endTime.tv_sec - startTime.tv_sec;
+ nsecs = endTime.tv_nsec - startTime.tv_nsec;
+ if (nsecs < 0) {
+ secs--;
+ nsecs += NUM_NSECS_IN_SEC;
+ }
+ totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
+ if (lobPtr && lobPtr->getStats())
+ lobPtr->getStats()->hdfsAccessLayerTime += totalnsecs;
+
+ return err;
}
-Ex_Lob_Error ExLobGlobals::startWorkerThreads()
-{
- int rc;
- for (int i=0; i<NUM_WORKER_THREADS; i++) {
- rc = pthread_create(&threadId_[i], NULL, workerThreadMain, this);
- if (rc != 0)
- return LOB_HDFS_THREAD_CREATE_ERROR;
- }
-
- return LOB_OPER_OK;
+void cleanupLOBDataDescFiles(const char *lobHdfsServer,int lobHdfsPort,const char *lobHdfsLoc)
+{
+ int numExistingFiles=0;
+ hdfsFS fs;
+ fs = hdfsConnect(lobHdfsServer, lobHdfsPort);
+ if (fs == NULL)
+ return;
+ // Get this list of all data and desc files in the lob sotrage location
+ hdfsFileInfo *fileInfos = hdfsListDirectory(fs, lobHdfsLoc, &numExistingFiles);
+ if (fileInfos == NULL)
+ return ;
+ //Delete each one in a loop
+ for (int i = 0; i < numExistingFiles; i++)
+ hdfsDelete(fs, fileInfos[i].mName, 0);
+
+ // *Note* : delete the memory allocated by libhdfs for the file info array
+ if (fileInfos)
+ {
+ hdfsFreeFileInfo(fileInfos, numExistingFiles);
+ }
}
-///////////////////////////////////////////////////////////////////////////////
-// ExLobRequest definitions
-///////////////////////////////////////////////////////////////////////////////
-ExLobRequest::ExLobRequest() :
- reqNum_(0),
- descNumIn_(-1),
- descNumOut_(-1),
- handleInLen_(-1),
- handleOutLen_(-1),
- dataOffset_(-1),
- type_(Lob_Req_Invalid),
- storage_(Lob_Invalid_Storage),
- operLen_(-1),
- error_(LOB_INVALID_ERROR_VAL),
- cliError_(-1),
- status_(LOB_INVALID_ERROR_VAL),
- transId_(0)
-{
- TRANSID_SET_NULL(transIdBig_);
-}
+// The following methods are used for hive access
+/*
+Main thread issues an open to open a range of 128 MB and wakes up a
+worker thread. It doesn’t wait.It calls pre open on the next range. This is
+done in method ::readDataCursorSimple.
-void ExLobRequest::setValues(char *descFileName, Int64 descNumIn, Int64 handleInLen,
- char *handleIn, LobsStorage storage, Int64 transId,
- SB_Transid_Type transIdBig,
- SB_Transseq_Type transStartId,
- char *blackBox, Int64 blackBoxLen)
-{
-
- descNumIn_ = descNumIn;
- handleInLen_ = handleInLen;
- storage_ = storage;
- strcpy(descFileName_, descFileName);
- if (handleIn != NULL && handleInLen > 0) {
- memcpy(handleIn_, handleIn, handleInLen);
- }
- cliError_ = -1;
- error_ = LOB_INVALID_ERROR_VAL;
- status_ = LOB_INVALID_ERROR_VAL;
+The worker threads do their work in ::doWorkInThread and ::performRequests, ::readCursorDataSimple.(note the diff from the method the mainthread calls above)
- transId_ = transId;
- transIdBig_ = transIdBig;
- transStartId_ = transStartId;
- blackBoxLen_ = blackBoxLen;
- if (blackBox != NULL && blackBoxLen > 0) {
- memcpy(blackBox_, blackBox, blackBoxLen);
- }
-
-}
+Main thread then issues a read. Since worker thread had already begun fetching
+16KB buffers in (1), the main thread most likely will not need to wait and the
+data will be ready. It keeps consuming the buffers, recycling them back into
+postFetchBufList.
+When done, the main thread closes the cursor(::closeDataCursorSimple). This is determined by whether we
+have reached the end of range or the end of data for that file.
+The worker threads on the other hand read 16KB of data and buffers them in a
+prefetchBufList. It continues doing this until end of range is reached or the
+buffer limit (128MB) has been reached.
+*/
-void ExLobRequest::getValues(Int64 &descNumOut, Int64 &handleOutLen,
- char *handleOut, Ex_Lob_Error &requestStatus,
- Int64 &cliError,
- char *blackBox, Int64 &blackBoxLen)
+Ex_Lob_Error ExLob::readDataCursorSimple(char *file, char *tgt, Int64 tgtSize,
+ Int64 &operLen, ExLobGlobals *lobGlobals)
{
-
- descNumOut = descNumOut_;
- handleOutLen = handleOutLen_;
- requestStatus = error_;
- cliError = cliError_;
- if (handleOut != NULL && handleOutLen_ > 0) {
- memcpy(handleOut, handleOut_, handleOutLen_);
- }
- blackBoxLen = blackBoxLen_;
- if (blackBox != NULL && blackBoxLen_ > 0) {
- memcpy(blackBox, blackBox_, blackBoxLen_);
- }
- // #endif
-}
+ int dataOffset;
+ Ex_Lob_Error result = LOB_OPER_OK;
+ cursor_t *cursor;
+ ExLobCursor::bufferList_t::iterator c_it;
+ ExLobCursorBuffer *buf = NULL;
+ Int64 bytesToCopy = 0;
+ operLen = 0;
+ Int64 len;
+ char *target = tgt;
+ bool done = false;
-ExLobRequest::~ExLobRequest()
-{
-}
+ struct timespec startTime;
+ struct timespec endTime;
-Ex_Lob_Error ExLobRequest::send()
-{
-
+ lobCursorLock_.lock();
- int msgid;
- int oid;
- MS_Result_Type result;
- short req_ctrl[BUFSIZ];
- short rep_ctrl[BUFSIZ];
- char *req_data = (char *)this;
- ExLobRequest rep_data;
- short req_data_len = sizeof(ExLobRequest);
- short rep_data_max = sizeof(ExLobRequest);
- int err=0;
- int inx=0;
- int retries = 3;
+ lobCursors_it it = lobCursors_.find(string(file, strlen(file)));
- incrReqNum();
+ if (it == lobCursors_.end())
+ {
+ lobCursorLock_.unlock();
+ return LOB_CURSOR_NOT_OPEN;
+ }
+ else
+ {
+ cursor = &(it->second);
+ }
- status_ = LOB_OPER_REQ_IN_PROGRESS;
+ lobCursorLock_.unlock();
- do
+ while ((operLen < tgtSize) && !done && !cursor->eol_)
{
- err = BMSG_LINK_(&serverPhandle,
- &msgid,
- req_ctrl,
- (ushort) (inx &1),
- rep_ctrl,
- 1,
- req_data,
- req_data_len,
- (char *)&rep_data,
- rep_data_max,
- 0,0,0,0);
- retries--;
-
- err = BMSG_BREAK_(msgid, (short *) &result, &serverPhandle);
+ lobGlobals->traceMessage("locking cursor",cursor,__LINE__);
+ cursor->lock_.lock();
- if (err == XZFIL_ERR_PATHDOWN) {
- //lobGlobals->resetServerPhandle();
- }
+ // if no buffers to read and is eor or eod, we are done.
+ // else wait for prefetch thread to wake us up.
+ if (cursor->prefetchBufList_.size() == 0) {
+ if (cursor->eor_ || cursor->eod_) {
+ done = true;
+ } else {
+ cursor->bufferMisses_++;
+ lobGlobals->traceMessage("wait on condition cursor",cursor,__LINE__);
+ cursor->lock_.wait();
+ }
+ lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
+ continue;
+ }
- } while ( (err == XZFIL_ERR_PATHDOWN) && (retries > 0) ); // 201 if lobserver got restared
+ // a buffer is available
+ c_it = cursor->prefetchBufList_.begin();
+ buf = *c_it;
+ lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
- status_ = LOB_OPER_REQ_DONE;
+ bytesToCopy = min(buf->bytesRemaining_, tgtSize - operLen);
+ memcpy(target, buf->data_ + buf->bytesUsed_, bytesToCopy);
+ target += bytesToCopy;
+ if (bytesToCopy == buf->bytesRemaining_) { // buffer is now empty
+ buf->bytesRemaining_ = -1;
+ buf->bytesUsed_ = -1;
+ lobGlobals->postfetchBufListLock_.lock();
+ lobGlobals->postfetchBufList_.push_back(buf);
+ lobGlobals->postfetchBufListLock_.unlock();
+ lobGlobals->traceMessage("locking cursor",cursor,__LINE__);
+ cursor->lock_.lock();
+ c_it = cursor->prefetchBufList_.erase(c_it);
+ lobGlobals->traceMessage("signal condition cursor",cursor,__LINE__);
+ cursor->lock_.wakeOne(); // wake up prefetch thread if it was waiting for an empty buffer.
+ lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
+ } else {
+ buf->bytesUsed_ += bytesToCopy;
+ buf->bytesRemaining_ -= bytesToCopy;
+ }
+ stats_.bytesPrefetched += bytesToCopy;
+ operLen += bytesToCopy;
+ }
- if (err != XZFIL_ERR_OK)
- return LOB_SEND_MSG_ERROR;
+ // update stats
+ stats_.bytesRead += operLen;
+ stats_.bytesToRead += tgtSize;
+ stats_.numReadReqs++;
- memcpy(this, &rep_data, rep_data_max);
-
return LOB_OPER_OK;
}
-void ExLobRequest::getDescOut(ExLobDesc &desc)
-{
- memcpy(&desc, &desc_, sizeof(ExLobDesc));
-}
-
-void ExLobRequest::putDescIn(ExLobDesc &desc)
-{
- memcpy(&desc_, &desc, sizeof(ExLobDesc));
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// ExLobHdfs definitions
-///////////////////////////////////////////////////////////////////////////////
-#ifdef SQ_USE_HDFS
-
-ExLobLock::ExLobLock()
- : bellRang_(false),
- waiters_(0)
-{
- pthread_mutexattr_t mutexAttr;
- pthread_mutexattr_init( &mutexAttr );
- pthread_mutex_init( &mutex_, &mutexAttr );
- pthread_cond_init( &workBell_, NULL );
-}
-
-ExLobLock::~ExLobLock()
+Ex_Lob_Error ExLob::closeDataCursorSimple(char *fileName, ExLobGlobals *lobGlobals)
{
- pthread_mutex_unlock( &mutex_ );
- pthread_mutex_destroy(&mutex_);
- pthread_cond_destroy(&workBell_);
-}
+ cursor_t *cursor = NULL;
+ Int64 secs = 0;
+ Int64 nsecs = 0;
-void ExLobLock::lock()
-{
- pthread_mutex_lock( &mutex_ );
-}
+ lobCursorLock_.lock();
-void ExLobLock::unlock()
-{
- pthread_mutex_unlock( &mutex_ );
-}
+ lobCursors_it it = lobCursors_.find(string(fileName, strlen(fileName)));
+ if (it != lobCursors_.end())
+ {
+ cursor = &(it->second);
+ lobGlobals->traceMessage("locking cursor",cursor,__LINE__);
+ cursor->lock_.lock();
-void ExLobLock::wakeOne()
-{
- pthread_cond_signal(&workBell_);
-}
+ clock_gettime(CLOCK_MONOTONIC, &cursor->closeTime_);
+ secs = cursor->closeTime_.tv_sec - cursor->openTime_.tv_sec;
+ nsecs = cursor->closeTime_.tv_nsec - cursor->openTime_.tv_nsec;
-void ExLobLock::wakeAll()
-{
- pthread_cond_broadcast(&workBell_);
-}
+ if (cursor->eod_ || cursor->eor_) { // prefetch thread already done,
+ cursor->emptyPrefetchList(lobGlobals);
+ lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
+ lobCursors_.erase(it); // so erase it here.
+ // no need to unlock as cursor object is gone.
+ } else {
+ cursor->eol_ = true; // prefetch thread will do the eol rituals
+ lobGlobals->traceMessage("signal condition cursor",cursor,__LINE__);
+ cursor->lock_.wakeOne(); // wakeup prefetch thread
+ lobGlobals->traceMessage("unlocking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
+ }
+ }
-void ExLobLock::wait()
-{
- waiters_++;
- pthread_cond_wait(&workBell_, &mutex_);
- waiters_--;
-}
+ lobCursorLock_.unlock();
-ExLobHdfsRequest::ExLobHdfsRequest(LobsHdfsRequestType reqType, hdfsFS fs,
- hdfsFile file, char *buffer, int size) :
- reqType_(reqType),
- fs_(fs),
- file_(file),
- buffer_(buffer),
- size_(size)
-{
- lobPtr_ = 0;
- error_ = LOB_OPER_OK;
-}
+ if (nsecs < 0) {
+ secs--;
+ nsecs += NUM_NSECS_IN_SEC;
+ }
+ Int64 totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
+ stats_.cursorElapsedTime += totalnsecs;
-ExLobHdfsRequest::ExLobHdfsRequest(LobsHdfsRequestType reqType, ExLobCursor *cursor) :
- reqType_(reqType),
- cursor_(cursor)
-{
- buffer_=0;
- lobPtr_=0;
- fs_=0;
- file_=0;
- size_=0;
- error_=LOB_OPER_OK;
+ return LOB_OPER_OK;
}
-ExLobHdfsRequest::ExLobHdfsRequest(LobsHdfsRequestType reqType, ExLob *lobPtr, ExLobCursor *cursor) :
- reqType_(reqType),
- lobPtr_(lobPtr),
- cursor_(cursor)
-{
- buffer_=0;
- fs_=0;
- file_=0;
- size_=0;
- error_=LOB_OPER_OK;
-}
-ExLobHdfsRequest::ExLobHdfsRequest(LobsHdfsRequestType reqType) :
- reqType_(reqType)
+Ex_Lob_Error ExLobGlobals::performRequest(ExLobHdfsRequest *request)
{
+ Ex_Lob_Error err = LOB_OPER_OK;
+ ExLob *lobPtr;
+ ExLobCursorBuffer *buf;
+ ExLobCursor *cursor;
+ Int64 size;
+ NABoolean seenEOR = false;
+ NABoolean seenEOD = false;
+ ExLobCursor::bufferList_t::iterator c_it;
+ Int64 totalBufSize;
- buffer_=0;
- cursor_=0;
- lobPtr_=0;
- fs_=0;
- file_=0;
- size_=0;
- error_=LOB_OPER_OK;
-}
-
-ExLobHdfsRequest::~ExLobHdfsRequest()
-{
+ switch (request->reqType_)
+ {
+ case Lob_Hdfs_Cursor_Prefetch :
+ lobPtr = request->lobPtr_;
+ cursor = request->cursor_;
+ traceMessage("locking cursor",cursor,__LINE__);
+ cursor->lock_.lock();
+ while (!cursor->eod_ && !cursor->eor_ && !cursor->eol_)
+ {
+ postfetchBufListLock_.lock();
+ c_it = postfetchBufList_.begin();
+ if (c_it != postfetchBufList_.end()) {
+ buf = *c_it;
+ postfetchBufList_.erase(c_it);
+ postfetchBufListLock_.unlock();
+ traceMessage("unlocking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
+ } else {
+ postfetchBufListLock_.unlock();
+ // there are no empty buffers.
+ // if prefetch list already has the max, wait for one to free up.
+ totalBufSize = cursor->prefetchBufList_.size() * cursor->bufMaxSize_;
+ if (totalBufSize > LOB_CURSOR_PREFETCH_BYTES_MAX) {
+ traceMessage("wait on condition cursor",cursor,__LINE__);
+ cursor->lock_.wait();
+ char buffer2[2048];
+ sprintf(buffer2, "cursor->eod_ %d cursor->eor_ %d "
+ "cursor->eol_ %d", cursor->eod_,
+ cursor->eor_, cursor->eol_);
+ traceMessage(buffer2, cursor, __LINE__);
+ continue;
+ }
+ // create a new buffer
+ traceMessage("unlocking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
+ buf = new (getHeap()) ExLobCursorBuffer();
+ buf->data_ = (char *) (getHeap())->allocateMemory( cursor->bufMaxSize_);
+ lobPtr->stats_.buffersUsed++;
+ }
+ size = min(cursor->bufMaxSize_, (cursor->maxBytes_ - cursor->bytesRead_ + (16 * 1024)));
+ if (buf->data_) {
+ lobPtr->readCursorDataSimple(buf->data_, size, *cursor, buf->bytesRemaining_);
+ buf->bytesUsed_ = 0;
+ traceMessage("locking cursor",cursor,__LINE__);
+ cursor->lock_.lock();
+ if (size < (cursor->bufMaxSize_)) {
+ cursor->eor_ = true;
+ seenEOR = true;
+ }
+ if (buf->bytesRemaining_) {
+ cursor->prefetchBufList_.push_back(buf);
+ traceMessage("signal condition cursor",cursor,__LINE__);
+ cursor->lock_.wakeOne();
+ traceMessage("unlocking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
+ } else {
+ cursor->eod_ = true;
+ seenEOD = true;
+ traceMessage("signal condition cursor",cursor,__LINE__);
+ cursor->lock_.wakeOne();
+ traceMessage("unlocking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
+ postfetchBufListLock_.lock();
+ postfetchBufList_.push_back(buf);
+ postfetchBufListLock_.unlock();
+ }
+ } else {
+ assert("data_ is null");
+ }
+ // Important! Break and do not access cursor object if we have reached
+ // end of data or range.
+ // The main thread could have destroyed the cursor
+ // in ::closeDataCursorSimple
+ if (seenEOD || seenEOR)
+ {
+ char buffer2[2048];
+ sprintf(buffer2, "seenEOD %d seenEOR %d",
+ seenEOD, seenEOR);
+ traceMessage(buffer2, cursor, __LINE__);
+ break;
+ }
+ traceMessage("locking cursor",cursor,__LINE__);
+ cursor->lock_.lock();
+ } // while
+
+ if (!seenEOD && !seenEOR)
+ {
+ traceMessage("locking cursor",cursor,__LINE__);
+ cursor->lock_.unlock();
+ if (cursor->eol_) { // never reaches here ??
+ lobPtr->deleteCursor(cursor->name_, this);
+ }
+ }
+ processPreOpens();
+ break;
+
+ default:
+ request->error_ = LOB_HDFS_REQUEST_UNKNOWN;
+ }
+
+ return LOB_OPER_OK;
}
-Ex_Lob_Error ExLobGlobals::enqueueRequest(ExLobHdfsRequest *request)
+
+
+Ex_Lob_Error ExLob::readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen)
{
- char buffer2[2048];
- sprintf(buffer2, "enqueue request %d", request->reqType_);
- traceMessage(buffer2, NULL, __LINE__);
- reqQueueLock_.lock();
- reqQueue_.push_back(request);
- reqQueueLock_.wakeOne();
- reqQueueLock_.unlock();
+ ExLobDesc desc;
+ Ex_Lob_Error err;
+ Int64 bytesAvailable = 0;
+ Int64 bytesToCopy = 0;
+ Int64 bytesRead = 0;
+ operLen = 0;
+ tOffset offset;
+ struct timespec startTime;
+ struct timespec endTime;
+ bool done = false;
- return LOB_OPER_OK;
-}
+ if (!fdData_) {
+ return LOB_CURSOR_NOT_OPEN_ERROR;
+ }
-Ex_Lob_Error ExLobGlobals::enqueuePrefetchRequest(ExLob *lobPtr, ExLobCursor *cursor)
-{// Leaving this allocated from system heap. Since this class contains hdfsFS unable to derive from LOB heap
- ExLobHdfsRequest *request = new ExLobHdfsRequest(Lob_Hdfs_Cursor_Prefetch, lobPtr, cursor);
-
- if (!request) {
- // return error
+ if (cursor.bytesRead_ == -1) { // starting
+ cursor.bytesRead_ = 0;
}
- enqueueRequest(request);
+ clock_gettime(
<TRUNCATED>
[10/11] incubator-trafodion git commit: Removing change to ignore
error on drop. This causes inconsistencies in some instances like "drop
schema cascade".Will rework with JIRA-1883.
Posted by sa...@apache.org.
Removing change to ignore error on drop. This causes inconsistencies in some instances like "drop schema cascade".Will rework with JIRA-1883.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/0ba0df93
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/0ba0df93
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/0ba0df93
Branch: refs/heads/master
Commit: 0ba0df936ecd881a81aeea5bc31bd7643ed8ec11
Parents: f3cbc3c
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Wed Mar 9 00:47:25 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Wed Mar 9 00:47:25 2016 +0000
----------------------------------------------------------------------
core/sql/cli/Cli.cpp | 18 ++----------------
1 file changed, 2 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/0ba0df93/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index 145b42f..82f1f19 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -9572,8 +9572,7 @@ Lng32 SQLCLI_LOBcliInterface
cliRC = cliInterface->executeImmediate(query);
currContext.resetSqlParserFlags(0x1);
- if (cliRC == -CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION)
- cliRC = 0;
+
if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(myDiags);
@@ -9591,8 +9590,6 @@ Lng32 SQLCLI_LOBcliInterface
currContext.resetSqlParserFlags(0x1);
- if (cliRC == -CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION)
- cliRC = 0;
if (cliRC < 0)
{
@@ -9622,11 +9619,6 @@ Lng32 SQLCLI_LOBcliInterface
if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(myDiags);
- if (myDiags->containsError(-CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION))
- {
- cliRC = 0;
- break;
- }
goto error_return;
}
@@ -9643,11 +9635,6 @@ Lng32 SQLCLI_LOBcliInterface
if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(myDiags);
- if (myDiags->containsError(-CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION))
- {
- cliRC = 0;
- break;
- }
goto error_return;
}
cliRC = 0;
@@ -10685,8 +10672,7 @@ Lng32 SQLCLI_LOBddlInterface
cliRC = cliInterface->executeImmediate(query);
currContext.resetSqlParserFlags(0x1);
- if (cliRC == -CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION)
- cliRC = 0;
+
if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(&diags);