You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sa...@apache.org on 2016/03/09 19:44:03 UTC
[05/11] incubator-trafodion git commit: Changes for performing GC.
Changes to pass in hdfsServer and port via CQD and remove hardcoding. Changes
to pass in lobLocation via CQD and remove hardcoding. Changes to remove all
mxlobsrvr process functionality
Changes for performing GC.
Changes to pass in hdfsServer and port via CQD and remove hardcoding.
Changes to pass in lobLocation via CQD and remove hardcoding.
Changes to remove all mxlobsrvr process functionality from ExLOBprocess.cpp.
Retaining only the infrasturcture so mxlobsrvr process can be used in the
future. The executable is still built but the process is not started as part of
Trafodion anymore. We may use it in the future.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/c29103fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/c29103fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/c29103fd
Branch: refs/heads/master
Commit: c29103fdd67342631bd5b0b3c01ab8295e46a1d8
Parents: 57ae601
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Mon Feb 29 06:12:17 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Mon Feb 29 06:12:17 2016 +0000
----------------------------------------------------------------------
core/sqf/conf/log4j.dtm.config | 2 +-
core/sqf/conf/log4j.hdfs.config | 4 +-
core/sqf/sql/scripts/gensq.pl | 17 -
core/sqf/sql/scripts/sqstart | 3 +-
core/sqf/sql/scripts/sqstop | 2 +-
core/sql/cli/Cli.cpp | 558 +++--
core/sql/cli/Cli.h | 15 +-
core/sql/cli/CliExpExchange.cpp | 16 +-
core/sql/cli/CliExtern.cpp | 54 +-
core/sql/common/CharType.cpp | 1 +
core/sql/common/CharType.h | 2 +-
core/sql/executor/ExExeUtilLoad.cpp | 3 +-
core/sql/exp/ExpLOB.cpp | 142 +-
core/sql/exp/ExpLOB.h | 41 +-
core/sql/exp/ExpLOBaccess.cpp | 2920 +++++++++++++-----------
core/sql/exp/ExpLOBaccess.h | 288 ++-
core/sql/exp/ExpLOBenums.h | 5 +-
core/sql/exp/ExpLOBexternal.h | 20 +-
core/sql/exp/ExpLOBinterface.cpp | 291 +--
core/sql/exp/ExpLOBinterface.h | 24 +-
core/sql/exp/ExpLOBprocess.cpp | 507 +---
core/sql/exp/ExpLOBprocess.h | 2 +-
core/sql/generator/GenItemFunc.cpp | 17 +-
core/sql/optimizer/ItemFunc.h | 17 +-
core/sql/optimizer/NATable.cpp | 5 +-
core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 7 +-
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 10 +-
core/sql/sqlcomp/DefaultConstants.h | 1 +
core/sql/sqlcomp/nadefaults.cpp | 12 +-
29 files changed, 2489 insertions(+), 2497 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/conf/log4j.dtm.config
----------------------------------------------------------------------
diff --git a/core/sqf/conf/log4j.dtm.config b/core/sqf/conf/log4j.dtm.config
index 69bea7e..b7fdb16 100644
--- a/core/sqf/conf/log4j.dtm.config
+++ b/core/sqf/conf/log4j.dtm.config
@@ -53,5 +53,5 @@ log4j.appender.hbaseclient.Append=true
log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.hadoop.hbase=ERROR
log4j.logger.org.trafodion.dtm=INFO
-#log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
+log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/conf/log4j.hdfs.config
----------------------------------------------------------------------
diff --git a/core/sqf/conf/log4j.hdfs.config b/core/sqf/conf/log4j.hdfs.config
index 9505331..0a0ac49 100644
--- a/core/sqf/conf/log4j.hdfs.config
+++ b/core/sqf/conf/log4j.hdfs.config
@@ -50,5 +50,5 @@ log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.hadoop.hbase=ERROR
log4j.logger.org.trafodion.sql=ERROR
-#log4j.logger.org.apache.hadoop.hbase=DEBUG
-#log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+log4j.logger.org.apache.hadoop.hbase.client.transactional=TRACE
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/sql/scripts/gensq.pl
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/gensq.pl b/core/sqf/sql/scripts/gensq.pl
index e826c99..ea18311 100755
--- a/core/sqf/sql/scripts/gensq.pl
+++ b/core/sqf/sql/scripts/gensq.pl
@@ -479,24 +479,7 @@ sub genComponentWait {
printScript(1, "fi\n");
}
-sub genLOBConfig {
- # Generate sqconfig.db config for LOB.
- # This allows the process startup daemon (pstartd)
- # to start it up after a node failure.
- for ($i=0; $i < $gdNumNodes; $i++) {
- my $l_progname="mxlobsrvr";
- my $l_procargs="";
- my $l_procname="\$ZLOBSRV$i";
- my $l_procname_config = sprintf('$ZLOBSRV%d', $i);
- my $l_stdout="stdout_\$ZLOBSRV_$i";
- addDbProcData($l_procname_config, "PERSIST_RETRIES", "10,60");
- addDbProcData($l_procname_config, "PERSIST_ZONES", $i);
- addDbPersistProc($l_procname_config, $i, 1);
- addDbProcDef( $ProcessType_Generic, $l_procname_config, $i, $l_progname, $l_stdout, $l_procargs);
- }
-
-}
sub genIdTmSrv {
if ($SQ_IDTMSRV > 0) {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/sql/scripts/sqstart
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/sqstart b/core/sqf/sql/scripts/sqstart
index 7c63eb1..93e49bd 100755
--- a/core/sqf/sql/scripts/sqstart
+++ b/core/sqf/sql/scripts/sqstart
@@ -542,7 +542,8 @@ dcsstart
sleep 10
sqcheck | tee -a $SQMON_LOG
# Start the LOB Service
-lobstart
+#comment out since it's unused right now
+#lobstart
# Start the REST Service
reststart
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sqf/sql/scripts/sqstop
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/sqstop b/core/sqf/sql/scripts/sqstop
index 55cfa84..e4ecbb1 100755
--- a/core/sqf/sql/scripts/sqstop
+++ b/core/sqf/sql/scripts/sqstop
@@ -92,7 +92,7 @@ reststop
dcsstop
# Stop the lob Service
-lobstop
+#lobstop
# Issue the shutdown request
echoLog "Shutting down ($shutdowntype) the SQ environment!"
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index 018bbfa..e762714 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -94,6 +94,7 @@
#include "LmLangManagerJava.h"
#include "LmRoutine.h"
#include "CmpDDLCatErrorCodes.h"
+#include "ExpLOBaccess.h"
#define DISPLAY_DONE_WARNING 1032
extern Lng32 getTotalTcbSpace(char * tdb, char * otherInfo,
@@ -9394,7 +9395,7 @@ Lng32 SQLCLI_LOBcliInterface
/*IN*/ char * inLobHandle,
/*IN*/ Lng32 inLobHandleLen,
/*IN*/ char * blackBox,
- /*INOUT*/ Lng32 *blackBoxLen,
+ /*INOUT*/ Int32 *blackBoxLen,
/*OUT*/ char * outLobHandle,
/*OUT*/ Lng32 * outLobHandleLen,
/*IN*/ LOBcliQueryType qType,
@@ -9403,8 +9404,9 @@ Lng32 SQLCLI_LOBcliInterface
OUT: for select */
/*INOUT*/ Int64 * dataLen, /* length of data.
IN: for insert, out: for select */
- /*OUT*/ Int64 * outDescPartnKey, /* returned after insert and select */
- /*OUT*/ Int64 * outDescSyskey, /* returned after insert and select */
+ /*INOUT*/ Int64 * inoutDescPartnKey, /* returned after insert and select
+
+ /*INOUT*/ Int64 * inoutDescSyskey, /* returned after insert and select
/*INOUT*/ void* *inCliInterface,
/*IN*/ Int64 xnId /* xn id of the parent process, if non-zero */
)
@@ -9459,13 +9461,7 @@ Lng32 SQLCLI_LOBcliInterface
ExpLOBoper::ExpGetLOBDescChunksName(schNameLen, schName, uid, lobNum,
lobDescChunksNameBuf, lobDescChunksNameLen);
- char lobOffsetsNameBuf[1024];
- Lng32 lobOffsetsNameLen = 1024;
- char * lobOffsetsName =
- ExpLOBoper::ExpGetLOBOffsetsName(schNameLen, schName, uid, lobNum,
- lobOffsetsNameBuf, lobOffsetsNameLen);
-
-
+
char * query = new(currContext.exHeap()) char[4096];
if (outLobHandleLen)
@@ -9499,52 +9495,6 @@ Lng32 SQLCLI_LOBcliInterface
SQL_EXEC_Xact(SQLTRANS_INHERIT, NULL);
}
- /* if ((qType == LOB_CLI_CREATE) ||
- (qType == LOB_CLI_DROP) ||
- (qType == LOB_CLI_INSERT) ||
- (qType == LOB_CLI_INSERT_APPEND) ||
- (qType == LOB_CLI_UPDATE_UNIQUE) ||
- (qType == LOB_CLI_DELETE) ||
- (qType == LOB_CLI_PURGEDATA))
- */
- if (0)
- {
- Int64 transId;
- cliRC = GETTRANSID((short *)&transId);
-
- if (xnId == 0) // no xn passed in
- {
- if (cliRC == 75) // no transid present
- {
- strcpy(query, "set transaction autocommit on;");
- cliRC = cliInterface->executeImmediate(query);
-
- xnAutoCommit = TRUE;
- }
- }
- else
- {
- if ((cliRC == 0) && // transid present
- (xnId == transId))
- {
- // Same as the current transaction in progress.
- // do nothing.
- }
- else
- {
- cliRC = JOINTRANSACTION(xnId);
- if (cliRC)
- return -EXE_BEGIN_TRANSACTION_ERROR;
-
- cliRC = GETTRANSID((short *)&transId);
- if ((cliRC) ||
- (xnId != transId))// transid present
- return -EXE_BEGIN_TRANSACTION_ERROR;
-
- xnJoined = TRUE;
- }
- }
- }
switch (qType)
{
@@ -9588,7 +9538,7 @@ Lng32 SQLCLI_LOBcliInterface
}
// create lob descriptor chunks table salted
- str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, descSysKey largeint not null, chunkNum int not null, chunkLen largeint not null, intParam largeint, stringParam varchar(400), primary key(descPartnKey, descSysKey, chunkNum)) salt using 8 partitions",
+ str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, descSysKey largeint not null, chunkNum int not null, chunkLen largeint not null, dataOffset largeint, stringParam varchar(400), primary key(descPartnKey, descSysKey, chunkNum)) salt using 8 partitions",
lobDescChunksName);
@@ -9606,30 +9556,6 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
-
-
-
- // create lob datafile descriptor unaudited table. Use a cqd
- // traf_no_dtm_xn while doing any dml to this table.
- //dataState = 1 - indicates used chunk
- //dataState = 0 - indicates unused chunk
- str_sprintf(query, "create ghost table %s (descPartnKey largeint not null, descSysKey largeint not null, chunkNum int not null, chunkLen largeint not null, offset largeint, dataState int , primary key(descPartnKey, descSysKey, chunkLen)) ",
- lobOffsetsName);
-
-
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
-
- currContext.resetSqlParserFlags(0x1);x
-
- if (cliRC < 0)
- {
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
cliRC = 0;
}
@@ -9674,25 +9600,8 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
- str_sprintf(query, "drop ghost table %s",
- lobOffsetsName);
-
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC == -CAT_OBJECT_DOES_NOT_EXIST_IN_TRAFODION)
- cliRC = 0;
-
- if (cliRC < 0)
- {
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
+
cliRC = 0;
}
@@ -9741,9 +9650,7 @@ Lng32 SQLCLI_LOBcliInterface
break;
}
goto error_return;
- }
-
-
+ }
cliRC = 0;
}
break;
@@ -9802,33 +9709,11 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
- //Insert into the lob offsets table (without transaction)
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
- str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld,1, %Ld, %Ld, 1)",
- lobOffsetsName, descPartnKey, descSyskey,
- (dataLen ? *dataLen : 0),
- (dataOffset ? *dataOffset : 0) );
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
-
- if (outDescPartnKey)
- *outDescPartnKey = descPartnKey;
-
- if (outDescSyskey)
- *outDescSyskey = descSyskey;
+ if (inoutDescSyskey)
+ *inoutDescSyskey = descSyskey;
// update lob handle with the returned values
if (outLobHandle)
@@ -9839,7 +9724,9 @@ Lng32 SQLCLI_LOBcliInterface
if (outLobHandleLen)
*outLobHandleLen = inLobHandleLen;
}
+
+
cliRC = 0;
}
break;
@@ -9915,30 +9802,14 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
- //Insert into the lob offsets table (without transaction)
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
- str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld,%d, %Ld, %Ld, 1)",
- lobOffsetsName, descPartnKey, descSyskey,numChunks
- (dataLen ? *dataLen : 0),
- (dataOffset ? *dataOffset : 0) );
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
-
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
+ if (inoutDescSyskey)
+ *inoutDescSyskey = inDescSyskey;
+ if(blackBoxLen)
+ *blackBoxLen = numChunks;
cliRC = 0;
// if (outDescSyskey)
@@ -10020,52 +9891,7 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
-
- // Update the lob offsets table. Set all the previous chunks to be
- // unused. Insert the new chunk as the new used chunk.
-
-
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
-
- //update old chunks as unused
-
- str_sprintf(query, "update table(ghost table %s) set dataState = 0 where descPartnKey = %Ld and descSyskey = %Ld",lobOffsetsName, descPartnKey, inDescSyskey);
-
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
- cliRC = cliInterface->executeImmediate(query);
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
-
- //Insert new chunk offset and length
- str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld,1, %Ld, %Ld, 1)",
- lobOffsetsName, descPartnKey, descSyskey,
- (dataLen ? *dataLen : 0),
- (dataOffset ? *dataOffset : 0) );
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
-
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
-
+
// update lob handle with the returned values
if (outLobHandle)
{
@@ -10075,7 +9901,11 @@ Lng32 SQLCLI_LOBcliInterface
if (outLobHandleLen)
*outLobHandleLen = inLobHandleLen;
}
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
+ if (inoutDescSyskey)
+ *inoutDescSyskey = inDescSyskey;
cliRC = 0;
}
break;
@@ -10117,30 +9947,7 @@ Lng32 SQLCLI_LOBcliInterface
goto error_return;
}
-
- //Delete from lob offsets table (without transaction)
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn hold;");
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn 'ON';");
- str_sprintf(query, "update table(ghost table %s) set dataState = 0 where descPartnKey = %Ld and descSyskey = %Ld",lobOffsetsName, descPartnKey, inDescSyskey);
- // set parserflags to allow ghost table
- currContext.setSqlParserFlags(0x1);
-
- cliRC = cliInterface->executeImmediate(query);
-
- currContext.resetSqlParserFlags(0x1);
-
- if (cliRC < 0)
- {
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
- cliInterface->retrieveSQLDiagnostics(myDiags);
-
- goto error_return;
- }
- cliRC = cliInterface->executeImmediate("cqd traf_no_dtm_xn reset;");
-
- cliRC = 0;
- // if (outDescSyskey)
- // *outDescSyskey = inDescSyskey;
+
}
break;
@@ -10200,7 +10007,7 @@ Lng32 SQLCLI_LOBcliInterface
}
// This lob has only one chunk. Read and return the single descriptor.
- str_sprintf(query, "select c.chunkLen, c.intParam from table(ghost table %s) h, table(ghost table %s) c where h.descPartnKey = c.descPartnKey and h.syskey = c.descSyskey and h.descPartnKey = %Ld and h.syskey = %Ld and c.chunkNum = h.numChunks for read committed access",
+ str_sprintf(query, "select c.chunkLen, c.dataOffset from table(ghost table %s) h, table(ghost table %s) c where h.descPartnKey = c.descPartnKey and h.syskey = c.descSyskey and h.descPartnKey = %Ld and h.syskey = %Ld and c.chunkNum = h.numChunks for read committed access",
lobDescHandleName, lobDescChunksName,
descPartnKey, inDescSyskey);
@@ -10243,11 +10050,11 @@ Lng32 SQLCLI_LOBcliInterface
if (dataOffset)
str_cpy_all((char*)dataOffset, ptr, len);
- if (outDescPartnKey)
- *outDescPartnKey = descPartnKey;
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
- if (outDescSyskey)
- *outDescSyskey = inDescSyskey;
+ if (inoutDescSyskey)
+ *inoutDescSyskey = inDescSyskey;
}
// else
//cliRC = -100;
@@ -10268,7 +10075,7 @@ Lng32 SQLCLI_LOBcliInterface
case LOB_CLI_SELECT_CURSOR:
{
- str_sprintf(query, "select intParam, chunkLen from table(ghost table %s) where descPartnKey = %Ld and descSyskey = %Ld order by chunkNum for read committed access",
+ str_sprintf(query, "select dataOffset, chunkLen from table(ghost table %s) where descPartnKey = %Ld and descSyskey = %Ld order by chunkNum for read committed access",
lobDescChunksName, descPartnKey, inDescSyskey);
// set parserflags to allow ghost table
@@ -10395,7 +10202,7 @@ Lng32 SQLCLI_LOBcliInterface
//aggregate on chunklen for this lob.
- str_sprintf (query, "select sum(chunklen) from %s where descpartnkey = %Ld and descsyskey = %Ld ", lobDescChunksName, descPartnKey, inDescSyskey );
+ str_sprintf (query, "select sum(chunklen) from %s where descpartnkey = %Ld and descsyskey = %Ld ", lobDescChunksName, descPartnKey, inDescSyskey );
// set parserflags to allow ghost table
currContext.setSqlParserFlags(0x1);
@@ -10403,11 +10210,11 @@ Lng32 SQLCLI_LOBcliInterface
Int64 outlen = 0;Lng32 len = 0;
cliRC = cliInterface->executeImmediate(query,(char *)dataLen, &len, FALSE);
- if (outDescPartnKey)
- *outDescPartnKey = descPartnKey;
+ if (inoutDescPartnKey)
+ *inoutDescPartnKey = descPartnKey;
- if (outDescSyskey)
- *outDescSyskey = inDescSyskey;
+ if (inoutDescSyskey)
+ *inoutDescSyskey = inDescSyskey;
@@ -10424,6 +10231,7 @@ Lng32 SQLCLI_LOBcliInterface
cliRC = saveCliErr;
}
break;
+
} // switch
// normal return. Fall down to deallocate of structures.
@@ -10464,10 +10272,274 @@ Lng32 SQLCLI_LOBcliInterface
else
return 0;
}
+/* The process for GC is as follows:
+1. Check LOB descriptor chunks table (per column) for any holes in the LOB sections.
+2. Create an in memory array that can be used to calculate new offsets.
+3. Update the lob descriptor chunks table to reflect the new offsets
+ if any error, return an error and rollback the user transaction.
+4. Compact the lob data file buy doing hte following :
+ a) Save a copy of the LOB data file.
+ b) Create a temp lob file to copy the sections contiguously from the lob data file to the temp file.
+ If any error return an error and rollback all updates to the lob descriptor chunks table.
+ c) Delete the lob data file.
+ d) Rename the tempfile to original lob data file name.
+ If any error, restore the lob data file from the saved backup in step (a)
+ e) Purge the saved backup.
+*/
+
+
+Lng32 SQLCLI_LOB_GC_Interface
+(
+ /*IN*/ CliGlobals *cliGlobals,
+ /*IN*/ void *lobGlobals, // can be passed or NULL
+ /*IN*/ char * handle,
+ /*IN*/ Lng32 handleLen,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
+ /*IN*/ char *lobLocation,
+ /*IN*/ Int64 lobMaxMemChunkLen // if passed in as 0, will use default value of 1G for the in memory buffer to do compaction.
+ )
+{
+ Lng32 cliRC = 0;
+
+ ContextCli & currContext = *(cliGlobals->currContext());
+ ComDiagsArea & diags = currContext.diags();
+
+ ComDiagsArea * myDiags = ComDiagsArea::allocate(currContext.exHeap());
+
+ ExeCliInterface *cliInterface = NULL;
+ cliInterface = new (currContext.exHeap())
+ ExeCliInterface(currContext.exHeap(),
+ SQLCHARSETCODE_UTF8,
+ &currContext,
+ NULL);
+ Int32 rc = 0;
+ Int16 flags = 0;
+ Lng32 lobType = 1;
+ Lng32 lobNum = 0;
+ Int64 uid = 0;
+ Int64 inDescSyskey, inDescPartnKey;
+ short schNameLen = 0;
+ char schName[512];
+ if (handle)
+ {
+ ExpLOBoper::extractFromLOBhandle(&flags, &lobType, &lobNum, &uid,
+ &inDescSyskey, &inDescPartnKey,
+ &schNameLen, schName,
+ handle);
+ }
+
+ char tgtLobNameBuf[100];
+ char * tgtLobName =
+ ExpLOBoper::ExpGetLOBname(uid, lobNum, tgtLobNameBuf, 100);
+
+ char lobDescHandleNameBuf[1024];
+ Lng32 lobDescHandleNameLen = 1024;
+ char * lobDescHandleName =
+ ExpLOBoper::ExpGetLOBDescHandleName(schNameLen, schName, uid, lobNum,
+ lobDescHandleNameBuf, lobDescHandleNameLen);
+
+ char lobDescChunksNameBuf[1024];
+ Lng32 lobDescChunksNameLen = 1024;
+ char * lobDescChunksName =
+ ExpLOBoper::ExpGetLOBDescChunksName(schNameLen, schName, uid, lobNum,
+ lobDescChunksNameBuf, lobDescChunksNameLen);
+
+
+ char * query = new(currContext.exHeap()) char[4096];
+ //Find how many entries in the descchunks table to allocate
+ //in memory array.
+ str_sprintf(query, "select count(*) from table(ghost table %s) ",
+ lobDescChunksName);
+
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ Lng32 numEntries = 0;
+ Lng32 len;
+ cliRC = cliInterface->executeImmediate(query, (char*)&numEntries, &len, FALSE);
+
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+ {
+ //Allocate an inmemory array of numEntries.
+ ExLobInMemoryDescChunksEntry *dcInMemoryArray = new ExLobInMemoryDescChunksEntry[numEntries];
+ //Read the desc chunks table into memory
+
+ str_sprintf(query, "select dataOffset, descPartnKey,descSyskey,chunkLen,chunkNum from table(ghost table %s) order by dataOffset for read committed access",
+ lobDescChunksName);
+
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->fetchRowsPrologue(query);
+
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+ cliRC = cliInterface->fetch();
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ cliInterface->fetchRowsEpilogue(0);
+
+ goto error_return;
+ }
+
+ short i = 0;
+ Int64 currentOffset = 0;
+ Int64 descPartnKey = 0;
+ Int64 descSyskey = 0;
+ Int64 chunkLen = 0;
+ Int64 chunkNum = 0;
+ while (cliRC != 100)
+ {
+ char * ptr;
+ Lng32 len;
+
+ cliInterface->getPtrAndLen(1, ptr, len);
+ str_cpy_all((char*)¤tOffset, ptr, len);
+
+ cliInterface->getPtrAndLen(2, ptr, len);
+ str_cpy_all((char*)&descPartnKey, ptr, len);
+
+ cliInterface->getPtrAndLen(3, ptr, len);
+ str_cpy_all((char*)&descSyskey, ptr, len);
+
+ cliInterface->getPtrAndLen(4, ptr, len);
+ str_cpy_all((char*)&chunkLen, ptr, len);
+
+ cliInterface->getPtrAndLen(5, ptr, len);
+ str_cpy_all((char*)&chunkNum, ptr, len);
+
+ dcInMemoryArray[i].setCurrentOffset(currentOffset);
+ dcInMemoryArray[i].setNewOffset(currentOffset);
+ dcInMemoryArray[i].setDescPartnKey(descPartnKey);
+ dcInMemoryArray[i].setSyskey(descSyskey);
+ dcInMemoryArray[i].setChunkLen(chunkLen);
+ dcInMemoryArray[i].setChunkNum(chunkNum);
+ cliRC = cliInterface->fetch();
+ i++;
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ cliInterface->fetchRowsEpilogue(0);
+
+ goto error_return;
+ }
+ }
+
+ cliRC = cliInterface->fetchRowsEpilogue(0);
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ goto error_return;
+ }
+
+ // adjust in memory array to calculate holes and new offsets.
+ ExpLOBoper::calculateNewOffsets(dcInMemoryArray,numEntries);
+
+
+ // Update the descChunks table with new offsets
+ //TBD start a separate transaction to manage this.
+ //For now use the transaction associated with the IUD operation
+ //that triggered this GC
+
+ i = 0;
+ while (i < numEntries)
+ {
+ if (dcInMemoryArray[i].getCurrentOffset() == dcInMemoryArray[i].getNewOffset())
+ i++;
+ else
+ {
+ str_sprintf(query, "update table(ghost table %s) set dataOffset=%Ld, chunkLen = %Ld where descPartnKey = %Ld and descSysKey = %Ld",
+ lobDescChunksName,
+ dcInMemoryArray[i].getNewOffset(),
+ dcInMemoryArray[i].getChunkLen(),
+ dcInMemoryArray[i].getDescPartnKey(),
+ dcInMemoryArray[i].getSyskey());
+ // set parserflags to allow ghost table
+ currContext.setSqlParserFlags(0x1);
+
+ cliRC = cliInterface->executeImmediate(query);
+ currContext.resetSqlParserFlags(0x1);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(myDiags);
+
+ //tbd Give warning and rollback transaction and return with warning. For now return error and abort the iud operation itself.
+ goto error_return;
+ }
+ i++;
+ }
+ }
+
+ // Compact into new temp file
+
+
+ rc = ExpLOBoper::compactLobDataFile(lobGlobals,dcInMemoryArray,numEntries,tgtLobName,lobMaxMemChunkLen, (void *)currContext.exHeap(), hdfsServer, hdfsPort,lobLocation);
+
+ if (rc )
+ {
+ cliRC = 9999; // Warning
+ ComDiagsArea * da = &diags;
+ ExRaiseSqlError(currContext.exHeap(), &da,
+ (ExeErrorCode)(8442), NULL, &cliRC ,
+ &rc, NULL, (char*)"Lob GC call",
+ getLobErrStr(rc));
+ // TBD When local transaction support is in
+ // rollback all the updates to the lob desc chunks file too.
+ // return with warning
+ // For now, return error for the IUD operation
+
+ // Restore original data file.
+ Int32 rc2=ExpLOBoper::restoreLobDataFile(lobGlobals,tgtLobName, (void *)currContext.exHeap(),hdfsServer,hdfsPort,lobLocation);
+ // if error restoring, mark this lob as corrupt.
+ goto error_return;
+ }
+ else
+ {
+ //TBD :commit all updates and remove the saved copy of datafile
+ ExpLOBoper::purgeBackupLobDataFile(lobGlobals, tgtLobName,(void *)currContext.exHeap(),hdfsServer,hdfsPort,lobLocation);
+ }
+ }
+ error_return:
+
+ Lng32 tempCliRC = 0;
+ NADELETEBASIC(query, currContext.exHeap());
+
+ if (cliRC < 0)
+ {
+ if (myDiags->getNumber() > 0)
+ {
+ diags.mergeAfter(*myDiags);
+ }
+ return cliRC;
+ }
+ else if (cliRC == 100)
+ return 100;
+ else
+ return 0;
+}
Lng32 SQLCLI_LOBddlInterface
(
- /*IN*/ CliGlobals *cliGlobals,
+/*IN*/ CliGlobals *cliGlobals,
/*IN*/ char * schName,
/*IN*/ Lng32 schNameLen,
/*IN*/ Int64 objectUID,
@@ -10476,8 +10548,11 @@ Lng32 SQLCLI_LOBddlInterface
/*IN*/ short *lobNumList,
/*IN*/ short *lobTypList,
/*IN*/ char* *lobLocList,
+ /*IN*/ char *hdfsServer,
+ /*IN*/ Int32 hdfsPort,
/*IN*/ Int64 lobMaxSize
)
+
{
Lng32 cliRC = 0;
@@ -10507,18 +10582,12 @@ Lng32 SQLCLI_LOBddlInterface
{
// create lob metadata table
- str_sprintf(query, "create ghost table %s (lobnum smallint not null, storagetype smallint not null, location varchar(4096) not null, primary key (lobnum)) location $system",
- // str_sprintf(query, "create ghost table %s (lobnum smallint not null, storagetype smallint not null, location varchar(4096) not null, primary key (lobnum)) no partition",
- lobMDName);
+ str_sprintf(query, "create ghost table %s (lobnum smallint not null, storagetype smallint not null, location varchar(4096) not null, primary key (lobnum)) ",lobMDName);
// set parserflags to allow ghost table
currContext.setSqlParserFlags(0x1);
- cliRC = cliInterface->executeImmediate(query);
-
-
-
-
+ cliRC = cliInterface->executeImmediate(query);
currContext.resetSqlParserFlags(0x1);
if (cliRC < 0)
@@ -10557,8 +10626,8 @@ Lng32 SQLCLI_LOBddlInterface
// create lob data tables
Lng32 rc = ExpLOBoper::createLOB
(NULL, currContext.exHeap(),
- lobLocList[i],
- objectUID, lobNumList[i], lobMaxSize);
+ lobLocList[i], hdfsPort,hdfsServer,
+ objectUID, lobNumList[i],lobMaxSize);
if (rc)
{
@@ -10567,6 +10636,7 @@ Lng32 SQLCLI_LOBddlInterface
ExRaiseSqlError(currContext.exHeap(), &da,
(ExeErrorCode)(8442), NULL, &cliRC ,
&rc, NULL, (char*)"ExpLOBInterfaceCreate",
+
getLobErrStr(rc));
goto error_return;
}
@@ -10631,7 +10701,7 @@ Lng32 SQLCLI_LOBddlInterface
{
Lng32 rc = ExpLOBoper::dropLOB
(NULL, currContext.exHeap(),
- lobLocList[i],
+ lobLocList[i],hdfsPort,hdfsServer,
objectUID, lobNumList[i]);
if (rc)
@@ -10703,7 +10773,7 @@ Lng32 SQLCLI_LOBddlInterface
{
Lng32 rc = ExpLOBoper::dropLOB
(NULL, currContext.exHeap(),
- lobLocList[i],
+ lobLocList[i],hdfsPort, hdfsServer,
objectUID, lobNumList[i]);
if (rc)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/cli/Cli.h
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.h b/core/sql/cli/Cli.h
index 9bffc5c..db96c87 100644
--- a/core/sql/cli/Cli.h
+++ b/core/sql/cli/Cli.h
@@ -895,7 +895,7 @@ Lng32 SQLCLI_LOBcliInterface
/*IN*/ char * inLobHandle,
/*IN*/ Lng32 inLobHandleLen,
/*IN*/ char * blackBox,
- /*IN*/ Lng32* blackBoxLen,
+ /*IN*/ Int32* blackBoxLen,
/*OUT*/ char * outLobHandle,
/*OUT*/ Lng32 * outLobHandleLen,
/*IN*/ LOBcliQueryType qType,
@@ -909,6 +909,17 @@ Lng32 SQLCLI_LOBcliInterface
/*INOUT*/ void* *inCliInterface,
/*IN*/ Int64 xnId /* xn id of the parent process, if non-zero */
);
+Lng32 SQLCLI_LOB_GC_Interface
+(
+ /*IN*/ CliGlobals *cliGlobals,
+ /*IN*/ void *lobGlobals, // can be passed or NULL
+ /*IN*/ char * handle,
+ /*IN*/ Lng32 handleLen,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
+ /*IN*/ char *lobLocation,
+ /*IN*/ Int64 lobMaxMemChunkLen // if passed in as 0, will use default value of 1G for the in memory buffer to do compaction.
+ );
Lng32 SQLCLI_LOBddlInterface
(
@@ -921,6 +932,8 @@ Lng32 SQLCLI_LOBddlInterface
/*IN*/ short *lobNumList,
/*IN*/ short *lobTypList,
/*IN*/ char* *lobLocList,
+ /*IN*/ char *hdfsServer,
+ /*IN*/ Int32 hdfsPort,
/*IN*/ Int64 lobMaxSize
);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/cli/CliExpExchange.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/CliExpExchange.cpp b/core/sql/cli/CliExpExchange.cpp
index 24853a3..f3b1967 100644
--- a/core/sql/cli/CliExpExchange.cpp
+++ b/core/sql/cli/CliExpExchange.cpp
@@ -207,14 +207,15 @@ ex_expr::exp_return_type InputOutputExpr::describeOutput(void * output_desc_,
// Use SQLDESC_CHAR_SET_NAM (one-part name) for charset
- if ( DFS2REC::isAnyCharacter(operand->getDatatype()) ) {
- output_desc->setDescItem(entry, SQLDESC_CHAR_SET_NAM, 0,
- (char*)CharInfo::getCharSetName(operand->getCharSet()));
+ if ( DFS2REC::isAnyCharacter(operand->getDatatype()) )
+ {
+ output_desc->setDescItem(entry, SQLDESC_CHAR_SET_NAM, 0,
+ (char*)CharInfo::getCharSetName(operand->getCharSet()));
- // reset the length for Unicode
+ // reset the length for Unicode
if ( operand->getCharSet() == CharInfo::UNICODE ||
CharInfo::is_NCHAR_MP(operand->getCharSet())
- )
+ )
{
length = operand->getLength()/SQL_DBCHAR_SIZE;
}
@@ -2240,8 +2241,9 @@ ex_expr::exp_return_type InputOutputExpr::describeInput(void * input_desc_,
}
// Use SQLDESC_CHAR_SET_NAM (one-part name) for charset
- if ((dataType >= REC_MIN_CHARACTER) &&
- (dataType <= REC_MAX_CHARACTER)) {
+ if (((dataType >= REC_MIN_CHARACTER) &&
+ (dataType <= REC_MAX_CHARACTER)) )
+ {
input_desc->setDescItem(entry, SQLDESC_CHAR_SET_NAM, 0,
(char*)CharInfo::getCharSetName(operand->getCharSet()));
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/cli/CliExtern.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/CliExtern.cpp b/core/sql/cli/CliExtern.cpp
index 127380f..e5dd796 100644
--- a/core/sql/cli/CliExtern.cpp
+++ b/core/sql/cli/CliExtern.cpp
@@ -7359,7 +7359,7 @@ SQLCLI_LIB_FUNC Lng32 SQL_EXEC_LOBcliInterface
/*IN*/ char * inLobHandle,
/*IN*/ Lng32 inLobHandleLen,
/*IN*/ char * blackBox,
- /*IN*/ Lng32* blackBoxLen,
+ /*IN*/ Int32* blackBoxLen,
/*OUT*/ char * outLobHandle,
/*OUT*/ Lng32 * outLobHandleLen,
/*IN*/ LOBcliQueryType qType,
@@ -7415,6 +7415,51 @@ SQLCLI_LIB_FUNC Lng32 SQL_EXEC_LOBcliInterface
RecordError(NULL, retcode);
return retcode;
}
+Lng32 SQL_EXEC_LOB_GC_Interface
+(
+ /*IN*/ void *lobGlobals, // can be passed or NULL
+ /*IN*/ char * handle,
+ /*IN*/ Lng32 handleLen,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
+ /*IN*/ char *lobLocation,
+ /*IN*/ Int64 lobMaxMemChunkLen // if passed in as 0, will use default value of 1G for the in memory buffer to do compaction.
+ )
+{
+ Lng32 retcode;
+ CLISemaphore *tmpSemaphore;
+ ContextCli *threadContext;
+ CLI_NONPRIV_PROLOGUE(retcode);
+ try
+ {
+ tmpSemaphore = getCliSemaphore(threadContext);
+ tmpSemaphore->get();
+ threadContext->incrNumOfCliCalls();
+ retcode = SQLCLI_LOB_GC_Interface(GetCliGlobals(),
+ lobGlobals,
+ handle,
+ handleLen,
+ hdfsServer,
+ hdfsPort,lobLocation,
+ lobMaxMemChunkLen);
+ }
+ catch(...)
+ {
+ retcode = -CLI_INTERNAL_ERROR;
+#if defined(_THROW_EXCEPTIONS)
+ if (cliWillThrow())
+ {
+ threadContext->decrNumOfCliCalls();
+ tmpSemaphore->release();
+ throw;
+ }
+#endif
+ }
+ threadContext->decrNumOfCliCalls();
+ tmpSemaphore->release();
+ RecordError(NULL, retcode);
+ return retcode;
+ }
Lng32 SQL_EXEC_LOBddlInterface
(
@@ -7426,6 +7471,8 @@ Lng32 SQL_EXEC_LOBddlInterface
/*IN*/ short *lobNumList,
/*IN*/ short *lobTypList,
/*IN*/ char* *lobLocList,
+ /*IN*/ char *hdfsServer,
+ /*IN*/ Int32 hdfsPort,
/*IN */ Int64 lobMaxSize
)
{
@@ -7446,7 +7493,10 @@ Lng32 SQL_EXEC_LOBddlInterface
qType,
lobNumList,
lobTypList,
- lobLocList, lobMaxSize);
+ lobLocList,
+ hdfsServer,
+ hdfsPort,
+ lobMaxSize);
}
catch(...)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/common/CharType.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/CharType.cpp b/core/sql/common/CharType.cpp
index 40d258d..1b21fd1 100644
--- a/core/sql/common/CharType.cpp
+++ b/core/sql/common/CharType.cpp
@@ -1536,6 +1536,7 @@ SQLBlob::SQLBlob(
externalFormat,
extFormatLen)
{
+ setCharSet(CharInfo::DefaultCharSet);
}
NAType *SQLBlob::newCopy(NAMemory* h) const
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/common/CharType.h
----------------------------------------------------------------------
diff --git a/core/sql/common/CharType.h b/core/sql/common/CharType.h
index f76f574..7d0c383 100644
--- a/core/sql/common/CharType.h
+++ b/core/sql/common/CharType.h
@@ -817,7 +817,7 @@ public:
NABoolean allowSQLnull = TRUE,
NABoolean inlineIfPossible = FALSE,
NABoolean externalFormat = FALSE,
- Lng32 extFormatLen = 100);
+ Lng32 extFormatLen = 1024);
SQLClob(const SQLClob & aClob,NAMemory * heap)
:SQLlob(aClob,heap)
{}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index 1db4bee..cc481a2 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -2980,7 +2980,8 @@ short ExExeUtilLobExtractTcb::work()
LOB_CLI_SELECT_UNIQUE,
lobNumList,
lobTypList,
- lobLocList,0);
+ lobLocList,lobTdb().getLobHdfsServer(),
+ lobTdb().getLobHdfsPort(),0);
if (cliRC < 0)
{
getDiagsArea()->mergeAfter(diags);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOB.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.cpp b/core/sql/exp/ExpLOB.cpp
index 00116b6..2b534ee 100644
--- a/core/sql/exp/ExpLOB.cpp
+++ b/core/sql/exp/ExpLOB.cpp
@@ -137,20 +137,7 @@ char * ExpLOBoper::ExpGetLOBDescChunksName(Lng32 schNameLen, char * schName,
return outBuf;
}
-char * ExpLOBoper::ExpGetLOBOffsetsName(Lng32 schNameLen, char * schName,
- Int64 uid, Lng32 num,
- char * outBuf, Lng32 outBufLen)
-{
- if ((outBufLen < 512) ||
- (schNameLen == 0) ||
- (schName == NULL))
- return NULL;
- str_sprintf(outBuf, "%s.\"LOBOffset_%020Ld_%04d\"",
- schName, uid, num);
-
- return outBuf;
-}
char * ExpLOBoper::ExpGetLOBMDName(Lng32 schNameLen, char * schName,
Int64 uid,
@@ -165,8 +152,8 @@ char * ExpLOBoper::ExpGetLOBMDName(Lng32 schNameLen, char * schName,
return outBuf;
}
Lng32 ExpLOBoper::createLOB(void * lobGlob, void * lobHeap,
- char * lobLoc,
- Int64 uid, Lng32 num, Int64 lobMaxSize)
+ char * lobLoc,Int32 hdfsPort,char *hdfsServer,
+ Int64 uid, Lng32 num, Int64 lobMaxSize )
{
char buf[100];
@@ -186,13 +173,114 @@ Lng32 ExpLOBoper::createLOB(void * lobGlob, void * lobHeap,
else
lobGlobL = lobGlob;
- rc = ExpLOBinterfaceCreate(lobGlobL, lobName, lobLoc, Lob_HDFS_File,(char *)"default",lobMaxSize);
+ rc = ExpLOBinterfaceCreate(lobGlobL, lobName, lobLoc, Lob_HDFS_File,hdfsServer,lobMaxSize, hdfsPort);
+
+ return rc;
+}
+void ExpLOBoper::calculateNewOffsets(ExLobInMemoryDescChunksEntry *dcArray, Lng32 numEntries)
+{
+ Int32 i = 0;
+ //Check if there is a hole right up front for the first entry. If so start compacting with the first entry.
+ if (dcArray[0].getCurrentOffset() != 0)
+ {
+ dcArray[0].setNewOffset(0);
+ for (i = 1; i < numEntries; i++)
+ {
+ dcArray[i].setNewOffset(dcArray[i-1].getNewOffset() + dcArray[i-1].getChunkLen());
+ }
+ }
+ else
+ //Look for the first unused section and start compacting from there.
+ {
+ NABoolean done = FALSE;
+ i = 0;
+ Int32 j = 0;
+ while (i < numEntries && !done )
+ {
+ if ((dcArray[i].getCurrentOffset()+dcArray[i].getChunkLen()) !=
+ dcArray[i+1].getCurrentOffset())
+ {
+ j = i+1;
+ while (j < numEntries)
+ {
+ dcArray[j].setNewOffset(dcArray[j-1].getNewOffset()+dcArray[j-1].getChunkLen());
+ j++;
+ }
+ done = TRUE;
+ }
+ i++;
+ }
+ }
+ return ;
+}
+
+Lng32 ExpLOBoper::compactLobDataFile(void *lobGlob,ExLobInMemoryDescChunksEntry *dcArray,Int32 numEntries,char *tgtLobName,Int64 lobMaxChunkMemSize, void *lobHeap, char *hdfsServer, Int32 hdfsPort, char *lobLoc)
+{
+ Int32 rc = 0;
+ void * lobGlobL = NULL;
+ // Call ExeLOBinterface to create the LOB
+ if (lobGlob == NULL)
+ {
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+ }
+ else
+ lobGlobL = lobGlob;
+
+ if (rc)
+ return -1;
+
+ rc = ExpLOBinterfacePerformGC(lobGlobL,tgtLobName, (void *)dcArray, numEntries,hdfsServer,hdfsPort,lobLoc,lobMaxChunkMemSize);
+
+ return rc;
+}
+
+Int32 ExpLOBoper::restoreLobDataFile(void *lobGlob, char *lobName, void *lobHeap, char *hdfsServer, Int32 hdfsPort, char *lobLoc)
+{
+ Int32 rc = 0;
+ void * lobGlobL = NULL;
+ if (lobGlob == NULL)
+ {
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+ }
+ else
+ lobGlobL = lobGlob;
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+
+ rc = ExpLOBinterfaceRestoreLobDataFile(lobGlobL,hdfsServer,hdfsPort,lobLoc,lobName);
+ return rc;
+}
+
+Int32 ExpLOBoper::purgeBackupLobDataFile(void *lobGlob,char *lobName, void *lobHeap, char * hdfsServer, Int32 hdfsPort, char *lobLoc)
+{
+ Int32 rc = 0;
+ void * lobGlobL = NULL;
+ if (lobGlob == NULL)
+ {
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+ }
+ else
+ lobGlobL = lobGlob;
+ rc = initLOBglobal(lobGlobL, lobHeap);
+ if (rc)
+ return -1;
+
+
+ rc = ExpLOBinterfacePurgeBackupLobDataFile(lobGlobL,(char *)hdfsServer,hdfsPort,lobLoc,lobName);
return rc;
}
+
Lng32 ExpLOBoper::dropLOB(void * lobGlob, void * lobHeap,
- char * lobLoc,
+ char * lobLoc,Int32 hdfsPort, char *hdfsServer,
Int64 uid, Lng32 num)
{
char buf[101];
@@ -214,12 +302,13 @@ Lng32 ExpLOBoper::dropLOB(void * lobGlob, void * lobHeap,
lobGlobL = lobGlob;
// Call ExeLOBinterface to drop the LOB
- rc = ExpLOBinterfaceDrop(lobGlobL,(char *)"default", 0, lobName, lobLoc);
+ rc = ExpLOBinterfaceDrop(lobGlobL,hdfsServer, hdfsPort, lobName, lobLoc);
return rc;
}
Lng32 ExpLOBoper::purgedataLOB(void * lobGlob, char * lobLoc,
+
Int64 uid, Lng32 num)
{
char buf[100];
@@ -229,7 +318,7 @@ Lng32 ExpLOBoper::purgedataLOB(void * lobGlob, char * lobLoc,
return -1;
// Call ExeLOBinterface to purgedata the LOB
- Lng32 rc = ExpLOBInterfacePurgedata(lobGlob,(char *)"default", 0, lobName, lobLoc);
+ Lng32 rc = ExpLOBInterfacePurgedata(lobGlob, lobName, lobLoc);
if (rc < 0)
return ex_expr::EXPR_ERROR;
@@ -252,11 +341,12 @@ ExpLOBoper::ExpLOBoper(OperatorTypeEnum oper_type,
lobHandleSaved_[0] = 0;
lobStorageLocation_[0] = 0;
lobHdfsServer_[0] = 0;
- strcpy(lobHdfsServer_,"default");
- lobHdfsPort_ = 0;
+ strcpy(lobHdfsServer_,"");
+ lobHdfsPort_ = -1;
descSchName_[0] = 0;
lobMaxSize_ = 0;
lobMaxChunkMemSize_ = 0;
+ lobGCLimit_ = 0;
};
@@ -725,7 +815,7 @@ ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[],
&cliError,
so,
waitedOp,
- lobData, lobLen, getLobMaxSize(), getLobMaxChunkMemSize());
+ lobData, lobLen, getLobMaxSize(), getLobMaxChunkMemSize(),getLobGCLimit());
if (rc == LOB_ACCESS_PREEMPT)
{
@@ -891,7 +981,9 @@ ex_expr::exp_return_type ExpLOBiud::insertData(Lng32 handleLen,
so,
waitedOp,
lobData,
- lobLen,getLobMaxSize(), getLobMaxChunkMemSize());
+ lobLen,getLobMaxSize(),
+ getLobMaxChunkMemSize(),
+ getLobGCLimit());
}
if (rc == LOB_ACCESS_PREEMPT)
@@ -1231,7 +1323,7 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[],
data,
fromLobName, fromSchNameLen, fromSchName,
fromDescKey, fromDescTS,
- getLobMaxSize(), getLobMaxChunkMemSize());
+ getLobMaxSize(), getLobMaxChunkMemSize(),getLobGCLimit());
}
else
{
@@ -1254,7 +1346,7 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[],
data,
fromLobName, fromSchNameLen, fromSchName,
fromDescKey, fromDescTS,
- getLobMaxSize(), getLobMaxChunkMemSize());
+ getLobMaxSize(), getLobMaxChunkMemSize(),getLobGCLimit());
}
if (rc < 0)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOB.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.h b/core/sql/exp/ExpLOB.h
index 316c259..d71a1d9 100644
--- a/core/sql/exp/ExpLOB.h
+++ b/core/sql/exp/ExpLOB.h
@@ -47,7 +47,7 @@
#define LOB_HANDLE_LEN 1024
-
+class ExLobInMemoryDescChunksEntry;
////////////////////////////////
// class LOBglobals
////////////////////////////////
@@ -225,22 +225,23 @@ public:
Int64 uid, Lng32 lobNum,
char * outBuf, Lng32 outBufLen);
- static char * ExpGetLOBOffsetsName(Lng32 schNameLen, char * schName,
- Int64 uid, Lng32 lobNum,
- char * outBuf, Lng32 outBufLen);
static Lng32 ExpGetLOBnumFromDescName(char * descName, Lng32 descNameLen);
static char * ExpGetLOBMDName(Lng32 schNameLen, char * schName,
Int64 uid,
char * outBuf, Lng32 outBufLen);
+ static void calculateNewOffsets(ExLobInMemoryDescChunksEntry *dcArray, Lng32 numEntries);
+ static Lng32 compactLobDataFile(void *lobGlob, ExLobInMemoryDescChunksEntry *dcArray, Int32 numEntries, char *tgtLobName, Int64 lobMaxChunkSize, void *lobHeap,char *hdfsServer, Int32 hdfsPort,char *lobLocation);
+ static Int32 restoreLobDataFile(void *lobGlob, char *lobName, void *lobHeap, char *hdfsServer, Int32 hdfsPort,char *lobLocation );
+ static Int32 purgeBackupLobDataFile(void *lobGlob,char *lobName, void *lobHeap, char *hdfsServer, Int32 hdfsPort, char *lobLocation);
static Lng32 createLOB(void * lobGlob, void * lobHeap,
- char * lobLoc,
+ char * lobLoc, Int32 hdfsPort, char *hdfsServer,
Int64 uid, Lng32 lobNum, Int64 lobMAxSize);
static Lng32 dropLOB(void * lobGlob, void * lobHeap,
- char * lobLoc,
+ char * lobLoc,Int32 hdfsPort, char *hdfsServer,
Int64 uid, Lng32 lobNum);
static Lng32 purgedataLOB(void * lobGlob,
@@ -303,6 +304,12 @@ public:
Int64 getLobMaxSize() { return lobMaxSize_;}
void setLobMaxChunkMemSize(Int64 maxsize) { lobMaxChunkMemSize_ = maxsize;}
Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_;}
+ void setLobGCLimit(Int64 gclimit) { lobGCLimit_ = gclimit;}
+ Int64 getLobGCLimit() { return lobGCLimit_;}
+ void setLobHdfsServer(char *hdfsServer)
+ {strcpy(lobHdfsServer_,hdfsServer);}
+ void setLobHdfsPort(Int32 hdfsPort)
+ {lobHdfsPort_ = hdfsPort;}
protected:
typedef enum
{
@@ -349,13 +356,14 @@ public:
};
Lng32 checkLobOperStatus();
-
+
protected:
char * descSchName() { return descSchName_; }
char * getLobHdfsServer() { return (strlen(lobHdfsServer_) == 0 ? NULL : lobHdfsServer_); }
Lng32 getLobHdfsPort() { return lobHdfsPort_; }
-
+
+
short flags_; // 00-02
short lobNum_;
@@ -371,7 +379,7 @@ protected:
char lobHandleSaved_[LOB_HANDLE_LEN];
char outLobHandle_[LOB_HANDLE_LEN];
- Int64 outHandleLen_;
+ Int32 outHandleLen_;
char blackBox_[1024];
Int64 blackBoxLen_;
@@ -386,6 +394,7 @@ protected:
Int64 lobMaxSize_;
Int64 lobMaxChunkMemSize_;
+ Int64 lobGCLimit_;
// NABasicPtr lobStorageLocation_;
}
;
@@ -478,6 +487,7 @@ class ExpLOBiud : public ExpLOBoper {
{
(v) ? liudFlags_ |= FROM_EXTERNAL: liudFlags_ &= ~FROM_EXTERNAL;
};
+
protected:
Int64 objectUID_;
@@ -534,18 +544,12 @@ public:
virtual short getClassSize() { return (short)sizeof(*this); }
// ---------------------------------------------------------------------
-
-
+
private:
- // char * descSchName() { return descSchName_; }
-
+
Lng32 liFlags_;
char filler1_[4];
- // Int64 objectUID_;
-
- // short descSchNameLen_;
- // char descSchName_[510];
- // NABasicPtr descSchName_;
+
};
class ExpLOBdelete : public ExpLOBiud {
@@ -705,6 +709,7 @@ public:
{
return tgtLocation_;
}
+
private:
enum
{