You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2015/08/28 00:00:30 UTC
[2/3] incubator-trafodion git commit: [TRAFODION-1453] Changes to
avoid scan for a range delimiter in the buffer when not needed. It is now ok
to encounter the range delimiter in the the column value,
but its behavior is unknown.
[TRAFODION-1453] Changes to avoid scan for a range delimiter in the buffer
when not needed. It is now ok to encounter the range delimiter in the
the column value, but its behavior is unknown.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/d547e1ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/d547e1ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/d547e1ec
Branch: refs/heads/master
Commit: d547e1ec5a8e69011990eec09f0e829c843e2e76
Parents: 5d4e0a9
Author: selvaganesang <se...@esgyn.com>
Authored: Thu Aug 27 21:43:16 2015 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Thu Aug 27 21:43:16 2015 +0000
----------------------------------------------------------------------
core/sql/executor/ExHdfsScan.cpp | 29 +++++++++++++++++++----------
core/sql/executor/ExHdfsScan.h | 9 +++++----
2 files changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d547e1ec/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index d2eac33..71c0118 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -114,6 +114,7 @@ ExHdfsScanTcb::ExHdfsScanTcb(
, hdfo_(NULL)
, numBytesProcessedInRange_(0)
, exception_(FALSE)
+ , checkRangeDelimiter_(FALSE)
{
Space * space = (glob ? glob->getSpace() : 0);
CollHeap * heap = (glob ? glob->getDefaultHeap() : 0);
@@ -388,6 +389,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
beginRangeNum_ = -1;
numRanges_ = -1;
hdfsOffset_ = 0;
+ checkRangeDelimiter_ = FALSE;
if (hdfsScanTdb().getHdfsFileInfoList()->isEmpty())
{
@@ -718,7 +720,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
// Position in the hdfsScanBuffer_ to the
// first record delimiter.
hdfsBufNextRow_ = hdfs_strchr(hdfsScanBuffer_,
- hdfsScanTdb().recordDelimiter_, hdfsScanBuffer_+trailingPrevRead_+ bytesRead_);
+ hdfsScanTdb().recordDelimiter_, hdfsScanBuffer_+trailingPrevRead_+ bytesRead_, checkRangeDelimiter_);
// May be that the record is too long? Or data isn't ascii?
// Or delimiter is incorrect.
if (! hdfsBufNextRow_)
@@ -1104,6 +1106,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
if ((trailingPrevRead_ > 0) &&
(hdfsBufNextRow_[0] == RANGE_DELIMITER))
{
+ checkRangeDelimiter_ = FALSE;
step_ = CLOSE_HDFS_CURSOR;
break;
}
@@ -1392,14 +1395,16 @@ char * ExHdfsScanTcb::extractAndTransformAsciiSourceToSqlRow(int &err,
hdfsLoggingRow_ = hdfsBufNextRow_;
if (asciiSourceTD->numAttrs() == 0)
{
- sourceRowEnd = hdfs_strchr(sourceData, rd, sourceDataEnd);
+ sourceRowEnd = hdfs_strchr(sourceData, rd, sourceDataEnd, checkRangeDelimiter_);
hdfsLoggingRowEnd_ = sourceRowEnd;
if (!sourceRowEnd)
return NULL;
if ((endOfRequestedRange_) &&
- (sourceRowEnd >= endOfRequestedRange_))
+ (sourceRowEnd >= endOfRequestedRange_)) {
+ checkRangeDelimiter_ = TRUE;
*(sourceRowEnd +1)= RANGE_DELIMITER;
+ }
// no columns need to be converted. For e.g. count(*) with no predicate
return sourceRowEnd+1;
@@ -1425,7 +1430,7 @@ char * ExHdfsScanTcb::extractAndTransformAsciiSourceToSqlRow(int &err,
attr = NULL;
if (!isTrailingMissingColumn) {
- sourceColEnd = hdfs_strchr(sourceData, rd, cd, sourceDataEnd, &rdSeen);
+ sourceColEnd = hdfs_strchr(sourceData, rd, cd, sourceDataEnd, checkRangeDelimiter_, &rdSeen);
if (sourceColEnd == NULL) {
if (rdSeen || (sourceRowEnd == NULL))
return NULL;
@@ -1438,8 +1443,10 @@ char * ExHdfsScanTcb::extractAndTransformAsciiSourceToSqlRow(int &err,
sourceRowEnd = sourceColEnd;
hdfsLoggingRowEnd_ = sourceRowEnd;
if ((endOfRequestedRange_) &&
- (sourceRowEnd >= endOfRequestedRange_))
- *(sourceRowEnd +1)= RANGE_DELIMITER;
+ (sourceRowEnd >= endOfRequestedRange_)) {
+ checkRangeDelimiter_ = TRUE;
+ *(sourceRowEnd +1)= RANGE_DELIMITER;
+ }
if (i != hdfsScanTdb().convertSkipListSize_ - 1)
isTrailingMissingColumn = TRUE;
}
@@ -1486,13 +1493,15 @@ char * ExHdfsScanTcb::extractAndTransformAsciiSourceToSqlRow(int &err,
// rowDelimiter is encountered
// So try to find the record delimiter
if (sourceRowEnd == NULL) {
- sourceRowEnd = hdfs_strchr(sourceData, rd, sourceDataEnd);
+ sourceRowEnd = hdfs_strchr(sourceData, rd, sourceDataEnd, checkRangeDelimiter_);
if (sourceRowEnd) {
hdfsLoggingRowEnd_ = sourceRowEnd;
if ((endOfRequestedRange_) &&
- (sourceRowEnd >= endOfRequestedRange_))
- *(sourceRowEnd +1)= RANGE_DELIMITER;
- }
+ (sourceRowEnd >= endOfRequestedRange_ )) {
+ checkRangeDelimiter_ = TRUE;
+ *(sourceRowEnd +1)= RANGE_DELIMITER;
+ }
+ }
}
workAtp_->getTupp(hdfsScanTdb().workAtpIndex_) = hdfsSqlTupp_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d547e1ec/core/sql/executor/ExHdfsScan.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.h b/core/sql/executor/ExHdfsScan.h
index 62f41f7..8ac16d3 100644
--- a/core/sql/executor/ExHdfsScan.h
+++ b/core/sql/executor/ExHdfsScan.h
@@ -282,6 +282,7 @@ protected:
NABoolean exception_;
ComCondition * lastErrorCnd_;
+ NABoolean checkRangeDelimiter_;
};
class ExOrcScanTcb : public ExHdfsScanTcb
@@ -428,14 +429,14 @@ protected:
#define RANGE_DELIMITER '\002'
-inline char *hdfs_strchr(const char *s, int c, const char *end)
+inline char *hdfs_strchr(const char *s, int c, const char *end, NABoolean checkRangeDelimiter)
{
char *curr = (char *)s;
while (curr < end) {
if (*curr == c)
return curr;
- if (*curr == RANGE_DELIMITER)
+ if (checkRangeDelimiter &&*curr == RANGE_DELIMITER)
return NULL;
curr++;
}
@@ -443,7 +444,7 @@ inline char *hdfs_strchr(const char *s, int c, const char *end)
}
-inline char *hdfs_strchr(const char *s, int rd, int cd, const char *end, NABoolean *rdSeen)
+inline char *hdfs_strchr(const char *s, int rd, int cd, const char *end, NABoolean checkRangeDelimiter, NABoolean *rdSeen)
{
char *curr = (char *)s;
@@ -458,7 +459,7 @@ inline char *hdfs_strchr(const char *s, int rd, int cd, const char *end, NABoole
return curr;
}
else
- if (*curr == RANGE_DELIMITER) {
+ if (checkRangeDelimiter && *curr == RANGE_DELIMITER) {
*rdSeen = TRUE;
return NULL;
}