You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2018/09/27 18:15:19 UTC
svn commit: r1842136 - in /pig/branches/branch-0.17: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java
src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java
Author: knoguchi
Date: Thu Sep 27 18:15:19 2018
New Revision: 1842136
URL: http://svn.apache.org/viewvc?rev=1842136&view=rev
Log:
PIG-5355: Negative progress report by HBaseTableRecordReader (satishsaley via knoguchi)
Modified:
pig/branches/branch-0.17/CHANGES.txt
pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java
pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java
Modified: pig/branches/branch-0.17/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.17/CHANGES.txt?rev=1842136&r1=1842135&r2=1842136&view=diff
==============================================================================
--- pig/branches/branch-0.17/CHANGES.txt (original)
+++ pig/branches/branch-0.17/CHANGES.txt Thu Sep 27 18:15:19 2018
@@ -28,6 +28,8 @@ OPTIMIZATIONS
Â
BUG FIXES
+PIG-5355: Negative progress report by HBaseTableRecordReader (satishsaley via knoguchi)
+
PIG-5341 PigStorage with -tagFile/-tagPath produces incorrect results with column pruning (knoguchi)
PIG-5299: PartitionFilterOptimizer failing at compile time (knoguchi)
Modified: pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java?rev=1842136&r1=1842135&r2=1842136&view=diff
==============================================================================
--- pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java (original)
+++ pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java Thu Sep 27 18:15:19 2018
@@ -157,7 +157,7 @@ public class PigRecordReader extends Rec
// idx is always one past the current subsplit's true index.
subprogress = (long)(curReader.getProgress() * pigSplit.getLength(idx - 1));
}
- return Math.min(1.0f, (progress + subprogress)/(float)(pigSplit.getLength()));
+ return Math.max(0.0f, Math.min(1.0f, (progress + subprogress)/(float)(pigSplit.getLength())));
}
@Override
Modified: pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java?rev=1842136&r1=1842135&r2=1842136&view=diff
==============================================================================
--- pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java (original)
+++ pig/branches/branch-0.17/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java Thu Sep 27 18:15:19 2018
@@ -118,7 +118,7 @@ public class HBaseTableInputFormat exten
private byte[] startRow_;
private byte[] endRow_;
private transient byte[] currRow_;
-
+ private int maxRowLength;
private BigInteger bigStart_;
private BigInteger bigEnd_;
private BigDecimal bigRange_;
@@ -151,6 +151,7 @@ public class HBaseTableInputFormat exten
bigStart_ = new BigInteger(Bytes.add(prependHeader, startPadded));
bigEnd_ = new BigInteger(Bytes.add(prependHeader, endPadded));
bigRange_ = new BigDecimal(bigEnd_.subtract(bigStart_));
+ maxRowLength = endRow_.length > startRow_.length ? endRow_.length : startRow_.length;
LOG.info("setScan with ranges: " + bigStart_ + " - " + bigEnd_ + " ( " + bigRange_ + ")");
}
@@ -173,11 +174,8 @@ public class HBaseTableInputFormat exten
return 0;
}
byte[] lastPadded = currRow_;
- if (currRow_.length < endRow_.length) {
- lastPadded = Bytes.padTail(currRow_, endRow_.length - currRow_.length);
- }
- if (currRow_.length < startRow_.length) {
- lastPadded = Bytes.padTail(currRow_, startRow_.length - currRow_.length);
+ if(maxRowLength > currRow_.length) {
+ lastPadded = Bytes.padTail(currRow_, maxRowLength - currRow_.length);
}
byte [] prependHeader = {1, 0};
BigInteger bigLastRow = new BigInteger(Bytes.add(prependHeader, lastPadded));