You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/05/31 22:29:01 UTC
[1/3] hbase git commit: Revert "HBASE-18116 fix replication source
in-memory calculation by excluding bulk load file"
Repository: hbase
Updated Branches:
refs/heads/branch-1 dcb5b5a01 -> 08b7ed165
refs/heads/branch-2 050fae501 -> b22409d51
refs/heads/master 6f3f34227 -> da3ecf1f1
Revert "HBASE-18116 fix replication source in-memory calculation by excluding bulk load file"
This reverts commit 6f3f34227e83dd1811a06066d1d2bb913299a297.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da3ecf1f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da3ecf1f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da3ecf1f
Branch: refs/heads/master
Commit: da3ecf1f13e373e64cb55b85d836c8cc509ee915
Parents: 6f3f342
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu May 31 15:28:28 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu May 31 15:28:28 2018 -0700
----------------------------------------------------------------------
.../ReplicationSourceWALReader.java | 22 +++++---------------
1 file changed, 5 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/da3ecf1f/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
index ee54931..64fd48d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.WAL.Entry;
import org.apache.hadoop.hbase.wal.WALEdit;
-import org.apache.hadoop.hbase.wal.WALKey;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
@@ -169,12 +168,10 @@ class ReplicationSourceWALReader extends Thread {
if (edit == null || edit.isEmpty()) {
return false;
}
- long entrySize = getEntrySizeIncludeBulkLoad(entry);
- long entrySizeExlucdeBulkLoad = getEntrySizeExcludeBulkLoad(entry);
+ long entrySize = getEntrySize(entry);
batch.addEntry(entry);
updateBatchStats(batch, entry, entrySize);
- boolean totalBufferTooLarge = acquireBufferQuota(entrySizeExlucdeBulkLoad);
-
+ boolean totalBufferTooLarge = acquireBufferQuota(entrySize);
// Stop if too many entries or too big
return totalBufferTooLarge || batch.getHeapSize() >= replicationBatchSizeCapacity ||
batch.getNbEntries() >= replicationBatchCountCapacity;
@@ -299,20 +296,11 @@ class ReplicationSourceWALReader extends Thread {
return entryBatchQueue.take();
}
- private long getEntrySizeIncludeBulkLoad(Entry entry) {
+ private long getEntrySize(Entry entry) {
WALEdit edit = entry.getEdit();
- WALKey key = entry.getKey();
- return edit.heapSize() + sizeOfStoreFilesIncludeBulkLoad(edit) +
- key.estimatedSerializedSizeOf();
+ return edit.heapSize() + calculateTotalSizeOfStoreFiles(edit);
}
- private long getEntrySizeExcludeBulkLoad(Entry entry) {
- WALEdit edit = entry.getEdit();
- WALKey key = entry.getKey();
- return edit.heapSize() + key.estimatedSerializedSizeOf();
- }
-
-
private void updateBatchStats(WALEntryBatch batch, Entry entry, long entrySize) {
WALEdit edit = entry.getEdit();
batch.incrementHeapSize(entrySize);
@@ -365,7 +353,7 @@ class ReplicationSourceWALReader extends Thread {
* @param edit edit to count row keys from
* @return the total size of the store files
*/
- private int sizeOfStoreFilesIncludeBulkLoad(WALEdit edit) {
+ private int calculateTotalSizeOfStoreFiles(WALEdit edit) {
List<Cell> cells = edit.getCells();
int totalStoreFilesSize = 0;
[2/3] hbase git commit: Revert "HBASE-18116 fix replication source
in-memory calculation by excluding bulk load file"
Posted by ap...@apache.org.
Revert "HBASE-18116 fix replication source in-memory calculation by excluding bulk load file"
This reverts commit 050fae501a6db95260356279e071c8577e87624f.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b22409d5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b22409d5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b22409d5
Branch: refs/heads/branch-2
Commit: b22409d51d170a5ccf0c1a29095d1442cb6db708
Parents: 050fae5
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu May 31 15:28:37 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu May 31 15:28:37 2018 -0700
----------------------------------------------------------------------
.../ReplicationSourceWALReader.java | 22 +++++---------------
1 file changed, 5 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b22409d5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
index ee54931..64fd48d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.WAL.Entry;
import org.apache.hadoop.hbase.wal.WALEdit;
-import org.apache.hadoop.hbase.wal.WALKey;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
@@ -169,12 +168,10 @@ class ReplicationSourceWALReader extends Thread {
if (edit == null || edit.isEmpty()) {
return false;
}
- long entrySize = getEntrySizeIncludeBulkLoad(entry);
- long entrySizeExlucdeBulkLoad = getEntrySizeExcludeBulkLoad(entry);
+ long entrySize = getEntrySize(entry);
batch.addEntry(entry);
updateBatchStats(batch, entry, entrySize);
- boolean totalBufferTooLarge = acquireBufferQuota(entrySizeExlucdeBulkLoad);
-
+ boolean totalBufferTooLarge = acquireBufferQuota(entrySize);
// Stop if too many entries or too big
return totalBufferTooLarge || batch.getHeapSize() >= replicationBatchSizeCapacity ||
batch.getNbEntries() >= replicationBatchCountCapacity;
@@ -299,20 +296,11 @@ class ReplicationSourceWALReader extends Thread {
return entryBatchQueue.take();
}
- private long getEntrySizeIncludeBulkLoad(Entry entry) {
+ private long getEntrySize(Entry entry) {
WALEdit edit = entry.getEdit();
- WALKey key = entry.getKey();
- return edit.heapSize() + sizeOfStoreFilesIncludeBulkLoad(edit) +
- key.estimatedSerializedSizeOf();
+ return edit.heapSize() + calculateTotalSizeOfStoreFiles(edit);
}
- private long getEntrySizeExcludeBulkLoad(Entry entry) {
- WALEdit edit = entry.getEdit();
- WALKey key = entry.getKey();
- return edit.heapSize() + key.estimatedSerializedSizeOf();
- }
-
-
private void updateBatchStats(WALEntryBatch batch, Entry entry, long entrySize) {
WALEdit edit = entry.getEdit();
batch.incrementHeapSize(entrySize);
@@ -365,7 +353,7 @@ class ReplicationSourceWALReader extends Thread {
* @param edit edit to count row keys from
* @return the total size of the store files
*/
- private int sizeOfStoreFilesIncludeBulkLoad(WALEdit edit) {
+ private int calculateTotalSizeOfStoreFiles(WALEdit edit) {
List<Cell> cells = edit.getCells();
int totalStoreFilesSize = 0;
[3/3] hbase git commit: Revert "HBASE-18116 fix replication source
in-memory calculation by excluding bulk load file"
Posted by ap...@apache.org.
Revert "HBASE-18116 fix replication source in-memory calculation by excluding bulk load file"
This reverts commit dcb5b5a0124d4204c979d13b84c60f7de1ab4929.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/08b7ed16
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/08b7ed16
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/08b7ed16
Branch: refs/heads/branch-1
Commit: 08b7ed16566aa5b979fba98f9299e9d831f49277
Parents: dcb5b5a
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu May 31 15:28:50 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu May 31 15:28:50 2018 -0700
----------------------------------------------------------------------
.../ReplicationSourceWALReaderThread.java | 22 +++++++-------------
1 file changed, 7 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/08b7ed16/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReaderThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReaderThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReaderThread.java
index ac70c33..306ba8f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReaderThread.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReaderThread.java
@@ -39,16 +39,17 @@ import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.WALEntryFilter;
import org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.WALEntryStreamRuntimeException;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.WAL.Entry;
-import org.apache.hadoop.hbase.wal.WALKey;
/**
* Reads and filters WAL entries, groups the filtered entries into batches, and puts the batches onto a queue
@@ -144,11 +145,10 @@ public class ReplicationSourceWALReaderThread extends Thread {
if (entry != null) {
WALEdit edit = entry.getEdit();
if (edit != null && !edit.isEmpty()) {
- long entrySize = getEntrySizeIncludeBulkLoad(entry);
- long entrySizeExlucdeBulkLoad = getEntrySizeExcludeBulkLoad(entry);
+ long entrySize = getEntrySize(entry);
batch.addEntry(entry);
updateBatchStats(batch, entry, entryStream.getPosition(), entrySize);
- boolean totalBufferTooLarge = acquireBufferQuota(entrySizeExlucdeBulkLoad);
+ boolean totalBufferTooLarge = acquireBufferQuota(entrySize);
// Stop if too many entries or too big
if (totalBufferTooLarge || batch.getHeapSize() >= replicationBatchSizeCapacity
|| batch.getNbEntries() >= replicationBatchCountCapacity) {
@@ -251,17 +251,9 @@ public class ReplicationSourceWALReaderThread extends Thread {
return entryBatchQueue.take();
}
- private long getEntrySizeIncludeBulkLoad(Entry entry) {
+ private long getEntrySize(Entry entry) {
WALEdit edit = entry.getEdit();
- WALKey key = entry.getKey();
- return edit.heapSize() + sizeOfStoreFilesIncludeBulkLoad(edit) +
- key.estimatedSerializedSizeOf();
- }
-
- private long getEntrySizeExcludeBulkLoad(Entry entry) {
- WALEdit edit = entry.getEdit();
- WALKey key = entry.getKey();
- return edit.heapSize() + key.estimatedSerializedSizeOf();
+ return edit.heapSize() + calculateTotalSizeOfStoreFiles(edit);
}
private void updateBatchStats(WALEntryBatch batch, Entry entry, long entryPosition, long entrySize) {
@@ -319,7 +311,7 @@ public class ReplicationSourceWALReaderThread extends Thread {
* @param edit edit to count row keys from
* @return the total size of the store files
*/
- private int sizeOfStoreFilesIncludeBulkLoad(WALEdit edit) {
+ private int calculateTotalSizeOfStoreFiles(WALEdit edit) {
List<Cell> cells = edit.getCells();
int totalStoreFilesSize = 0;