You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2014/11/08 06:54:04 UTC
[1/3] hbase git commit: HBASE-12448 Fix rate reporting in compaction
progress DEBUG logging
Repository: hbase
Updated Branches:
refs/heads/0.98 2a12bac89 -> ab3b26c03
refs/heads/branch-1 0145650cb -> b1f7d7cd3
refs/heads/master 3b8c0769c -> 7657090ad
HBASE-12448 Fix rate reporting in compaction progress DEBUG logging
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7657090a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7657090a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7657090a
Branch: refs/heads/master
Commit: 7657090ad9ab1243041b9bc344a6a44a252f4239
Parents: 3b8c076
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Nov 7 18:36:43 2014 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Nov 7 18:36:43 2014 -0800
----------------------------------------------------------------------
.../regionserver/compactions/Compactor.java | 45 +++++++++++---------
1 file changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7657090a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index 6b07d73..2ddc06a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.util.StringUtils;
/**
@@ -228,43 +229,39 @@ public abstract class Compactor {
*/
protected boolean performCompaction(InternalScanner scanner,
CellSink writer, long smallestReadPoint, boolean cleanSeqId) throws IOException {
- int bytesWritten = 0;
+ long bytesWritten = 0;
+ long bytesWrittenProgress = 0;
// Since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
List<Cell> cells = new ArrayList<Cell>();
- int closeCheckInterval = HStore.getCloseCheckInterval();
- long lastMillis;
+ long closeCheckInterval = HStore.getCloseCheckInterval();
+ long lastMillis = 0;
if (LOG.isDebugEnabled()) {
- lastMillis = System.currentTimeMillis();
- } else {
- lastMillis = 0;
+ lastMillis = EnvironmentEdgeManager.currentTime();
}
+ long now = 0;
boolean hasMore;
do {
hasMore = scanner.next(cells, compactionKVMax);
+ if (LOG.isDebugEnabled()) {
+ now = EnvironmentEdgeManager.currentTime();
+ }
// output to writer:
for (Cell c : cells) {
if (cleanSeqId && c.getSequenceId() <= smallestReadPoint) {
CellUtil.setSequenceId(c, 0);
}
writer.append(c);
+ int len = KeyValueUtil.length(c);
++progress.currentCompactedKVs;
- progress.totalCompactedSize += KeyValueUtil.length(c);
-
+ progress.totalCompactedSize += len;
+ if (LOG.isDebugEnabled()) {
+ bytesWrittenProgress += len;
+ }
// check periodically to see if a system stop is requested
if (closeCheckInterval > 0) {
- bytesWritten += KeyValueUtil.length(c);
+ bytesWritten += len;
if (bytesWritten > closeCheckInterval) {
- // Log the progress of long running compactions every minute if
- // logging at DEBUG level
- if (LOG.isDebugEnabled()) {
- long now = System.currentTimeMillis();
- if ((now - lastMillis) >= 60 * 1000) {
- LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec",
- (bytesWritten / 1024.0) / ((now - lastMillis) / 1000.0)));
- lastMillis = now;
- }
- }
bytesWritten = 0;
if (!store.areWritesEnabled()) {
progress.cancel();
@@ -273,6 +270,16 @@ public abstract class Compactor {
}
}
}
+ // Log the progress of long running compactions every minute if
+ // logging at DEBUG level
+ if (LOG.isDebugEnabled()) {
+ if ((now - lastMillis) >= 60 * 1000) {
+ LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec",
+ (bytesWrittenProgress / 1024.0) / ((now - lastMillis) / 1000.0)));
+ lastMillis = now;
+ bytesWrittenProgress = 0;
+ }
+ }
cells.clear();
} while (hasMore);
progress.complete();
[2/3] hbase git commit: HBASE-12448 Fix rate reporting in compaction
progress DEBUG logging
Posted by ap...@apache.org.
HBASE-12448 Fix rate reporting in compaction progress DEBUG logging
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b1f7d7cd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b1f7d7cd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b1f7d7cd
Branch: refs/heads/branch-1
Commit: b1f7d7cd32d4c1ea1b9207472dfab6ca257aa800
Parents: 0145650
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Nov 7 18:36:44 2014 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Nov 7 18:36:44 2014 -0800
----------------------------------------------------------------------
.../regionserver/compactions/Compactor.java | 45 +++++++++++---------
1 file changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1f7d7cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index 6b07d73..2ddc06a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.util.StringUtils;
/**
@@ -228,43 +229,39 @@ public abstract class Compactor {
*/
protected boolean performCompaction(InternalScanner scanner,
CellSink writer, long smallestReadPoint, boolean cleanSeqId) throws IOException {
- int bytesWritten = 0;
+ long bytesWritten = 0;
+ long bytesWrittenProgress = 0;
// Since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
List<Cell> cells = new ArrayList<Cell>();
- int closeCheckInterval = HStore.getCloseCheckInterval();
- long lastMillis;
+ long closeCheckInterval = HStore.getCloseCheckInterval();
+ long lastMillis = 0;
if (LOG.isDebugEnabled()) {
- lastMillis = System.currentTimeMillis();
- } else {
- lastMillis = 0;
+ lastMillis = EnvironmentEdgeManager.currentTime();
}
+ long now = 0;
boolean hasMore;
do {
hasMore = scanner.next(cells, compactionKVMax);
+ if (LOG.isDebugEnabled()) {
+ now = EnvironmentEdgeManager.currentTime();
+ }
// output to writer:
for (Cell c : cells) {
if (cleanSeqId && c.getSequenceId() <= smallestReadPoint) {
CellUtil.setSequenceId(c, 0);
}
writer.append(c);
+ int len = KeyValueUtil.length(c);
++progress.currentCompactedKVs;
- progress.totalCompactedSize += KeyValueUtil.length(c);
-
+ progress.totalCompactedSize += len;
+ if (LOG.isDebugEnabled()) {
+ bytesWrittenProgress += len;
+ }
// check periodically to see if a system stop is requested
if (closeCheckInterval > 0) {
- bytesWritten += KeyValueUtil.length(c);
+ bytesWritten += len;
if (bytesWritten > closeCheckInterval) {
- // Log the progress of long running compactions every minute if
- // logging at DEBUG level
- if (LOG.isDebugEnabled()) {
- long now = System.currentTimeMillis();
- if ((now - lastMillis) >= 60 * 1000) {
- LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec",
- (bytesWritten / 1024.0) / ((now - lastMillis) / 1000.0)));
- lastMillis = now;
- }
- }
bytesWritten = 0;
if (!store.areWritesEnabled()) {
progress.cancel();
@@ -273,6 +270,16 @@ public abstract class Compactor {
}
}
}
+ // Log the progress of long running compactions every minute if
+ // logging at DEBUG level
+ if (LOG.isDebugEnabled()) {
+ if ((now - lastMillis) >= 60 * 1000) {
+ LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec",
+ (bytesWrittenProgress / 1024.0) / ((now - lastMillis) / 1000.0)));
+ lastMillis = now;
+ bytesWrittenProgress = 0;
+ }
+ }
cells.clear();
} while (hasMore);
progress.complete();
[3/3] hbase git commit: HBASE-12448 Fix rate reporting in compaction
progress DEBUG logging
Posted by ap...@apache.org.
HBASE-12448 Fix rate reporting in compaction progress DEBUG logging
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ab3b26c0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ab3b26c0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ab3b26c0
Branch: refs/heads/0.98
Commit: ab3b26c03de301a33e2090a952a4e9a48d6587f1
Parents: 2a12bac
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Nov 7 18:36:45 2014 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Nov 7 18:36:45 2014 -0800
----------------------------------------------------------------------
.../regionserver/compactions/Compactor.java | 44 ++++++++++++--------
1 file changed, 26 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ab3b26c0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index df44899..ff20f6f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.util.StringUtils;
/**
@@ -206,20 +207,23 @@ public abstract class Compactor {
*/
protected boolean performCompaction(InternalScanner scanner,
CellSink writer, long smallestReadPoint) throws IOException {
- int bytesWritten = 0;
+ long bytesWritten = 0;
+ long bytesWrittenProgress = 0;
// Since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
List<Cell> kvs = new ArrayList<Cell>();
- int closeCheckInterval = HStore.getCloseCheckInterval();
- long lastMillis;
+ long closeCheckInterval = HStore.getCloseCheckInterval();
+ long lastMillis = 0;
if (LOG.isDebugEnabled()) {
- lastMillis = System.currentTimeMillis();
- } else {
- lastMillis = 0;
+ lastMillis = EnvironmentEdgeManager.currentTimeMillis();
}
+ long now = 0;
boolean hasMore;
do {
hasMore = scanner.next(kvs, compactionKVMax);
+ if (LOG.isDebugEnabled()) {
+ now = EnvironmentEdgeManager.currentTimeMillis();
+ }
// output to writer:
for (Cell c : kvs) {
KeyValue kv = KeyValueUtil.ensureKeyValue(c);
@@ -227,23 +231,17 @@ public abstract class Compactor {
kv.setMvccVersion(0);
}
writer.append(kv);
+ int len = kv.getLength();
++progress.currentCompactedKVs;
- progress.totalCompactedSize += kv.getLength();
+ progress.totalCompactedSize += len;
+ if (LOG.isDebugEnabled()) {
+ bytesWrittenProgress += len;
+ }
// check periodically to see if a system stop is requested
if (closeCheckInterval > 0) {
- bytesWritten += kv.getLength();
+ bytesWritten += len;
if (bytesWritten > closeCheckInterval) {
- // Log the progress of long running compactions every minute if
- // logging at DEBUG level
- if (LOG.isDebugEnabled()) {
- long now = System.currentTimeMillis();
- if ((now - lastMillis) >= 60 * 1000) {
- LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec",
- (bytesWritten / 1024.0) / ((now - lastMillis) / 1000.0)));
- lastMillis = now;
- }
- }
bytesWritten = 0;
if (!store.areWritesEnabled()) {
progress.cancel();
@@ -252,6 +250,16 @@ public abstract class Compactor {
}
}
}
+ // Log the progress of long running compactions every minute if
+ // logging at DEBUG level
+ if (LOG.isDebugEnabled()) {
+ if ((now - lastMillis) >= 60 * 1000) {
+ LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec",
+ (bytesWrittenProgress / 1024.0) / ((now - lastMillis) / 1000.0)));
+ lastMillis = now;
+ bytesWrittenProgress = 0;
+ }
+ }
kvs.clear();
} while (hasMore);
progress.complete();