You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ww...@apache.org on 2017/09/26 11:33:50 UTC
hadoop git commit: HDFS-12524. Ozone: Record number of keys scanned
and hinted for getRangeKVs call. Contributed by Weiwei Yang.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 f6d656642 -> 8f4c56537
HDFS-12524. Ozone: Record number of keys scanned and hinted for getRangeKVs call. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f4c5653
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f4c5653
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f4c5653
Branch: refs/heads/HDFS-7240
Commit: 8f4c56537f2671e3e69c9ae83f25b7a3b31bdac7
Parents: f6d6566
Author: Weiwei Yang <ww...@apache.org>
Authored: Tue Sep 26 19:22:18 2017 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Tue Sep 26 19:22:18 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/utils/LevelDBStore.java | 20 +++++++++---
.../apache/hadoop/utils/MetadataKeyFilters.java | 32 ++++++++++++++++++--
.../org/apache/hadoop/utils/RocksDBStore.java | 13 +++++++-
.../apache/hadoop/ozone/TestMetadataStore.java | 2 ++
4 files changed, 59 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4c5653/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
index 8b0d905..4aa26ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
@@ -353,11 +353,23 @@ public class LevelDBStore implements MetadataStore {
if (dbIter != null) {
dbIter.close();
}
- long end = System.currentTimeMillis();
- long timeConsumed = end - start;
if (LOG.isDebugEnabled()) {
- LOG.debug("Time consumed for getRangeKVs() is {},"
- + " result length is {}.", timeConsumed, result.size());
+ if (filters != null) {
+ for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
+ int scanned = filter.getKeysScannedNum();
+ int hinted = filter.getKeysHintedNum();
+ if (scanned > 0 || hinted > 0) {
+ LOG.debug(
+ "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
+ filter.getClass().getSimpleName(), filter.getKeysScannedNum(),
+ filter.getKeysHintedNum());
+ }
+ }
+ }
+ long end = System.currentTimeMillis();
+ long timeConsumed = end - start;
+ LOG.debug("Time consumed for getRangeKVs() is {}ms,"
+ + " result length is {}.", timeConsumed, result.size());
}
}
return result;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4c5653/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
index 1688ece..f98d077 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
@@ -38,6 +38,14 @@ public class MetadataKeyFilters {
* @return true if a certain condition satisfied, return false otherwise.
*/
boolean filterKey(byte[] preKey, byte[] currentKey, byte[] nextKey);
+
+ default int getKeysScannedNum() {
+ return 0;
+ }
+
+ default int getKeysHintedNum() {
+ return 0;
+ }
}
/**
@@ -47,19 +55,37 @@ public class MetadataKeyFilters {
public static class KeyPrefixFilter implements MetadataKeyFilter {
private String keyPrefix = null;
+ private int keysScanned = 0;
+ private int keysHinted = 0;
public KeyPrefixFilter(String keyPrefix) {
this.keyPrefix = keyPrefix;
}
- @Override public boolean filterKey(byte[] preKey, byte[] currentKey,
+ @Override
+ public boolean filterKey(byte[] preKey, byte[] currentKey,
byte[] nextKey) {
+ keysScanned++;
if (Strings.isNullOrEmpty(keyPrefix)) {
return true;
} else {
- return currentKey != null &&
- DFSUtil.bytes2String(currentKey).startsWith(keyPrefix);
+ if (currentKey != null &&
+ DFSUtil.bytes2String(currentKey).startsWith(keyPrefix)) {
+ keysHinted++;
+ return true;
+ }
+ return false;
}
}
+
+ @Override
+ public int getKeysScannedNum() {
+ return keysScanned;
+ }
+
+ @Override
+ public int getKeysHintedNum() {
+ return keysHinted;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4c5653/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
index 56a79a1..54da94c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
@@ -201,11 +201,22 @@ public class RocksDBStore implements MetadataStore {
long end = System.currentTimeMillis();
long timeConsumed = end - start;
if (LOG.isDebugEnabled()) {
+ if (filters != null) {
+ for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
+ int scanned = filter.getKeysScannedNum();
+ int hinted = filter.getKeysHintedNum();
+ if (scanned > 0 || hinted > 0) {
+ LOG.debug(
+ "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
+ filter.getClass().getSimpleName(), filter.getKeysScannedNum(),
+ filter.getKeysHintedNum());
+ }
+ }
+ }
LOG.debug("Time consumed for getRangeKVs() is {}ms,"
+ " result length is {}.", timeConsumed, result.size());
}
}
-
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4c5653/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
index f4757bb..6a91860 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
@@ -265,6 +265,8 @@ public class TestMetadataStore {
Assert.assertTrue(result.stream().allMatch(entry ->
new String(entry.getKey()).startsWith("b")
));
+ Assert.assertEquals(20, filter1.getKeysScannedNum());
+ Assert.assertEquals(10, filter1.getKeysHintedNum());
result = store.getRangeKVs(null, 3, filter1);
Assert.assertEquals(3, result.size());
result = store.getRangeKVs(getBytes("b3"), 1, filter1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org