You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by sr...@apache.org on 2018/11/20 21:02:18 UTC
samza git commit: SAMZA-1994: Table API: Add missed key lookups
metric for table reads
Repository: samza
Updated Branches:
refs/heads/master b05845912 -> 2ee12d042
SAMZA-1994: Table API: Add missed key lookups metric for table reads
Author: Aditya Toomula <at...@linkedin.com>
Reviewers: Srinivasulu Punuru <sp...@linkedin.com>, Wei Song <ws...@linkedin.com>
Closes #811 from atoomula/metric
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/2ee12d04
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/2ee12d04
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/2ee12d04
Branch: refs/heads/master
Commit: 2ee12d042471e3ebcca439c29297bd17e6a32a1c
Parents: b058459
Author: Aditya Toomula <at...@linkedin.com>
Authored: Tue Nov 20 13:02:13 2018 -0800
Committer: Srinivasulu Punuru <sp...@linkedin.com>
Committed: Tue Nov 20 13:02:13 2018 -0800
----------------------------------------------------------------------
.../apache/samza/table/remote/RemoteReadableTable.java | 12 ++++++++++--
.../samza/table/utils/DefaultTableReadMetrics.java | 2 ++
.../org/apache/samza/storage/kv/LocalReadableTable.java | 5 +++++
3 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/2ee12d04/samza-core/src/main/java/org/apache/samza/table/remote/RemoteReadableTable.java
----------------------------------------------------------------------
diff --git a/samza-core/src/main/java/org/apache/samza/table/remote/RemoteReadableTable.java b/samza-core/src/main/java/org/apache/samza/table/remote/RemoteReadableTable.java
index 0ae2728..f0d781a 100644
--- a/samza-core/src/main/java/org/apache/samza/table/remote/RemoteReadableTable.java
+++ b/samza-core/src/main/java/org/apache/samza/table/remote/RemoteReadableTable.java
@@ -21,6 +21,7 @@ package org.apache.samza.table.remote;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import java.util.Objects;
import org.apache.samza.SamzaException;
import org.apache.samza.context.Context;
import org.apache.samza.metrics.Timer;
@@ -129,8 +130,14 @@ public class RemoteReadableTable<K, V> implements ReadableTable<K, V> {
Preconditions.checkNotNull(key);
readMetrics.numGets.inc();
return execute(readRateLimiter, key, readFn::getAsync, readMetrics.getNs)
- .exceptionally(e -> {
- throw new SamzaException("Failed to get the record for " + key, e);
+ .handle((result, e) -> {
+ if (e != null) {
+ throw new SamzaException("Failed to get the records for " + key, e);
+ }
+ if (result == null) {
+ readMetrics.numMissedLookups.inc();
+ }
+ return result;
});
}
@@ -159,6 +166,7 @@ public class RemoteReadableTable<K, V> implements ReadableTable<K, V> {
if (e != null) {
throw new SamzaException("Failed to get the records for " + keys, e);
}
+ result.values().stream().filter(Objects::isNull).map(v -> readMetrics.numMissedLookups.inc());
return result;
});
}
http://git-wip-us.apache.org/repos/asf/samza/blob/2ee12d04/samza-core/src/main/java/org/apache/samza/table/utils/DefaultTableReadMetrics.java
----------------------------------------------------------------------
diff --git a/samza-core/src/main/java/org/apache/samza/table/utils/DefaultTableReadMetrics.java b/samza-core/src/main/java/org/apache/samza/table/utils/DefaultTableReadMetrics.java
index 090c8c1..525a0cb 100644
--- a/samza-core/src/main/java/org/apache/samza/table/utils/DefaultTableReadMetrics.java
+++ b/samza-core/src/main/java/org/apache/samza/table/utils/DefaultTableReadMetrics.java
@@ -34,6 +34,7 @@ public class DefaultTableReadMetrics {
public final Counter numGets;
public final Counter numGetAlls;
public final Timer getCallbackNs;
+ public final Counter numMissedLookups;
/**
* Constructor based on container and task container context
@@ -49,6 +50,7 @@ public class DefaultTableReadMetrics {
numGets = tableMetricsUtil.newCounter("num-gets");
numGetAlls = tableMetricsUtil.newCounter("num-getAlls");
getCallbackNs = tableMetricsUtil.newTimer("get-callback-ns");
+ numMissedLookups = tableMetricsUtil.newCounter("num-missed-lookups");
}
}
http://git-wip-us.apache.org/repos/asf/samza/blob/2ee12d04/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalReadableTable.java
----------------------------------------------------------------------
diff --git a/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalReadableTable.java b/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalReadableTable.java
index 4eed288..9cc9c49 100644
--- a/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalReadableTable.java
+++ b/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalReadableTable.java
@@ -21,6 +21,7 @@ package org.apache.samza.storage.kv;
import com.google.common.base.Preconditions;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import org.apache.samza.context.Context;
import org.apache.samza.table.ReadableTable;
@@ -66,6 +67,9 @@ public class LocalReadableTable<K, V> implements ReadableTable<K, V> {
long startNs = System.nanoTime();
V result = kvStore.get(key);
readMetrics.getNs.update(System.nanoTime() - startNs);
+ if (result == null) {
+ readMetrics.numMissedLookups.inc();
+ }
return result;
}
@@ -86,6 +90,7 @@ public class LocalReadableTable<K, V> implements ReadableTable<K, V> {
long startNs = System.nanoTime();
Map<K, V> result = kvStore.getAll(keys);
readMetrics.getAllNs.update(System.nanoTime() - startNs);
+ result.values().stream().filter(Objects::isNull).map(v -> readMetrics.numMissedLookups.inc());
return result;
}