You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by za...@apache.org on 2022/10/13 22:52:02 UTC
[druid] branch master updated: Improve global-cached-lookups metric reporting (#13219)
This is an automated email from the ASF dual-hosted git repository.
zachjsh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2f2fe20089 Improve global-cached-lookups metric reporting (#13219)
2f2fe20089 is described below
commit 2f2fe20089d54188374ee8072ed8d39564f6f9ab
Author: zachjsh <za...@gmail.com>
AuthorDate: Thu Oct 13 18:51:54 2022 -0400
Improve global-cached-lookups metric reporting (#13219)
It was found that the namespace/cache/heapSizeInBytes metric that tracks the total heap size in bytes of all lookup caches loaded on a service instance was being under reported. We were not accounting for the memory overhead of the String object, which I've found in testing to be ~40 bytes. While this overhead may be java version dependent, it should not vary much, and accounting for this provides a better estimate. Also fixed some logging, and reading bytes from the JDBI result set a [...]
---
.../statsd-emitter/src/main/resources/defaultMetricDimensions.json | 5 ++++-
.../src/main/java/org/apache/druid/data/input/MapPopulator.java | 3 ++-
.../org/apache/druid/server/lookup/namespace/JdbcCacheGenerator.java | 4 ++--
.../org/apache/druid/server/lookup/namespace/UriCacheGenerator.java | 2 +-
.../namespace/cache/OnHeapNamespaceExtractionCacheManager.java | 4 +++-
.../src/test/java/org/apache/druid/data/input/MapPopulatorTest.java | 4 ++--
6 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
index 34ece5e0bf..3a7767c418 100644
--- a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
+++ b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
@@ -169,5 +169,8 @@
"segment/compacted/bytes" : { "dimensions" : ["dataSource"], "type" : "gauge" },
"segment/compacted/count" : { "dimensions" : ["dataSource"], "type" : "count" },
- "interval/compacted/count" : { "dimensions" : ["dataSource"], "type" : "count" }
+ "interval/compacted/count" : { "dimensions" : ["dataSource"], "type" : "count" },
+
+ "namespace/cache/numEntries" : { "dimensions" : [], "type" : "gauge" },
+ "namespace/cache/heapSizeInBytes" : { "dimensions" : [], "type" : "gauge" }
}
diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/data/input/MapPopulator.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/data/input/MapPopulator.java
index 41bcc189ea..c5e8c6e4bf 100644
--- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/data/input/MapPopulator.java
+++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/data/input/MapPopulator.java
@@ -245,7 +245,8 @@ public class MapPopulator<K, V>
{
if (null != o) {
if (o.getClass().getName().equals(STRING_CLASS_NAME)) {
- return ((String) (o)).length();
+ // Each String object has ~40 bytes of overhead
+ return ((long) ((String) (o)).length() * Character.BYTES) + 40;
} else if (o.getClass().getName().equals(DOUBLE_CLASS_NAME)) {
return 8;
} else if (o.getClass().getName().equals(FLOAT_CLASS_NAME)) {
diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/JdbcCacheGenerator.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/JdbcCacheGenerator.java
index ba35a5f0b1..f05ff70250 100644
--- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/JdbcCacheGenerator.java
+++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/JdbcCacheGenerator.java
@@ -105,7 +105,7 @@ public final class JdbcCacheGenerator implements CacheGenerator<JdbcExtractionNa
);
final long duration = System.nanoTime() - startNs;
LOG.info(
- "Finished loading %,d values (%d bytes) for [%s] in %,d ns",
+ "Finished loading %d values (%d bytes) for [%s] in %d ns",
populateResult.getEntries(),
populateResult.getBytes(),
entryId,
@@ -151,7 +151,7 @@ public final class JdbcCacheGenerator implements CacheGenerator<JdbcExtractionNa
final String keyColumn = namespace.getKeyColumn();
return handle.createQuery(buildLookupQuery(table, filter, keyColumn, valueColumn))
- .map((index1, r1, ctx1) -> new Pair<>(r1.getString(keyColumn), r1.getString(valueColumn)))
+ .map((index1, r1, ctx1) -> new Pair<>(r1.getString(1), r1.getString(2)))
.iterator();
}
diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/UriCacheGenerator.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/UriCacheGenerator.java
index 3c10ef6620..c42bc91d5b 100644
--- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/UriCacheGenerator.java
+++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/UriCacheGenerator.java
@@ -156,7 +156,7 @@ public final class UriCacheGenerator implements CacheGenerator<UriExtractionName
);
final long duration = System.nanoTime() - startNs;
log.info(
- "Finished loading %,d values (%d bytes) from %,d lines for [%s] in %,d ns",
+ "Finished loading %d values (%d bytes) from %d lines for [%s] in %d ns",
populateResult.getEntries(),
populateResult.getBytes(),
populateResult.getLines(),
diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java
index 287d927f49..fe013d5e73 100644
--- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java
+++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java
@@ -150,7 +150,9 @@ public class OnHeapNamespaceExtractionCacheManager extends NamespaceExtractionCa
size += key.length() + value.length();
}
}
+ // Each String object has ~40 bytes of overhead, and x 2 for key and value strings
+ long heapSizeInBytes = (80 * numEntries) + size * Character.BYTES;
serviceEmitter.emit(ServiceMetricEvent.builder().build("namespace/cache/numEntries", numEntries));
- serviceEmitter.emit(ServiceMetricEvent.builder().build("namespace/cache/heapSizeInBytes", size * Character.BYTES));
+ serviceEmitter.emit(ServiceMetricEvent.builder().build("namespace/cache/heapSizeInBytes", heapSizeInBytes));
}
}
diff --git a/extensions-core/lookups-cached-global/src/test/java/org/apache/druid/data/input/MapPopulatorTest.java b/extensions-core/lookups-cached-global/src/test/java/org/apache/druid/data/input/MapPopulatorTest.java
index 5028a7bdcf..99aa8ac1c8 100644
--- a/extensions-core/lookups-cached-global/src/test/java/org/apache/druid/data/input/MapPopulatorTest.java
+++ b/extensions-core/lookups-cached-global/src/test/java/org/apache/druid/data/input/MapPopulatorTest.java
@@ -39,7 +39,7 @@ public class MapPopulatorTest
public void test_getByteLengthOfObject_string_stringLength()
{
String o = "string";
- Assert.assertEquals(o.length(), MapPopulator.getByteLengthOfObject(o));
+ Assert.assertEquals((o.length() * Character.BYTES) + 40, MapPopulator.getByteLengthOfObject(o));
}
@Test
@@ -184,7 +184,7 @@ public class MapPopulatorTest
Assert.assertEquals(expectedMap, map);
Assert.assertEquals(4, result.getEntries());
Assert.assertEquals(0, result.getLines());
- Assert.assertEquals(16L, result.getBytes());
+ Assert.assertEquals(192L, result.getBytes());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org