You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by "mjsax (via GitHub)" <gi...@apache.org> on 2025/06/23 17:56:13 UTC
[PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
mjsax opened a new pull request, #20022:
URL: https://github.com/apache/kafka/pull/20022
The metric for oldest-iterator-open-since-ms might report a null value if there is not open iterator.
This PR changes the behavior to dynamically register/deregister the entire metric instead of allowing it to return a null value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
Posted by "bbejeck (via GitHub)" <gi...@apache.org>.
bbejeck commented on code in PR #20022:
URL: https://github.com/apache/kafka/pull/20022#discussion_r2165155288
##########
streams/src/main/java/org/apache/kafka/streams/internals/metrics/OpenIterators.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.MeteredIterator;
+import org.apache.kafka.streams.state.internals.metrics.StateStoreMetrics;
+
+import java.util.Comparator;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.LongAdder;
+
+public class OpenIterators {
+ private final TaskId taskId;
+ private final String metricsScope;
+ private final String name;
+ private final StreamsMetricsImpl streamsMetrics;
+
+ private final LongAdder numOpenIterators = new LongAdder();
+ private final NavigableSet<MeteredIterator> openIterators = new ConcurrentSkipListSet<>(Comparator.comparingLong(MeteredIterator::startTimestamp));
+
+ private MetricName metricName;
+
+ public OpenIterators(final TaskId taskId,
+ final String metricsScope,
+ final String name,
+ final StreamsMetricsImpl streamsMetrics) {
+ this.taskId = taskId;
+ this.metricsScope = metricsScope;
+ this.name = name;
+ this.streamsMetrics = streamsMetrics;
+ }
+
+ public void add(final MeteredIterator iterator) {
+ openIterators.add(iterator);
+ numOpenIterators.increment();
Review Comment:
> Guess we can also do this unification only in trunk to get this into 4.1 on time?
That works for me
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #20022:
URL: https://github.com/apache/kafka/pull/20022#discussion_r2165139265
##########
streams/src/main/java/org/apache/kafka/streams/internals/metrics/OpenIterators.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.MeteredIterator;
+import org.apache.kafka.streams.state.internals.metrics.StateStoreMetrics;
+
+import java.util.Comparator;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.LongAdder;
+
+public class OpenIterators {
+ private final TaskId taskId;
+ private final String metricsScope;
+ private final String name;
+ private final StreamsMetricsImpl streamsMetrics;
+
+ private final LongAdder numOpenIterators = new LongAdder();
+ private final NavigableSet<MeteredIterator> openIterators = new ConcurrentSkipListSet<>(Comparator.comparingLong(MeteredIterator::startTimestamp));
+
+ private MetricName metricName;
+
+ public OpenIterators(final TaskId taskId,
+ final String metricsScope,
+ final String name,
+ final StreamsMetricsImpl streamsMetrics) {
+ this.taskId = taskId;
+ this.metricsScope = metricsScope;
+ this.name = name;
+ this.streamsMetrics = streamsMetrics;
+ }
+
+ public void add(final MeteredIterator iterator) {
+ openIterators.add(iterator);
+ numOpenIterators.increment();
Review Comment:
Seems this has historic reasons -- I just "refactored" existing code.
[KIP-989](https://cwiki.apache.org/confluence/display/KAFKA/KIP-989%3A+Improved+StateStore+Iterator+metrics+for+detecting+leaks) introduced 4 new metrics which got added by 4 PRs.
- `numOpenIterators` -> https://github.com/apache/kafka/pull/15975
- `openIterators` -> https://github.com/apache/kafka/pull/16041
(omitting the other two PRs as they are irrelevant)
There was also a follow up PR fro `numOpenIterators`: https://github.com/apache/kafka/pull/16076
@nicktelford do you see any reason why we need both variables, or can we unify them?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #20022:
URL: https://github.com/apache/kafka/pull/20022#discussion_r2167983511
##########
streams/src/main/java/org/apache/kafka/streams/internals/metrics/OpenIterators.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.MeteredIterator;
+import org.apache.kafka.streams.state.internals.metrics.StateStoreMetrics;
+
+import java.util.Comparator;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.LongAdder;
+
+public class OpenIterators {
+ private final TaskId taskId;
+ private final String metricsScope;
+ private final String name;
+ private final StreamsMetricsImpl streamsMetrics;
+
+ private final LongAdder numOpenIterators = new LongAdder();
+ private final NavigableSet<MeteredIterator> openIterators = new ConcurrentSkipListSet<>(Comparator.comparingLong(MeteredIterator::startTimestamp));
+
+ private MetricName metricName;
+
+ public OpenIterators(final TaskId taskId,
+ final String metricsScope,
+ final String name,
+ final StreamsMetricsImpl streamsMetrics) {
+ this.taskId = taskId;
+ this.metricsScope = metricsScope;
+ this.name = name;
+ this.streamsMetrics = streamsMetrics;
+ }
+
+ public void add(final MeteredIterator iterator) {
+ openIterators.add(iterator);
+ numOpenIterators.increment();
Review Comment:
Are you interested to do a PR for this @nicktelford? If not, happy to do one myself.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
Posted by "nicktelford (via GitHub)" <gi...@apache.org>.
nicktelford commented on code in PR #20022:
URL: https://github.com/apache/kafka/pull/20022#discussion_r2167210497
##########
streams/src/main/java/org/apache/kafka/streams/internals/metrics/OpenIterators.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.MeteredIterator;
+import org.apache.kafka.streams.state.internals.metrics.StateStoreMetrics;
+
+import java.util.Comparator;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.LongAdder;
+
+public class OpenIterators {
+ private final TaskId taskId;
+ private final String metricsScope;
+ private final String name;
+ private final StreamsMetricsImpl streamsMetrics;
+
+ private final LongAdder numOpenIterators = new LongAdder();
+ private final NavigableSet<MeteredIterator> openIterators = new ConcurrentSkipListSet<>(Comparator.comparingLong(MeteredIterator::startTimestamp));
+
+ private MetricName metricName;
+
+ public OpenIterators(final TaskId taskId,
+ final String metricsScope,
+ final String name,
+ final StreamsMetricsImpl streamsMetrics) {
+ this.taskId = taskId;
+ this.metricsScope = metricsScope;
+ this.name = name;
+ this.streamsMetrics = streamsMetrics;
+ }
+
+ public void add(final MeteredIterator iterator) {
+ openIterators.add(iterator);
+ numOpenIterators.increment();
Review Comment:
I'm not certain, but it's possible the `LongAdder` and `Set` duplicating the counting functionality is just an oversight. In theory, using the `LongAdder` would yield improved performance vs. `openIterators.size()`, however, in practice the total number of open iterators is not likely to be large enough to be a problem, so it seems reasonable to remove `numOpenIterators`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax merged PR #20022:
URL: https://github.com/apache/kafka/pull/20022
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #20022:
URL: https://github.com/apache/kafka/pull/20022#discussion_r2165139265
##########
streams/src/main/java/org/apache/kafka/streams/internals/metrics/OpenIterators.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.MeteredIterator;
+import org.apache.kafka.streams.state.internals.metrics.StateStoreMetrics;
+
+import java.util.Comparator;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.LongAdder;
+
+public class OpenIterators {
+ private final TaskId taskId;
+ private final String metricsScope;
+ private final String name;
+ private final StreamsMetricsImpl streamsMetrics;
+
+ private final LongAdder numOpenIterators = new LongAdder();
+ private final NavigableSet<MeteredIterator> openIterators = new ConcurrentSkipListSet<>(Comparator.comparingLong(MeteredIterator::startTimestamp));
+
+ private MetricName metricName;
+
+ public OpenIterators(final TaskId taskId,
+ final String metricsScope,
+ final String name,
+ final StreamsMetricsImpl streamsMetrics) {
+ this.taskId = taskId;
+ this.metricsScope = metricsScope;
+ this.name = name;
+ this.streamsMetrics = streamsMetrics;
+ }
+
+ public void add(final MeteredIterator iterator) {
+ openIterators.add(iterator);
+ numOpenIterators.increment();
Review Comment:
Seems this has historic reasons -- I just "refactored" existing code.
[KIP-989](https://cwiki.apache.org/confluence/display/KAFKA/KIP-989%3A+Improved+StateStore+Iterator+metrics+for+detecting+leaks) introduced 4 new metrics which got added by 4 PRs.
- `numOpenIterators` -> https://github.com/apache/kafka/pull/15975
- ` openIterators` -> https://github.com/apache/kafka/pull/16041
There was also a follow up PR fro `numOpenIterators`: https://github.com/apache/kafka/pull/16076
@nicktelford do you see any reason why we need both variables, or can we unify them?
##########
streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/StateStoreMetrics.java:
##########
@@ -455,12 +456,12 @@ public static void addNumOpenIteratorsGauge(final String taskId,
}
- public static void addOldestOpenIteratorGauge(final String taskId,
- final String storeType,
- final String storeName,
- final StreamsMetricsImpl streamsMetrics,
- final Gauge<Long> oldestOpenIteratorGauge) {
- streamsMetrics.addStoreLevelMutableMetric(
+ public static MetricName addOldestOpenIteratorGauge(final String taskId,
+ final String storeType,
+ final String storeName,
+ final StreamsMetricsImpl streamsMetrics,
+ final Gauge<Long> oldestOpenIteratorGauge) {
+ return streamsMetrics.addStoreLevelMutableMetric(
Review Comment:
Same answer :)
##########
streams/src/main/java/org/apache/kafka/streams/internals/metrics/OpenIterators.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.MeteredIterator;
+import org.apache.kafka.streams.state.internals.metrics.StateStoreMetrics;
+
+import java.util.Comparator;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.LongAdder;
+
+public class OpenIterators {
+ private final TaskId taskId;
+ private final String metricsScope;
+ private final String name;
+ private final StreamsMetricsImpl streamsMetrics;
+
+ private final LongAdder numOpenIterators = new LongAdder();
+ private final NavigableSet<MeteredIterator> openIterators = new ConcurrentSkipListSet<>(Comparator.comparingLong(MeteredIterator::startTimestamp));
+
+ private MetricName metricName;
+
+ public OpenIterators(final TaskId taskId,
+ final String metricsScope,
+ final String name,
+ final StreamsMetricsImpl streamsMetrics) {
+ this.taskId = taskId;
+ this.metricsScope = metricsScope;
+ this.name = name;
+ this.streamsMetrics = streamsMetrics;
+ }
+
+ public void add(final MeteredIterator iterator) {
+ openIterators.add(iterator);
+ numOpenIterators.increment();
+
+ if (numOpenIterators.intValue() == 1) {
+ metricName = StateStoreMetrics.addOldestOpenIteratorGauge(taskId.toString(), metricsScope, name, streamsMetrics,
Review Comment:
Answer to your question: we need to remember the `MetricName` here, to be able to use it to remove the metric if the iterator count goes back to zero.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java:
##########
@@ -535,6 +539,8 @@ public <T> void addStoreLevelMutableMetric(final String taskId,
final String key = storeSensorPrefix(Thread.currentThread().getName(), taskId, storeName);
storeLevelMetrics.computeIfAbsent(key, ignored -> new LinkedList<>()).push(metricName);
}
+
+ return metricName;
Review Comment:
We need the `MetricName` to be able to remove it when the iterator count goes to zero.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on PR #20022:
URL: https://github.com/apache/kafka/pull/20022#issuecomment-3002209280
Merged to `trunk` and cherry-picked to `4.1` branch.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
Posted by "bbejeck (via GitHub)" <gi...@apache.org>.
bbejeck commented on code in PR #20022:
URL: https://github.com/apache/kafka/pull/20022#discussion_r2165017062
##########
streams/src/main/java/org/apache/kafka/streams/internals/metrics/OpenIterators.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.MeteredIterator;
+import org.apache.kafka.streams.state.internals.metrics.StateStoreMetrics;
+
+import java.util.Comparator;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.LongAdder;
+
+public class OpenIterators {
+ private final TaskId taskId;
+ private final String metricsScope;
+ private final String name;
+ private final StreamsMetricsImpl streamsMetrics;
+
+ private final LongAdder numOpenIterators = new LongAdder();
+ private final NavigableSet<MeteredIterator> openIterators = new ConcurrentSkipListSet<>(Comparator.comparingLong(MeteredIterator::startTimestamp));
+
+ private MetricName metricName;
+
+ public OpenIterators(final TaskId taskId,
+ final String metricsScope,
+ final String name,
+ final StreamsMetricsImpl streamsMetrics) {
+ this.taskId = taskId;
+ this.metricsScope = metricsScope;
+ this.name = name;
+ this.streamsMetrics = streamsMetrics;
+ }
+
+ public void add(final MeteredIterator iterator) {
+ openIterators.add(iterator);
+ numOpenIterators.increment();
Review Comment:
Why not `openIterators.size()`?
##########
streams/src/main/java/org/apache/kafka/streams/state/internals/metrics/StateStoreMetrics.java:
##########
@@ -455,12 +456,12 @@ public static void addNumOpenIteratorsGauge(final String taskId,
}
- public static void addOldestOpenIteratorGauge(final String taskId,
- final String storeType,
- final String storeName,
- final StreamsMetricsImpl streamsMetrics,
- final Gauge<Long> oldestOpenIteratorGauge) {
- streamsMetrics.addStoreLevelMutableMetric(
+ public static MetricName addOldestOpenIteratorGauge(final String taskId,
+ final String storeType,
+ final String storeName,
+ final StreamsMetricsImpl streamsMetrics,
+ final Gauge<Long> oldestOpenIteratorGauge) {
+ return streamsMetrics.addStoreLevelMutableMetric(
Review Comment:
Same question here
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java:
##########
@@ -535,6 +539,8 @@ public <T> void addStoreLevelMutableMetric(final String taskId,
final String key = storeSensorPrefix(Thread.currentThread().getName(), taskId, storeName);
storeLevelMetrics.computeIfAbsent(key, ignored -> new LinkedList<>()).push(metricName);
}
+
+ return metricName;
Review Comment:
Why does `addX` return the `MetricName` obejct?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-19398: (De)Register oldest-iterator-open-since-ms metric dynamically [kafka]
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #20022:
URL: https://github.com/apache/kafka/pull/20022#discussion_r2165143314
##########
streams/src/main/java/org/apache/kafka/streams/internals/metrics/OpenIterators.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.internals.metrics;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.streams.processor.TaskId;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
+import org.apache.kafka.streams.state.internals.MeteredIterator;
+import org.apache.kafka.streams.state.internals.metrics.StateStoreMetrics;
+
+import java.util.Comparator;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.LongAdder;
+
+public class OpenIterators {
+ private final TaskId taskId;
+ private final String metricsScope;
+ private final String name;
+ private final StreamsMetricsImpl streamsMetrics;
+
+ private final LongAdder numOpenIterators = new LongAdder();
+ private final NavigableSet<MeteredIterator> openIterators = new ConcurrentSkipListSet<>(Comparator.comparingLong(MeteredIterator::startTimestamp));
+
+ private MetricName metricName;
+
+ public OpenIterators(final TaskId taskId,
+ final String metricsScope,
+ final String name,
+ final StreamsMetricsImpl streamsMetrics) {
+ this.taskId = taskId;
+ this.metricsScope = metricsScope;
+ this.name = name;
+ this.streamsMetrics = streamsMetrics;
+ }
+
+ public void add(final MeteredIterator iterator) {
+ openIterators.add(iterator);
+ numOpenIterators.increment();
Review Comment:
Guess we can also do this unification only in `trunk` to get this into `4.1` on time?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org