You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by "liangyepianzhou (via GitHub)" <gi...@apache.org> on 2023/03/02 05:38:41 UTC
[GitHub] [pulsar] liangyepianzhou commented on a diff in pull request #19641: [feat][txn] Transaction buffer snapshot writer reuse
liangyepianzhou commented on code in PR #19641:
URL: https://github.com/apache/pulsar/pull/19641#discussion_r1122592374
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java:
##########
@@ -21,63 +21,150 @@
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
import org.apache.pulsar.broker.systopic.SystemTopicClient;
import org.apache.pulsar.broker.systopic.SystemTopicClientBase;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.common.events.EventType;
+import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.util.FutureUtil;
+@Slf4j
public class SystemTopicTxnBufferSnapshotService<T> {
- protected final Map<TopicName, SystemTopicClient<T>> clients;
+ protected final ConcurrentHashMap<NamespaceName, SystemTopicClient<T>> clients;
protected final NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory;
protected final Class<T> schemaType;
protected final EventType systemTopicType;
+ private final ConcurrentHashMap<NamespaceName, ReferenceCountedWriter<T>> refCountedWriterMap;
+
+ // The class ReferenceCountedWriter will maintain the reference count,
+ // when the reference count decrement to 0, it will be removed from writerFutureMap, the writer will be closed.
+ public static class ReferenceCountedWriter<T> {
+
+ private final AtomicLong referenceCount;
+ private final NamespaceName namespaceName;
+ private final CompletableFuture<SystemTopicClient.Writer<T>> future;
+ private final SystemTopicTxnBufferSnapshotService<T> snapshotService;
+
+ public ReferenceCountedWriter(NamespaceName namespaceName,
+ CompletableFuture<SystemTopicClient.Writer<T>> future,
+ SystemTopicTxnBufferSnapshotService<T> snapshotService) {
+ this.referenceCount = new AtomicLong(1);
+ this.namespaceName = namespaceName;
+ this.snapshotService = snapshotService;
+ this.future = future;
+ this.future.exceptionally(t -> {
+ log.error("[{}] Failed to create transaction buffer snapshot writer.", namespaceName, t);
+ snapshotService.refCountedWriterMap.remove(namespaceName, this);
+ return null;
+ });
+ }
+
+ public CompletableFuture<SystemTopicClient.Writer<T>> getFuture() {
+ return future;
+ }
+
+ private void retain() {
+ operationValidate(true);
+ this.referenceCount.incrementAndGet();
+ }
+
+ private long release() {
+ operationValidate(false);
+ return this.referenceCount.decrementAndGet();
+ }
+
+ private void operationValidate(boolean isRetain) {
+ if (this.referenceCount.get() == 0) {
+ throw new RuntimeException(
+ "[" + namespaceName + "] The reference counted transaction buffer snapshot writer couldn't "
+ + "be " + (isRetain ? "retained" : "released") + ", refCnt is 0.");
+ }
+ }
+
+ public void close() {
+ if (release() == 0) {
+ snapshotService.refCountedWriterMap.remove(namespaceName, this);
+ future.thenAccept(writer -> {
+ final String topicName = writer.getSystemTopicClient().getTopicName().toString();
+ writer.closeAsync().exceptionally(t -> {
+ if (t != null) {
+ log.error("[{}] Failed to close writer.", topicName, t);
+ } else {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Success to close writer.", topicName);
+ }
+ }
+ return null;
+ });
+ });
+ }
+ }
+
+ }
+
public SystemTopicTxnBufferSnapshotService(PulsarClient client, EventType systemTopicType,
Class<T> schemaType) {
this.namespaceEventsSystemTopicFactory = new NamespaceEventsSystemTopicFactory(client);
this.systemTopicType = systemTopicType;
this.schemaType = schemaType;
this.clients = new ConcurrentHashMap<>();
+ this.refCountedWriterMap = new ConcurrentHashMap<>();
}
public CompletableFuture<SystemTopicClient.Writer<T>> createWriter(TopicName topicName) {
Review Comment:
This method is useless. We can delete it, right?
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java:
##########
@@ -21,63 +21,150 @@
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
import org.apache.pulsar.broker.systopic.SystemTopicClient;
import org.apache.pulsar.broker.systopic.SystemTopicClientBase;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.common.events.EventType;
+import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.util.FutureUtil;
+@Slf4j
public class SystemTopicTxnBufferSnapshotService<T> {
- protected final Map<TopicName, SystemTopicClient<T>> clients;
+ protected final ConcurrentHashMap<NamespaceName, SystemTopicClient<T>> clients;
protected final NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory;
protected final Class<T> schemaType;
protected final EventType systemTopicType;
+ private final ConcurrentHashMap<NamespaceName, ReferenceCountedWriter<T>> refCountedWriterMap;
+
+ // The class ReferenceCountedWriter will maintain the reference count,
+ // when the reference count decrement to 0, it will be removed from writerFutureMap, the writer will be closed.
+ public static class ReferenceCountedWriter<T> {
+
+ private final AtomicLong referenceCount;
+ private final NamespaceName namespaceName;
+ private final CompletableFuture<SystemTopicClient.Writer<T>> future;
+ private final SystemTopicTxnBufferSnapshotService<T> snapshotService;
+
+ public ReferenceCountedWriter(NamespaceName namespaceName,
+ CompletableFuture<SystemTopicClient.Writer<T>> future,
+ SystemTopicTxnBufferSnapshotService<T> snapshotService) {
+ this.referenceCount = new AtomicLong(1);
+ this.namespaceName = namespaceName;
+ this.snapshotService = snapshotService;
+ this.future = future;
+ this.future.exceptionally(t -> {
+ log.error("[{}] Failed to create transaction buffer snapshot writer.", namespaceName, t);
+ snapshotService.refCountedWriterMap.remove(namespaceName, this);
+ return null;
+ });
+ }
+
+ public CompletableFuture<SystemTopicClient.Writer<T>> getFuture() {
+ return future;
+ }
+
+ private void retain() {
Review Comment:
If we only lock release and retain.
|time| task execute `ReferenceCountedWriter::close`| task execute `getReferenceWriter`|
| --- | --- | --- |
|0| check referenceCount != 0 | |
|1|decrement referenceCount = 0| |
|2|remove and close the writer| |
|3|| `retain` check referenceCount == 0 |
|4||throw RuntimeException|
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java:
##########
@@ -21,63 +21,150 @@
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
import org.apache.pulsar.broker.systopic.SystemTopicClient;
import org.apache.pulsar.broker.systopic.SystemTopicClientBase;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.common.events.EventType;
+import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.util.FutureUtil;
+@Slf4j
public class SystemTopicTxnBufferSnapshotService<T> {
- protected final Map<TopicName, SystemTopicClient<T>> clients;
+ protected final ConcurrentHashMap<NamespaceName, SystemTopicClient<T>> clients;
protected final NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory;
protected final Class<T> schemaType;
protected final EventType systemTopicType;
+ private final ConcurrentHashMap<NamespaceName, ReferenceCountedWriter<T>> refCountedWriterMap;
+
+ // The class ReferenceCountedWriter will maintain the reference count,
+ // when the reference count decrement to 0, it will be removed from writerFutureMap, the writer will be closed.
+ public static class ReferenceCountedWriter<T> {
+
+ private final AtomicLong referenceCount;
+ private final NamespaceName namespaceName;
+ private final CompletableFuture<SystemTopicClient.Writer<T>> future;
+ private final SystemTopicTxnBufferSnapshotService<T> snapshotService;
+
+ public ReferenceCountedWriter(NamespaceName namespaceName,
+ CompletableFuture<SystemTopicClient.Writer<T>> future,
+ SystemTopicTxnBufferSnapshotService<T> snapshotService) {
+ this.referenceCount = new AtomicLong(1);
+ this.namespaceName = namespaceName;
+ this.snapshotService = snapshotService;
+ this.future = future;
+ this.future.exceptionally(t -> {
+ log.error("[{}] Failed to create transaction buffer snapshot writer.", namespaceName, t);
+ snapshotService.refCountedWriterMap.remove(namespaceName, this);
+ return null;
+ });
+ }
+
+ public CompletableFuture<SystemTopicClient.Writer<T>> getFuture() {
+ return future;
+ }
+
+ private void retain() {
Review Comment:
We may not only need to lock release and retain.
If we do not lock release and retain.
|time| task execute `ReferenceCountedWriter::close`| task execute `getReferenceWriter`|
| --- | --- | --- |
|0| check referenceCount != 0 | `retain` check referenceCount != 0|
|1|decrement referenceCount = 0| |
|2|remove and close the writer| |
|3||increment referenceCount |
|4||return the closed writer|
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java:
##########
@@ -21,63 +21,150 @@
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
import org.apache.pulsar.broker.systopic.SystemTopicClient;
import org.apache.pulsar.broker.systopic.SystemTopicClientBase;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.common.events.EventType;
+import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.util.FutureUtil;
+@Slf4j
public class SystemTopicTxnBufferSnapshotService<T> {
- protected final Map<TopicName, SystemTopicClient<T>> clients;
+ protected final ConcurrentHashMap<NamespaceName, SystemTopicClient<T>> clients;
protected final NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory;
protected final Class<T> schemaType;
protected final EventType systemTopicType;
+ private final ConcurrentHashMap<NamespaceName, ReferenceCountedWriter<T>> refCountedWriterMap;
+
+ // The class ReferenceCountedWriter will maintain the reference count,
+ // when the reference count decrement to 0, it will be removed from writerFutureMap, the writer will be closed.
+ public static class ReferenceCountedWriter<T> {
+
+ private final AtomicLong referenceCount;
+ private final NamespaceName namespaceName;
+ private final CompletableFuture<SystemTopicClient.Writer<T>> future;
+ private final SystemTopicTxnBufferSnapshotService<T> snapshotService;
+
+ public ReferenceCountedWriter(NamespaceName namespaceName,
+ CompletableFuture<SystemTopicClient.Writer<T>> future,
+ SystemTopicTxnBufferSnapshotService<T> snapshotService) {
+ this.referenceCount = new AtomicLong(1);
+ this.namespaceName = namespaceName;
+ this.snapshotService = snapshotService;
+ this.future = future;
+ this.future.exceptionally(t -> {
+ log.error("[{}] Failed to create transaction buffer snapshot writer.", namespaceName, t);
+ snapshotService.refCountedWriterMap.remove(namespaceName, this);
+ return null;
+ });
+ }
+
+ public CompletableFuture<SystemTopicClient.Writer<T>> getFuture() {
+ return future;
+ }
+
+ private void retain() {
Review Comment:
We can change the retain method to
```java
private synchronized boolean retain() {
if (referenceCount.get() == 0) {
return false;
} else {
this.referenceCount.incrementAndGet();
return true;
}
}
```
And change getReferenceWriter to
```java
public ReferenceCountedWriter<T> getReferenceWriter(TopicName topicName) {
return refCountedWriterMap.compute(topicName.getNamespaceObject(), (k, v) -> {
if (v != null && v.retain()) {
return v;
} else {
return new ReferenceCountedWriter<>(topicName.getNamespaceObject(),
getTransactionBufferSystemTopicClient(topicName).newWriterAsync(), this);
}
});
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pulsar.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org