You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2021/08/06 01:45:37 UTC
[pulsar] branch branch-2.8 updated (b6d0934 -> 9268cb1)
This is an automated email from the ASF dual-hosted git repository.
penghui pushed a change to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git.
from b6d0934 [ManagedLedger] Compress managed ledger info (#11490)
new 79f180d Replace orElse with orElseGet to avoid calling too many times. (#11542) (#11542)
new 3ce4c59 Fix some topic policy operation without backoff (#11560)
new c406153 Add test for auto-created partitioned system topic (#11545)
new 9268cb1 [Patch] Fix decode compression managedLedgerInfo data (#11569)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../mledger/ManagedLedgerFactoryConfig.java | 4 +-
.../bookkeeper/mledger/impl/MetaStoreImpl.java | 8 +-
.../broker/admin/impl/PersistentTopicsBase.java | 127 +++++++--------------
.../pulsar/broker/admin/v2/PersistentTopics.java | 2 +-
.../service/ManagedLedgerCompressionTest.java | 47 ++++----
.../systopic/PartitionedSystemTopicTest.java | 69 +++++++++++
.../pulsar/common/events/EventsTopicNames.java | 2 +-
.../pulsar/common/protocol/schema/SchemaHash.java | 2 +-
.../pulsar/functions/source/TopicSchema.java | 2 +-
.../zookeeper/ZkBookieRackAffinityMapping.java | 4 +-
10 files changed, 151 insertions(+), 116 deletions(-)
create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java
[pulsar] 04/04: [Patch] Fix decode compression managedLedgerInfo
data (#11569)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 9268cb121503fb22132a36cc0108469f00e3bc08
Author: ran <ga...@126.com>
AuthorDate: Fri Aug 6 05:13:35 2021 +0800
[Patch] Fix decode compression managedLedgerInfo data (#11569)
* unify using the class `MLDataFormats.CompressionType` in package `managed-ledger`.
* fix compression data decode error
(cherry picked from commit f008777b8d32a58b9ce9d9ad6299ec3a0b181d27)
---
.../mledger/ManagedLedgerFactoryConfig.java | 4 +-
.../bookkeeper/mledger/impl/MetaStoreImpl.java | 8 ++--
.../service/ManagedLedgerCompressionTest.java | 47 +++++++++++++---------
3 files changed, 34 insertions(+), 25 deletions(-)
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
index 02ac4e9..ef92957 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
@@ -21,7 +21,7 @@ package org.apache.bookkeeper.mledger;
import lombok.Data;
import org.apache.bookkeeper.common.annotation.InterfaceAudience;
import org.apache.bookkeeper.common.annotation.InterfaceStability;
-import org.apache.pulsar.common.api.proto.CompressionType;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats;
/**
* Configuration for a {@link ManagedLedgerFactory}.
@@ -80,5 +80,5 @@ public class ManagedLedgerFactoryConfig {
/**
* ManagedLedgerInfo compression type. If the compression type is null or invalid, don't compress data.
*/
- private String managedLedgerInfoCompressionType = CompressionType.NONE.name();
+ private String managedLedgerInfoCompressionType = MLDataFormats.CompressionType.NONE.name();
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
index cfe3aaf..9558d11 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
@@ -317,7 +317,8 @@ public class MetaStoreImpl implements MetaStore {
metadataByteBuf.writeInt(mlInfoMetadata.getSerializedSize());
metadataByteBuf.writeBytes(mlInfoMetadata.toByteArray());
- encodeByteBuf = getCompressionCodec().encode(Unpooled.wrappedBuffer(managedLedgerInfo.toByteArray()));
+ encodeByteBuf = getCompressionCodec(compressionType)
+ .encode(Unpooled.wrappedBuffer(managedLedgerInfo.toByteArray()));
CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeBuffer();
compositeByteBuf.addComponent(true, metadataByteBuf);
@@ -347,7 +348,8 @@ public class MetaStoreImpl implements MetaStore {
MLDataFormats.ManagedLedgerInfoMetadata.parseFrom(metadataBytes);
long unpressedSize = metadata.getUncompressedSize();
- decodeByteBuf = getCompressionCodec().decode(byteBuf, (int) unpressedSize);
+ decodeByteBuf = getCompressionCodec(metadata.getCompressionType())
+ .decode(byteBuf, (int) unpressedSize);
byte[] decodeBytes;
// couldn't decode data by ZLIB compression byteBuf array() directly
if (decodeByteBuf.hasArray() && !CompressionType.ZLIB.equals(metadata.getCompressionType())) {
@@ -371,7 +373,7 @@ public class MetaStoreImpl implements MetaStore {
}
}
- private CompressionCodec getCompressionCodec() {
+ private CompressionCodec getCompressionCodec(CompressionType compressionType) {
return CompressionCodecProvider.getCompressionCodec(
org.apache.pulsar.common.api.proto.CompressionType.valueOf(compressionType.name()));
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
index 626522d..9a3bc2f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
@@ -20,10 +20,11 @@ package org.apache.pulsar.broker.service;
import java.util.concurrent.TimeUnit;
import lombok.Cleanup;
-import org.apache.pulsar.client.api.CompressionType;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClientException;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -37,7 +38,7 @@ public class ManagedLedgerCompressionTest extends BrokerTestBase {
@BeforeClass
@Override
protected void setup() throws Exception {
- conf.setManagedLedgerInfoCompressionType(CompressionType.NONE.name());
+ conf.setManagedLedgerInfoCompressionType(MLDataFormats.CompressionType.NONE.name());
super.baseSetup();
}
@@ -47,7 +48,7 @@ public class ManagedLedgerCompressionTest extends BrokerTestBase {
super.internalCleanup();
}
- @Test(timeOut = 1000 * 10)
+ @Test(timeOut = 1000 * 20)
public void testRestartBrokerEnableManagedLedgerInfoCompression() throws Exception {
String topic = newTopicName();
@Cleanup
@@ -61,27 +62,17 @@ public class ManagedLedgerCompressionTest extends BrokerTestBase {
.subscribe();
int messageCnt = 100;
- for (int i = 0; i < messageCnt; i++) {
- producer.newMessage().value("test".getBytes()).send();
- }
- for (int i = 0; i < messageCnt; i++) {
- Message<byte[]> message = consumer.receive(1000, TimeUnit.SECONDS);
- consumer.acknowledge(message);
- Assert.assertNotNull(message);
- }
+ produceAndConsume(producer, consumer, messageCnt);
stopBroker();
- conf.setManagedLedgerInfoCompressionType(CompressionType.ZSTD.name());
+ conf.setManagedLedgerInfoCompressionType(MLDataFormats.CompressionType.ZSTD.name());
startBroker();
+ produceAndConsume(producer, consumer, messageCnt);
- for (int i = 0; i < messageCnt; i++) {
- producer.newMessage().value("test".getBytes()).send();
- }
- for (int i = 0; i < messageCnt; i++) {
- Message<byte[]> message = consumer.receive(1000, TimeUnit.SECONDS);
- Assert.assertNotNull(message);
- consumer.acknowledge(message);
- }
+ stopBroker();
+ conf.setManagedLedgerInfoCompressionType(MLDataFormats.CompressionType.LZ4.name());
+ startBroker();
+ produceAndConsume(producer, consumer, messageCnt);
stopBroker();
conf.setManagedLedgerInfoCompressionType("INVALID");
@@ -94,6 +85,22 @@ public class ManagedLedgerCompressionTest extends BrokerTestBase {
"No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType.INVALID",
e.getCause().getMessage());
}
+
+ conf.setManagedLedgerInfoCompressionType(MLDataFormats.CompressionType.NONE.name());
+ startBroker();
+ produceAndConsume(producer, consumer, messageCnt);
+ }
+
+ private void produceAndConsume(Producer<byte[]> producer,
+ Consumer<byte[]> consumer, int messageCnt) throws PulsarClientException {
+ for (int i = 0; i < messageCnt; i++) {
+ producer.newMessage().value("test".getBytes()).send();
+ }
+ for (int i = 0; i < messageCnt; i++) {
+ Message<byte[]> message = consumer.receive(1000, TimeUnit.SECONDS);
+ consumer.acknowledge(message);
+ Assert.assertNotNull(message);
+ }
}
}
[pulsar] 01/04: Replace orElse with orElseGet to avoid calling too
many times. (#11542) (#11542)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 79f180d8d39961cc8c4e695ae8e5e3810e869c2b
Author: GuoJiwei <te...@apache.org>
AuthorDate: Thu Aug 5 18:56:31 2021 +0800
Replace orElse with orElseGet to avoid calling too many times. (#11542) (#11542)
(cherry picked from commit 27855f8eac7c926b4da9fda46379b64f8b522e62)
---
.../main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java | 2 +-
.../java/org/apache/pulsar/common/protocol/schema/SchemaHash.java | 2 +-
.../src/main/java/org/apache/pulsar/functions/source/TopicSchema.java | 2 +-
.../java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java | 4 ++--
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index a0699a4..2c54908 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -417,7 +417,7 @@ public class PersistentTopics extends PersistentTopicsBase {
preValidation(authoritative)
.thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName))
.thenAccept(op -> {
- TopicPolicies topicPolicies = op.orElse(new TopicPolicies());
+ TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
asyncResponse.resume(topicPolicies.getDeduplicationSnapshotIntervalSeconds());
})
.exceptionally(ex -> {
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaHash.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaHash.java
index 57aa3d4..40220e6 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaHash.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/schema/SchemaHash.java
@@ -46,7 +46,7 @@ public class SchemaHash {
public static SchemaHash of(Schema schema) {
Optional<SchemaInfo> schemaInfo = Optional.ofNullable(schema).map(Schema::getSchemaInfo);
- return of(schemaInfo.map(SchemaInfo::getSchema).orElse(new byte[0]),
+ return of(schemaInfo.map(SchemaInfo::getSchema).orElseGet(() -> new byte[0]),
schemaInfo.map(SchemaInfo::getType).orElse(null));
}
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java
index 5b7aed2..caf7ff8 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java
@@ -74,7 +74,7 @@ public class TopicSchema {
public Schema<?> getSchema(String topic, Class<?> clazz, Optional<SchemaType> schemaType) {
return cachedSchemas.computeIfAbsent(topic, key -> {
// If schema type was not provided, try to get it from schema registry, or fallback to default types
- SchemaType type = schemaType.orElse(getSchemaTypeOrDefault(topic, clazz));
+ SchemaType type = schemaType.orElseGet(() -> getSchemaTypeOrDefault(topic, clazz));
return newSchemaInstance(clazz, type);
});
}
diff --git a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java
index 5915c6d..19dcb99 100644
--- a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java
+++ b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java
@@ -77,7 +77,7 @@ public class ZkBookieRackAffinityMapping extends AbstractDNSToSwitchMapping
}
try {
- BookiesRackConfiguration racks = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).orElse(new BookiesRackConfiguration());
+ BookiesRackConfiguration racks = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).orElseGet(BookiesRackConfiguration::new);
updateRacksWithHost(racks);
} catch (Exception e) {
throw new RuntimeException(e);
@@ -185,7 +185,7 @@ public class ZkBookieRackAffinityMapping extends AbstractDNSToSwitchMapping
try {
// Trigger load of z-node in case it didn't exist
Optional<BookiesRackConfiguration> racks = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH);
- updateRacksWithHost(racks.orElse(new BookiesRackConfiguration()));
+ updateRacksWithHost(racks.orElseGet(BookiesRackConfiguration::new));
if (!racks.isPresent()) {
// since different placement policy will have different default rack,
// don't be smart here and just return null
[pulsar] 02/04: Fix some topic policy operation without backoff
(#11560)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 3ce4c59a1b1f871af696a4abea23cbf1e02658e7
Author: lipenghui <pe...@apache.org>
AuthorDate: Thu Aug 5 18:54:36 2021 +0800
Fix some topic policy operation without backoff (#11560)
* Fix some topic policy operation without backoff
Related to https://github.com/apache/pulsar/pull/11487
(cherry picked from commit 92a3ac7f2d9ba94da43eea28ad3d88ceffb6015d)
---
.../broker/admin/impl/PersistentTopicsBase.java | 127 +++++++--------------
1 file changed, 42 insertions(+), 85 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 3d78116..0c1a410 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -67,7 +67,6 @@ import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.admin.AdminResource;
import org.apache.pulsar.broker.admin.ZkAdminPaths;
import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
-import org.apache.pulsar.broker.service.BrokerServiceException;
import org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException;
import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException;
import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException;
@@ -780,41 +779,25 @@ public class PersistentTopicsBase extends AdminResource {
}
protected CompletableFuture<Void> internalSetOffloadPolicies(OffloadPoliciesImpl offloadPolicies) {
- TopicPolicies topicPolicies = null;
- try {
- topicPolicies = pulsar().getTopicPoliciesService().getTopicPolicies(topicName);
- } catch (BrokerServiceException.TopicPoliciesCacheNotInitException e) {
- log.error("Topic {} policies have not been initialized yet.", topicName);
- throw new RestException(Status.PRECONDITION_FAILED, "Policies have not been initialized yet");
- }
- if (topicPolicies == null) {
- topicPolicies = new TopicPolicies();
- }
- topicPolicies.setOffloadPolicies(offloadPolicies);
- CompletableFuture<Void> completableFuture = new CompletableFuture<>();
- pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies)
- .thenCompose((res) -> {
- //The policy update is asynchronous. Cache at this step may not be updated yet.
- //So we need to set the loader by the incoming offloadPolicies instead of topic policies cache.
- PartitionedTopicMetadata metadata = fetchPartitionedTopicMetadata(pulsar(), topicName);
- if (metadata.partitions > 0) {
- List<CompletableFuture<Void>> futures = new ArrayList<>(metadata.partitions);
- for (int i = 0; i < metadata.partitions; i++) {
- futures.add(internalUpdateOffloadPolicies(offloadPolicies, topicName.getPartition(i)));
- }
- return FutureUtil.waitForAll(futures);
- } else {
- return internalUpdateOffloadPolicies(offloadPolicies, topicName);
- }
- })
- .whenComplete((result, e) -> {
- if (e != null) {
- completableFuture.completeExceptionally(e);
- } else {
- completableFuture.complete(null);
+ return getTopicPoliciesAsyncWithRetry(topicName)
+ .thenCompose(op -> {
+ TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
+ topicPolicies.setOffloadPolicies(offloadPolicies);
+ return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ }).thenCompose(__ -> {
+ //The policy update is asynchronous. Cache at this step may not be updated yet.
+ //So we need to set the loader by the incoming offloadPolicies instead of topic policies cache.
+ PartitionedTopicMetadata metadata = fetchPartitionedTopicMetadata(pulsar(), topicName);
+ if (metadata.partitions > 0) {
+ List<CompletableFuture<Void>> futures = new ArrayList<>(metadata.partitions);
+ for (int i = 0; i < metadata.partitions; i++) {
+ futures.add(internalUpdateOffloadPolicies(offloadPolicies, topicName.getPartition(i)));
}
- });
- return completableFuture;
+ return FutureUtil.waitForAll(futures);
+ } else {
+ return internalUpdateOffloadPolicies(offloadPolicies, topicName);
+ }
+ });
}
protected CompletableFuture<InactiveTopicPolicies> internalGetInactiveTopicPolicies(boolean applied) {
@@ -833,19 +816,12 @@ public class PersistentTopicsBase extends AdminResource {
}
protected CompletableFuture<Void> internalSetInactiveTopicPolicies(InactiveTopicPolicies inactiveTopicPolicies) {
- TopicPolicies topicPolicies = null;
- try {
- topicPolicies = pulsar().getTopicPoliciesService().getTopicPolicies(topicName);
- } catch (BrokerServiceException.TopicPoliciesCacheNotInitException e) {
- log.error("Topic {} policies have not been initialized yet.", topicName);
- return FutureUtil.failedFuture(new RestException(Status.PRECONDITION_FAILED,
- "Policies have not been initialized yet"));
- }
- if (topicPolicies == null) {
- topicPolicies = new TopicPolicies();
- }
- topicPolicies.setInactiveTopicPolicies(inactiveTopicPolicies);
- return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ return getTopicPoliciesAsyncWithRetry(topicName)
+ .thenCompose(op -> {
+ TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
+ topicPolicies.setInactiveTopicPolicies(inactiveTopicPolicies);
+ return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ });
}
private CompletableFuture<Void> internalUpdateOffloadPolicies(OffloadPoliciesImpl offloadPolicies,
@@ -896,18 +872,12 @@ public class PersistentTopicsBase extends AdminResource {
"maxUnackedNum must be 0 or more");
}
- TopicPolicies topicPolicies = null;
- try {
- topicPolicies = pulsar().getTopicPoliciesService().getTopicPolicies(topicName);
- } catch (BrokerServiceException.TopicPoliciesCacheNotInitException e) {
- log.error("Topic {} policies have not been initialized yet.", topicName);
- throw new RestException(Status.PRECONDITION_FAILED, "Policies have not been initialized yet");
- }
- if (topicPolicies == null) {
- topicPolicies = new TopicPolicies();
- }
- topicPolicies.setMaxUnackedMessagesOnSubscription(maxUnackedNum);
- return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ return getTopicPoliciesAsyncWithRetry(topicName)
+ .thenCompose(op -> {
+ TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
+ topicPolicies.setMaxUnackedMessagesOnSubscription(maxUnackedNum);
+ return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ });
}
protected CompletableFuture<Integer> internalGetMaxUnackedMessagesOnConsumer(boolean applied) {
@@ -928,19 +898,12 @@ public class PersistentTopicsBase extends AdminResource {
"maxUnackedNum must be 0 or more");
}
- TopicPolicies topicPolicies = null;
- try {
- topicPolicies = pulsar().getTopicPoliciesService().getTopicPolicies(topicName);
- } catch (BrokerServiceException.TopicPoliciesCacheNotInitException e) {
- log.error("Topic {} policies have not been initialized yet.", topicName);
- return FutureUtil.failedFuture(new RestException(Status.PRECONDITION_FAILED,
- "Policies have not been initialized yet"));
- }
- if (topicPolicies == null) {
- topicPolicies = new TopicPolicies();
- }
- topicPolicies.setMaxUnackedMessagesOnConsumer(maxUnackedNum);
- return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ return getTopicPoliciesAsyncWithRetry(topicName)
+ .thenCompose(op -> {
+ TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
+ topicPolicies.setMaxUnackedMessagesOnConsumer(maxUnackedNum);
+ return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ });
}
protected CompletableFuture<Void> internalSetDeduplicationSnapshotInterval(Integer interval) {
@@ -2724,18 +2687,12 @@ public class PersistentTopicsBase extends AdminResource {
}
protected CompletableFuture<Void> internalSetDeduplication(Boolean enabled) {
- TopicPolicies topicPolicies = null;
- try {
- topicPolicies = pulsar().getTopicPoliciesService().getTopicPolicies(topicName);
- } catch (BrokerServiceException.TopicPoliciesCacheNotInitException e) {
- log.error("Topic {} policies have not been initialized yet.", topicName);
- throw new RestException(Status.PRECONDITION_FAILED, "Policies have not been initialized yet");
- }
- if (topicPolicies == null) {
- topicPolicies = new TopicPolicies();
- }
- topicPolicies.setDeduplicationEnabled(enabled);
- return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ return getTopicPoliciesAsyncWithRetry(topicName)
+ .thenCompose(op -> {
+ TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
+ topicPolicies.setDeduplicationEnabled(enabled);
+ return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ });
}
protected CompletableFuture<Void> internalSetMessageTTL(Integer ttlInSecond) {
[pulsar] 03/04: Add test for auto-created partitioned system topic
(#11545)
Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit c4061539c3e57c91693570cf1ffc24ff019f0bf7
Author: Zhanpeng Wu <zh...@qq.com>
AuthorDate: Thu Aug 5 20:55:58 2021 +0800
Add test for auto-created partitioned system topic (#11545)
(cherry picked from commit 27dcb0a6febd0d88438e18fae72aad8ee38c0738)
---
.../systopic/PartitionedSystemTopicTest.java | 69 ++++++++++++++++++++++
.../pulsar/common/events/EventsTopicNames.java | 2 +-
2 files changed, 70 insertions(+), 1 deletion(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java
new file mode 100644
index 0000000..bd4ef78
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.systopic;
+
+import org.apache.pulsar.broker.service.BrokerTestBase;
+import org.apache.pulsar.common.events.EventsTopicNames;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class PartitionedSystemTopicTest extends BrokerTestBase {
+
+ static final int PARTITIONS = 5;
+
+ @BeforeMethod
+ @Override
+ protected void setup() throws Exception {
+ resetConfig();
+ conf.setAllowAutoTopicCreation(false);
+ conf.setAllowAutoTopicCreationType("partitioned");
+ conf.setDefaultNumPartitions(PARTITIONS);
+
+ conf.setSystemTopicEnabled(true);
+ conf.setTopicLevelPoliciesEnabled(true);
+
+ super.baseSetup();
+ }
+
+ @AfterMethod(alwaysRun = true)
+ @Override
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ @Test
+ public void testAutoCreatedPartitionedSystemTopic() throws Exception {
+ final String ns = "prop/ns-test";
+ admin.namespaces().createNamespace(ns, 2);
+ NamespaceEventsSystemTopicFactory systemTopicFactory = new NamespaceEventsSystemTopicFactory(pulsarClient);
+ TopicPoliciesSystemTopicClient systemTopicClientForNamespace = systemTopicFactory
+ .createTopicPoliciesSystemTopicClient(NamespaceName.get(ns));
+ SystemTopicClient.Reader reader = systemTopicClientForNamespace.newReader();
+
+ int partitions = admin.topics().getPartitionedTopicMetadata(
+ String.format("persistent://%s/%s", ns, EventsTopicNames.NAMESPACE_EVENTS_LOCAL_NAME)).partitions;
+ Assert.assertEquals(admin.topics().getPartitionedTopicList(ns).size(), 1);
+ Assert.assertEquals(partitions, PARTITIONS);
+ Assert.assertEquals(admin.topics().getList(ns).size(), PARTITIONS);
+ }
+}
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/events/EventsTopicNames.java b/pulsar-common/src/main/java/org/apache/pulsar/common/events/EventsTopicNames.java
index cf37a30..5c1440b 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/events/EventsTopicNames.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/events/EventsTopicNames.java
@@ -48,6 +48,6 @@ public class EventsTopicNames {
new HashSet<>(Arrays.asList(NAMESPACE_EVENTS_LOCAL_NAME, TRANSACTION_BUFFER_SNAPSHOT)));
public static boolean checkTopicIsEventsNames(TopicName topicName) {
- return EVENTS_TOPIC_NAMES.contains(topicName.getLocalName());
+ return EVENTS_TOPIC_NAMES.contains(TopicName.get(topicName.getPartitionedTopicName()).getLocalName());
}
}