You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2022/11/09 06:17:07 UTC
[pulsar] branch branch-2.9 updated (a87fadd25c2 -> 58d0cbf2d1a)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a change to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/pulsar.git
from a87fadd25c2 cherry-pick/#17659 (#18391)
new 88b1a96412f Ensure cache is refreshed (and not just invalidated) after a store write (#12788)
new 58d0cbf2d1a [fix][broker] Fix issue where leader broker information isn't available after 10 minutes (#17401)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
...ncyTest.java => MultiBrokerTestZKBaseTest.java} | 31 +-----
.../MultiBrokerLeaderElectionExpirationTest.java | 114 +++++++++++++++++++++
.../loadbalance/MultiBrokerLeaderElectionTest.java | 38 +------
.../apache/pulsar/metadata/api/MetadataCache.java | 7 ++
...taStoreConfig.java => MetadataCacheConfig.java} | 20 ++--
.../apache/pulsar/metadata/api/MetadataStore.java | 57 ++++++++++-
.../metadata/cache/impl/MetadataCacheImpl.java | 41 +++++---
.../coordination/impl/LeaderElectionImpl.java | 28 +++--
.../metadata/impl/AbstractMetadataStore.java | 15 +--
.../metadata/impl/FaultInjectionMetadataStore.java | 13 +--
.../apache/pulsar/metadata/MetadataCacheTest.java | 34 +++++-
11 files changed, 285 insertions(+), 113 deletions(-)
copy pulsar-broker/src/test/java/org/apache/pulsar/broker/{zookeeper/MultiBrokerMetadataConsistencyTest.java => MultiBrokerTestZKBaseTest.java} (65%)
create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionExpirationTest.java
copy pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/{MetadataStoreConfig.java => MetadataCacheConfig.java} (51%)
[pulsar] 02/02: [fix][broker] Fix issue where leader broker information isn't available after 10 minutes (#17401)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 58d0cbf2d1add775fca753a771883d31ba82edc5
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Wed Nov 9 08:08:10 2022 +0200
[fix][broker] Fix issue where leader broker information isn't available after 10 minutes (#17401)
* [fix][broker] Leader election cache should not invalidate entries
- cache expiration leads to problems and it is better to serve a stale entry than to ever expire the
entry
- MetadataStore didn't support passing cache configuration for a MetadataCache, it
was necessary to modify the interface to support that.
* Add test case
* Prevent stale values in leader election cache
Co-authored-by: Enrico Olivelli <eo...@apache.org>
(cherry picked from commit 774c50b955d5be799ffbeee8c10d79f7dcb31d3b)
# Conflicts:
# pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
# pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
# pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
(cherry picked from commit cd731f451cb80e21cadf732ad3e39ee98b87bd45)
---
.../pulsar/broker/MultiBrokerTestZKBaseTest.java | 61 +++++++++++
.../MultiBrokerLeaderElectionExpirationTest.java | 114 +++++++++++++++++++++
.../loadbalance/MultiBrokerLeaderElectionTest.java | 38 +------
.../pulsar/metadata/api/MetadataCacheConfig.java | 50 +++++++++
.../apache/pulsar/metadata/api/MetadataStore.java | 57 ++++++++++-
.../metadata/cache/impl/MetadataCacheImpl.java | 24 +++--
.../coordination/impl/LeaderElectionImpl.java | 28 +++--
.../metadata/impl/AbstractMetadataStore.java | 13 +--
.../metadata/impl/FaultInjectionMetadataStore.java | 13 +--
9 files changed, 327 insertions(+), 71 deletions(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerTestZKBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerTestZKBaseTest.java
new file mode 100644
index 00000000000..e6cf86c05b5
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerTestZKBaseTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.metadata.TestZKServer;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+
+/**
+ * Multiple brokers with a real test Zookeeper server (instead of the mock server)
+ */
+@Slf4j
+public abstract class MultiBrokerTestZKBaseTest extends MultiBrokerBaseTest {
+ TestZKServer testZKServer;
+
+ @Override
+ protected void doInitConf() throws Exception {
+ super.doInitConf();
+ testZKServer = new TestZKServer();
+ }
+
+ @Override
+ protected void onCleanup() {
+ super.onCleanup();
+ if (testZKServer != null) {
+ try {
+ testZKServer.close();
+ } catch (Exception e) {
+ log.error("Error in stopping ZK server", e);
+ }
+ }
+ }
+
+ @Override
+ protected MetadataStoreExtended createLocalMetadataStore() throws MetadataStoreException {
+ return MetadataStoreExtended.create(testZKServer.getConnectionString(), MetadataStoreConfig.builder().build());
+ }
+
+ @Override
+ protected MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException {
+ return MetadataStoreExtended.create(testZKServer.getConnectionString(), MetadataStoreConfig.builder().build());
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionExpirationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionExpirationTest.java
new file mode 100644
index 00000000000..65962868b87
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionExpirationTest.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pulsar.broker.loadbalance;
+
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.MultiBrokerTestZKBaseTest;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.metadata.api.MetadataCacheConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.awaitility.Awaitility;
+import org.testng.annotations.Test;
+
+@Slf4j
+@Test(groups = "broker")
+public class MultiBrokerLeaderElectionExpirationTest extends MultiBrokerTestZKBaseTest {
+ private static final long EXPIRE_AFTER_WRITE_MILLIS_IN_TEST = 2000L;
+ private static final long REFRESH_AFTER_WRITE_MILLIS_IN_TEST = 1000L;
+
+ @Override
+ protected int numberOfAdditionalBrokers() {
+ return 9;
+ }
+
+ @Test
+ public void shouldElectOneLeader() {
+ int leaders = 0;
+ for (PulsarService broker : getAllBrokers()) {
+ if (broker.getLeaderElectionService().isLeader()) {
+ leaders++;
+ }
+ }
+ assertEquals(leaders, 1);
+ }
+
+ @Override
+ protected MetadataStoreExtended createLocalMetadataStore() throws MetadataStoreException {
+ return changeDefaultMetadataCacheConfig(super.createLocalMetadataStore());
+ }
+
+ @Override
+ protected MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException {
+ return changeDefaultMetadataCacheConfig(super.createConfigurationMetadataStore());
+ }
+
+ MetadataStoreExtended changeDefaultMetadataCacheConfig(MetadataStoreExtended metadataStore) {
+ MetadataStoreExtended spy = spy(metadataStore);
+ when(spy.getDefaultMetadataCacheConfig()).thenReturn(MetadataCacheConfig
+ .builder()
+ .refreshAfterWriteMillis(REFRESH_AFTER_WRITE_MILLIS_IN_TEST)
+ .expireAfterWriteMillis(EXPIRE_AFTER_WRITE_MILLIS_IN_TEST)
+ .build());
+ return spy;
+ }
+
+ @Test
+ public void shouldAllBrokersBeAbleToGetTheLeaderAfterExpiration()
+ throws ExecutionException, InterruptedException, TimeoutException {
+
+ // if you want to see this test fail, modify the line in LeaderElectionImpl constructor for creating
+ // the metadata cache to not skip expirations:
+ // this.cache = store.getMetadataCache(clazz);
+
+ // Given that all brokers have the leader elected
+ Awaitility.await().untilAsserted(() -> {
+ for (PulsarService broker : getAllBrokers()) {
+ Optional<LeaderBroker> currentLeader = broker.getLeaderElectionService().getCurrentLeader();
+ assertTrue(currentLeader.isPresent(), "Leader wasn't known on broker " + broker.getBrokerServiceUrl());
+ }
+ });
+
+ // Wait for metadata cache entries to expire
+ Thread.sleep(EXPIRE_AFTER_WRITE_MILLIS_IN_TEST);
+
+ // then leader should be known on all brokers and it should be the same leader
+ LeaderBroker leader = null;
+ for (PulsarService broker : getAllBrokers()) {
+ Optional<LeaderBroker> currentLeader =
+ broker.getLeaderElectionService().readCurrentLeader().get(1, TimeUnit.SECONDS);
+ assertTrue(currentLeader.isPresent(), "Leader wasn't known on broker " + broker.getBrokerServiceUrl());
+ if (leader != null) {
+ assertEquals(currentLeader.get(), leader,
+ "Different leader on broker " + broker.getBrokerServiceUrl());
+ } else {
+ leader = currentLeader.get();
+ }
+ }
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java
index 462b640c175..36ed1ca6c61 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java
@@ -34,55 +34,21 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import lombok.Cleanup;
import lombok.extern.slf4j.Slf4j;
-import org.apache.pulsar.broker.MultiBrokerBaseTest;
+import org.apache.pulsar.broker.MultiBrokerTestZKBaseTest;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.metadata.TestZKServer;
-import org.apache.pulsar.metadata.api.MetadataStoreConfig;
-import org.apache.pulsar.metadata.api.MetadataStoreException;
-import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
import org.awaitility.Awaitility;
import org.testng.annotations.Test;
@Slf4j
@Test(groups = "broker")
-public class MultiBrokerLeaderElectionTest extends MultiBrokerBaseTest {
+public class MultiBrokerLeaderElectionTest extends MultiBrokerTestZKBaseTest {
@Override
protected int numberOfAdditionalBrokers() {
return 9;
}
- TestZKServer testZKServer;
-
- @Override
- protected void doInitConf() throws Exception {
- super.doInitConf();
- testZKServer = new TestZKServer();
- }
-
- @Override
- protected void onCleanup() {
- super.onCleanup();
- if (testZKServer != null) {
- try {
- testZKServer.close();
- } catch (Exception e) {
- log.error("Error in stopping ZK server", e);
- }
- }
- }
-
- @Override
- protected MetadataStoreExtended createLocalMetadataStore() throws MetadataStoreException {
- return MetadataStoreExtended.create(testZKServer.getConnectionString(), MetadataStoreConfig.builder().build());
- }
-
- @Override
- protected MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException {
- return MetadataStoreExtended.create(testZKServer.getConnectionString(), MetadataStoreConfig.builder().build());
- }
-
@Test
public void shouldElectOneLeader() {
int leaders = 0;
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCacheConfig.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCacheConfig.java
new file mode 100644
index 00000000000..1671fd39956
--- /dev/null
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCacheConfig.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata.api;
+
+import java.util.concurrent.TimeUnit;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.ToString;
+
+/**
+ * The configuration builder for a {@link MetadataCache} config.
+ */
+@Builder
+@Getter
+@ToString
+public class MetadataCacheConfig {
+ private static final long DEFAULT_CACHE_REFRESH_TIME_MILLIS = TimeUnit.MINUTES.toMillis(5);
+
+ /**
+ * Specifies that active entries are eligible for automatic refresh once a fixed duration has
+ * elapsed after the entry's creation, or the most recent replacement of its value.
+ * A negative or zero value disables automatic refresh.
+ */
+ @Builder.Default
+ private final long refreshAfterWriteMillis = DEFAULT_CACHE_REFRESH_TIME_MILLIS;
+
+ /**
+ * Specifies that each entry should be automatically removed from the cache once a fixed duration
+ * has elapsed after the entry's creation, or the most recent replacement of its value.
+ * A negative or zero value disables automatic expiration.
+ */
+ @Builder.Default
+ private final long expireAfterWriteMillis = 2 * DEFAULT_CACHE_REFRESH_TIME_MILLIS;
+}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java
index fbebc7b1741..8c0f24a571c 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java
@@ -138,9 +138,23 @@ public interface MetadataStore extends AutoCloseable {
* @param <T>
* @param clazz
* the class type to be used for serialization/deserialization
+ * @param cacheConfig
+ * the cache configuration to be used
* @return the metadata cache object
*/
- <T> MetadataCache<T> getMetadataCache(Class<T> clazz);
+ <T> MetadataCache<T> getMetadataCache(Class<T> clazz, MetadataCacheConfig cacheConfig);
+
+ /**
+ * Create a metadata cache specialized for a specific class.
+ *
+ * @param <T>
+ * @param clazz
+ * the class type to be used for serialization/deserialization
+ * @return the metadata cache object
+ */
+ default <T> MetadataCache<T> getMetadataCache(Class<T> clazz) {
+ return getMetadataCache(clazz, getDefaultMetadataCacheConfig());
+ }
/**
* Create a metadata cache specialized for a specific class.
@@ -148,9 +162,23 @@ public interface MetadataStore extends AutoCloseable {
* @param <T>
* @param typeRef
* the type ref description to be used for serialization/deserialization
+ * @param cacheConfig
+ * the cache configuration to be used
* @return the metadata cache object
*/
- <T> MetadataCache<T> getMetadataCache(TypeReference<T> typeRef);
+ <T> MetadataCache<T> getMetadataCache(TypeReference<T> typeRef, MetadataCacheConfig cacheConfig);
+
+ /**
+ * Create a metadata cache specialized for a specific class.
+ *
+ * @param <T>
+ * @param typeRef
+ * the type ref description to be used for serialization/deserialization
+ * @return the metadata cache object
+ */
+ default <T> MetadataCache<T> getMetadataCache(TypeReference<T> typeRef) {
+ return getMetadataCache(typeRef, getDefaultMetadataCacheConfig());
+ }
/**
* Create a metadata cache that uses a particular serde object.
@@ -158,7 +186,30 @@ public interface MetadataStore extends AutoCloseable {
* @param <T>
* @param serde
* the custom serialization/deserialization object
+ * @param cacheConfig
+ * the cache configuration to be used
* @return the metadata cache object
*/
- <T> MetadataCache<T> getMetadataCache(MetadataSerde<T> serde);
+ <T> MetadataCache<T> getMetadataCache(MetadataSerde<T> serde, MetadataCacheConfig cacheConfig);
+
+ /**
+ * Create a metadata cache that uses a particular serde object.
+ *
+ * @param <T>
+ * @param serde
+ * the custom serialization/deserialization object
+ * @return the metadata cache object
+ */
+ default <T> MetadataCache<T> getMetadataCache(MetadataSerde<T> serde) {
+ return getMetadataCache(serde, getDefaultMetadataCacheConfig());
+ }
+
+ /**
+ * Returns the default metadata cache config.
+ *
+ * @return default metadata cache config
+ */
+ default MetadataCacheConfig getDefaultMetadataCacheConfig() {
+ return MetadataCacheConfig.builder().build();
+ }
}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
index 0ab56bb1bc1..49c8680550c 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
@@ -39,6 +39,7 @@ import org.apache.bookkeeper.common.concurrent.FutureUtils;
import org.apache.pulsar.metadata.api.CacheGetResult;
import org.apache.pulsar.metadata.api.GetResult;
import org.apache.pulsar.metadata.api.MetadataCache;
+import org.apache.pulsar.metadata.api.MetadataCacheConfig;
import org.apache.pulsar.metadata.api.MetadataSerde;
import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyExistsException;
@@ -50,29 +51,32 @@ import org.apache.pulsar.metadata.impl.AbstractMetadataStore;
@Slf4j
public class MetadataCacheImpl<T> implements MetadataCache<T>, Consumer<Notification> {
-
- private static final long CACHE_REFRESH_TIME_MILLIS = TimeUnit.MINUTES.toMillis(5);
-
@Getter
private final MetadataStore store;
private final MetadataSerde<T> serde;
private final AsyncLoadingCache<String, Optional<CacheGetResult<T>>> objCache;
- public MetadataCacheImpl(MetadataStore store, TypeReference<T> typeRef) {
- this(store, new JSONMetadataSerdeTypeRef<>(typeRef));
+ public MetadataCacheImpl(MetadataStore store, TypeReference<T> typeRef, MetadataCacheConfig cacheConfig) {
+ this(store, new JSONMetadataSerdeTypeRef<>(typeRef), cacheConfig);
}
- public MetadataCacheImpl(MetadataStore store, JavaType type) {
- this(store, new JSONMetadataSerdeSimpleType<>(type));
+ public MetadataCacheImpl(MetadataStore store, JavaType type, MetadataCacheConfig cacheConfig) {
+ this(store, new JSONMetadataSerdeSimpleType<>(type), cacheConfig);
}
- public MetadataCacheImpl(MetadataStore store, MetadataSerde<T> serde) {
+ public MetadataCacheImpl(MetadataStore store, MetadataSerde<T> serde, MetadataCacheConfig cacheConfig) {
this.store = store;
this.serde = serde;
- this.objCache = Caffeine.newBuilder()
- .refreshAfterWrite(CACHE_REFRESH_TIME_MILLIS, TimeUnit.MILLISECONDS)
+ Caffeine<Object, Object> cacheBuilder = Caffeine.newBuilder();
+ if (cacheConfig.getRefreshAfterWriteMillis() > 0) {
+ cacheBuilder.refreshAfterWrite(cacheConfig.getRefreshAfterWriteMillis(), TimeUnit.MILLISECONDS);
+ }
+ if (cacheConfig.getExpireAfterWriteMillis() > 0) {
+ cacheBuilder.expireAfterWrite(cacheConfig.getExpireAfterWriteMillis(), TimeUnit.MILLISECONDS);
+ }
+ this.objCache = cacheBuilder
.buildAsync(new AsyncCacheLoader<String, Optional<CacheGetResult<T>>>() {
@Override
public CompletableFuture<Optional<CacheGetResult<T>>> asyncLoad(String key, Executor executor) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
index 6599c625f78..5e60b14f358 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
@@ -31,6 +31,7 @@ import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
@@ -41,6 +42,7 @@ import org.apache.bookkeeper.common.util.SafeRunnable;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.GetResult;
import org.apache.pulsar.metadata.api.MetadataCache;
+import org.apache.pulsar.metadata.api.MetadataCacheConfig;
import org.apache.pulsar.metadata.api.MetadataStoreException;
import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyClosedException;
import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException;
@@ -61,6 +63,7 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
private final MetadataStoreExtended store;
private final MetadataCache<T> cache;
private final Consumer<LeaderElectionState> stateChangesListener;
+ private final ScheduledFuture<?> updateCachedValueFuture;
private LeaderElectionState leaderElectionState;
private Optional<Long> version = Optional.empty();
@@ -82,7 +85,10 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
this.path = path;
this.serde = new JSONMetadataSerdeSimpleType<>(TypeFactory.defaultInstance().constructSimpleType(clazz, null));
this.store = store;
- this.cache = store.getMetadataCache(clazz);
+ MetadataCacheConfig metadataCacheConfig = MetadataCacheConfig.builder()
+ .expireAfterWriteMillis(-1L)
+ .build();
+ this.cache = store.getMetadataCache(clazz, metadataCacheConfig);
this.leaderElectionState = LeaderElectionState.NoLeader;
this.internalState = InternalState.Init;
this.stateChangesListener = stateChangesListener;
@@ -90,6 +96,9 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
store.registerListener(this::handlePathNotification);
store.registerSessionListener(this::handleSessionNotification);
+ updateCachedValueFuture = executor.scheduleWithFixedDelay(SafeRunnable.safeRun(this::getLeaderValue),
+ metadataCacheConfig.getRefreshAfterWriteMillis() / 2,
+ metadataCacheConfig.getRefreshAfterWriteMillis(), TimeUnit.MILLISECONDS);
}
@Override
@@ -111,10 +120,13 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
} else {
return tryToBecomeLeader();
}
- }).thenCompose(leaderElectionState ->
- // make sure that the cache contains the current leader
- // so that getLeaderValueIfPresent works on all brokers
- cache.get(path).thenApply(__ -> leaderElectionState));
+ }).thenComposeAsync(leaderElectionState -> {
+ // make sure that the cache contains the current leader
+ // so that getLeaderValueIfPresent works on all brokers
+ cache.refresh(path);
+ return cache.get(path)
+ .thenApply(__ -> leaderElectionState);
+ }, executor);
}
private synchronized CompletableFuture<LeaderElectionState> handleExistingLeaderValue(GetResult res) {
@@ -216,11 +228,6 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
// There was a conflict between 2 participants trying to become leaders at same time. Retry
// to fetch info on new leader.
- // We force the invalidation of the cache entry. Since we received a BadVersion error, we
- // already know that the entry is out of date. If we don't invalidate, we'd be retrying the
- // leader election many times until we finally receive the notification that invalidates the
- // cache.
- cache.invalidate(path);
elect()
.thenAccept(lse -> result.complete(lse))
.exceptionally(ex2 -> {
@@ -238,6 +245,7 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
@Override
public void close() throws Exception {
+ updateCachedValueFuture.cancel(true);
try {
asyncClose().join();
} catch (CompletionException e) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
index ac7feb4747f..bd871605e25 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
@@ -46,6 +46,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.MetadataCache;
+import org.apache.pulsar.metadata.api.MetadataCacheConfig;
import org.apache.pulsar.metadata.api.MetadataSerde;
import org.apache.pulsar.metadata.api.Notification;
import org.apache.pulsar.metadata.api.NotificationType;
@@ -123,23 +124,23 @@ public abstract class AbstractMetadataStore implements MetadataStoreExtended, Co
}
@Override
- public <T> MetadataCache<T> getMetadataCache(Class<T> clazz) {
+ public <T> MetadataCache<T> getMetadataCache(Class<T> clazz, MetadataCacheConfig cacheConfig) {
MetadataCacheImpl<T> metadataCache = new MetadataCacheImpl<T>(this,
- TypeFactory.defaultInstance().constructSimpleType(clazz, null));
+ TypeFactory.defaultInstance().constructSimpleType(clazz, null), cacheConfig);
metadataCaches.add(metadataCache);
return metadataCache;
}
@Override
- public <T> MetadataCache<T> getMetadataCache(TypeReference<T> typeRef) {
- MetadataCacheImpl<T> metadataCache = new MetadataCacheImpl<T>(this, typeRef);
+ public <T> MetadataCache<T> getMetadataCache(TypeReference<T> typeRef, MetadataCacheConfig cacheConfig) {
+ MetadataCacheImpl<T> metadataCache = new MetadataCacheImpl<T>(this, typeRef, cacheConfig);
metadataCaches.add(metadataCache);
return metadataCache;
}
@Override
- public <T> MetadataCache<T> getMetadataCache(MetadataSerde<T> serde) {
- MetadataCacheImpl<T> metadataCache = new MetadataCacheImpl<>(this, serde);
+ public <T> MetadataCache<T> getMetadataCache(MetadataSerde<T> serde, MetadataCacheConfig cacheConfig) {
+ MetadataCacheImpl<T> metadataCache = new MetadataCacheImpl<>(this, serde, cacheConfig);
metadataCaches.add(metadataCache);
return metadataCache;
}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/FaultInjectionMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/FaultInjectionMetadataStore.java
index 65731c0fed8..e351433feb9 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/FaultInjectionMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/FaultInjectionMetadataStore.java
@@ -31,6 +31,7 @@ import lombok.Data;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.GetResult;
import org.apache.pulsar.metadata.api.MetadataCache;
+import org.apache.pulsar.metadata.api.MetadataCacheConfig;
import org.apache.pulsar.metadata.api.MetadataSerde;
import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.MetadataStoreException;
@@ -147,18 +148,18 @@ public class FaultInjectionMetadataStore implements MetadataStoreExtended {
}
@Override
- public <T> MetadataCache<T> getMetadataCache(Class<T> clazz) {
- return store.getMetadataCache(clazz);
+ public <T> MetadataCache<T> getMetadataCache(Class<T> clazz, MetadataCacheConfig cacheConfig) {
+ return store.getMetadataCache(clazz, cacheConfig);
}
@Override
- public <T> MetadataCache<T> getMetadataCache(TypeReference<T> typeRef) {
- return store.getMetadataCache(typeRef);
+ public <T> MetadataCache<T> getMetadataCache(TypeReference<T> typeRef, MetadataCacheConfig cacheConfig) {
+ return store.getMetadataCache(typeRef, cacheConfig);
}
@Override
- public <T> MetadataCache<T> getMetadataCache(MetadataSerde<T> serde) {
- return store.getMetadataCache(serde);
+ public <T> MetadataCache<T> getMetadataCache(MetadataSerde<T> serde, MetadataCacheConfig cacheConfig) {
+ return store.getMetadataCache(serde, cacheConfig);
}
@Override
[pulsar] 01/02: Ensure cache is refreshed (and not just invalidated) after a store write (#12788)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 88b1a96412f6d1e5e32c3eca8cf390f59fdfd725
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Tue Nov 16 16:15:07 2021 -0800
Ensure cache is refreshed (and not just invalidated) after a store write (#12788)
### Motivation
When we're doing a write to the store from outside the `MetadataCache`, we are immediately invalidating the cache to ensure read-after-write consistency through the cache.
The only issue is that the invalidation, will not trigger a reloading of the value. Instead it is relying on the next call to `cache.get()` which will see the cache miss and it will load the new value into the cache.
This means that calls `cache.getIfCached()`, which is not triggering a cache load, will keep seeing the key as missing.
### Modification
Ensure we're calling refresh on the cache to get the value automatically reloaded in background and make sure the `getIfCached()` will eventually return the new value, even if there are no calls to `cache.get()`.
(cherry picked from commit 2bc449933f72f28dfae24ca8a6fb022be7c55a44)
---
.../apache/pulsar/metadata/api/MetadataCache.java | 7 +++++
.../metadata/cache/impl/MetadataCacheImpl.java | 17 ++++++-----
.../metadata/impl/AbstractMetadataStore.java | 2 +-
.../apache/pulsar/metadata/MetadataCacheTest.java | 34 ++++++++++++++++++++--
4 files changed, 50 insertions(+), 10 deletions(-)
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java
index 360c092b6f1..1272130eb76 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java
@@ -148,4 +148,11 @@ public interface MetadataCache<T> {
* @param path the path of the object in the metadata store
*/
void invalidate(String path);
+
+ /**
+ * Invalidate and reload an object in the metadata cache.
+ *
+ * @param path the path of the object in the metadata store
+ */
+ void refresh(String path);
}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
index 16b419fe9ed..0ab56bb1bc1 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
@@ -168,8 +168,7 @@ public class MetadataCacheImpl<T> implements MetadataCache<T>, Consumer<Notifica
}
return store.put(path, newValue, Optional.of(expectedVersion)).thenAccept(__ -> {
- objCache.synchronous().invalidate(path);
- objCache.synchronous().refresh(path);
+ refresh(path);
}).thenApply(__ -> newValueObj);
}), path);
}
@@ -198,8 +197,7 @@ public class MetadataCacheImpl<T> implements MetadataCache<T>, Consumer<Notifica
}
return store.put(path, newValue, Optional.of(expectedVersion)).thenAccept(__ -> {
- objCache.synchronous().invalidate(path);
- objCache.synchronous().refresh(path);
+ refresh(path);
}).thenApply(__ -> newValueObj);
}), path);
}
@@ -220,7 +218,7 @@ public class MetadataCacheImpl<T> implements MetadataCache<T>, Consumer<Notifica
// In addition to caching the value, we need to add a watch on the path,
// so when/if it changes on any other node, we are notified and we can
// update the cache
- objCache.get(path).whenComplete( (stat2, ex) -> {
+ objCache.get(path).whenComplete((stat2, ex) -> {
if (ex == null) {
future.complete(null);
} else {
@@ -261,6 +259,12 @@ public class MetadataCacheImpl<T> implements MetadataCache<T>, Consumer<Notifica
objCache.synchronous().invalidate(path);
}
+ @Override
+ public void refresh(String path) {
+ objCache.synchronous().invalidate(path);
+ objCache.synchronous().refresh(path);
+ }
+
@VisibleForTesting
public void invalidateAll() {
objCache.synchronous().invalidateAll();
@@ -275,8 +279,7 @@ public class MetadataCacheImpl<T> implements MetadataCache<T>, Consumer<Notifica
if (objCache.synchronous().getIfPresent(path) != null) {
// Trigger background refresh of the cached item, but before make sure
// to invalidate the entry so that we won't serve a stale cached version
- objCache.synchronous().invalidate(path);
- objCache.synchronous().refresh(path);
+ refresh(path);
}
break;
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
index 40c209f4d3e..ac7feb4747f 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java
@@ -252,7 +252,7 @@ public abstract class AbstractMetadataStore implements MetadataStoreExtended, Co
}
}
- metadataCaches.forEach(c -> c.invalidate(path));
+ metadataCaches.forEach(c -> c.refresh(path));
return stat;
});
}
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java
index 322c9bb779b..e7680296563 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java
@@ -278,6 +278,34 @@ public class MetadataCacheTest extends BaseMetadataStoreTest {
assertEquals(objCache.get(key1).join(), Optional.empty());
}
+ @Test(dataProvider = "impl")
+ public void insertionWithInvalidation(String provider, Supplier<String> urlSupplier) throws Exception {
+ @Cleanup
+ MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().build());
+ MetadataCache<MyClass> objCache = store.getMetadataCache(MyClass.class);
+
+ String key1 = newKey();
+
+ assertEquals(objCache.getIfCached(key1), Optional.empty());
+ assertEquals(objCache.get(key1).join(), Optional.empty());
+
+ MyClass value1 = new MyClass("a", 1);
+ store.put(key1, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(value1), Optional.of(-1L)).join();
+
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(objCache.getIfCached(key1), Optional.of(value1));
+ assertEquals(objCache.get(key1).join(), Optional.of(value1));
+ });
+
+ MyClass value2 = new MyClass("a", 2);
+ store.put(key1, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(value2), Optional.of(0L)).join();
+
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(objCache.getIfCached(key1), Optional.of(value2));
+ assertEquals(objCache.get(key1).join(), Optional.of(value2));
+ });
+ }
+
@Test(dataProvider = "impl")
public void insertionOutsideCache(String provider, Supplier<String> urlSupplier) throws Exception {
@Cleanup
@@ -310,8 +338,10 @@ public class MetadataCacheTest extends BaseMetadataStoreTest {
v.put("b", "2");
store.put(key1, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(v), Optional.of(-1L)).join();
- assertEquals(objCache.getIfCached(key1), Optional.empty());
- assertEquals(objCache.get(key1).join(), Optional.of(v));
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(objCache.getIfCached(key1), Optional.of(v));
+ assertEquals(objCache.get(key1).join(), Optional.of(v));
+ });
}
@Test(dataProvider = "impl")