You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2026/03/04 18:39:56 UTC
(pulsar) branch master updated: [feat] PIP-454: Metadata Store Migration Framework (implementation) (#25219)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new e1afa05bc2a [feat] PIP-454: Metadata Store Migration Framework (implementation) (#25219)
e1afa05bc2a is described below
commit e1afa05bc2a0ee0dbce083632ef6311747a6bd3f
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Wed Mar 4 10:39:50 2026 -0800
[feat] PIP-454: Metadata Store Migration Framework (implementation) (#25219)
---
distribution/server/src/assemble/LICENSE.bin.txt | 4 +-
pom.xml | 2 +-
.../apache/pulsar/PulsarClusterMetadataSetup.java | 3 +-
.../broker/admin/impl/MetadataMigrationBase.java | 116 +++++
.../pulsar/broker/admin/v2/MetadataMigration.java | 34 ++
...ServiceUnitStateMetadataStoreTableViewImpl.java | 4 +-
.../broker/service/BrokerServiceChaosTest.java | 5 +-
.../CanReconnectZKClientPulsarServiceBaseTest.java | 12 +-
.../broker/service/OneWayReplicatorTest.java | 4 +-
.../pulsar/client/admin/MetadataMigration.java | 42 ++
.../apache/pulsar/client/admin/PulsarAdmin.java | 2 +
.../pulsar/common/migration/MigrationPhase.java | 52 ++
.../pulsar/common/migration/MigrationState.java | 46 ++
.../pulsar/common/migration/package-info.java | 22 +
.../admin/internal/MetadataMigrationImpl.java | 46 ++
.../client/admin/internal/PulsarAdminImpl.java | 8 +
.../pulsar/admin/cli/CmdMetadataMigration.java | 67 +++
.../apache/pulsar/admin/cli/PulsarAdminTool.java | 2 +
.../api/extended/MetadataStoreExtended.java | 8 +
.../bookkeeper/PulsarLedgerIdGenerator.java | 13 +-
.../PulsarLedgerUnderreplicationManager.java | 12 +-
.../bookkeeper/PulsarRegistrationClient.java | 6 +-
.../metadata/cache/impl/MetadataCacheImpl.java | 3 +-
.../coordination/impl/MigrationCoordinator.java | 228 +++++++++
.../pulsar/metadata/impl/DualMetadataCache.java | 142 ++++++
.../pulsar/metadata/impl/DualMetadataStore.java | 453 ++++++++++++++++++
.../pulsar/metadata/impl/ZKMetadataStore.java | 7 +-
.../impl/oxia/OxiaMetadataStoreProvider.java | 20 +-
.../tableview/impl/MetadataStoreTableViewImpl.java | 6 +-
.../AuditorPeriodicBookieCheckTest.java | 2 +-
.../replication/AutoRecoveryMainTest.java | 5 +-
.../replication/TestReplicationWorker.java | 4 +-
.../pulsar/metadata/DualMetadataCacheTest.java | 531 +++++++++++++++++++++
.../pulsar/metadata/DualMetadataStoreTest.java | 453 ++++++++++++++++++
.../apache/pulsar/metadata/MetadataCacheTest.java | 17 +-
.../apache/pulsar/metadata/MetadataStoreTest.java | 12 +-
.../pulsar/metadata/MigrationCoordinatorTest.java | 301 ++++++++++++
.../org/apache/pulsar/metadata/ZKSessionTest.java | 8 +-
38 files changed, 2657 insertions(+), 45 deletions(-)
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 4f582a1fd7b..32bcba96ba8 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -498,8 +498,8 @@ The Apache Software License, Version 2.0
* Prometheus
- io.prometheus-simpleclient_httpserver-0.16.0.jar
* Oxia
- - io.github.oxia-db-oxia-client-api-0.7.2.jar
- - io.github.oxia-db-oxia-client-0.7.2.jar
+ - io.github.oxia-db-oxia-client-api-0.7.4.jar
+ - io.github.oxia-db-oxia-client-0.7.4.jar
* OpenHFT
- net.openhft-zero-allocation-hashing-0.16.jar
* Java JSON WebTokens
diff --git a/pom.xml b/pom.xml
index 0d0e9039704..490b284649e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -298,7 +298,7 @@ flexible messaging model and an intuitive client API.</description>
<apache-http-client.version>4.5.13</apache-http-client.version>
<apache-httpcomponents.version>4.4.15</apache-httpcomponents.version>
<jetcd.version>0.7.7</jetcd.version>
- <oxia.version>0.7.2</oxia.version>
+ <oxia.version>0.7.4</oxia.version>
<snakeyaml.version>2.0</snakeyaml.version>
<ant.version>1.10.12</ant.version>
<seancfoley.ipaddress.version>5.5.0</seancfoley.ipaddress.version>
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
index ce3bffe63d9..31b12795e20 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
@@ -53,6 +53,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreLifecycle;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver;
import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.apache.pulsar.metadata.impl.MetadataStoreFactoryImpl;
import org.apache.pulsar.metadata.impl.ZKMetadataStore;
import org.slf4j.Logger;
@@ -315,7 +316,7 @@ public class PulsarClusterMetadataSetup {
}
}
- if (localStore instanceof ZKMetadataStore && configStore instanceof ZKMetadataStore) {
+ if (localStore instanceof DualMetadataStore && configStore instanceof DualMetadataStore) {
String uriStr;
if (arguments.existingBkMetadataServiceUri != null) {
uriStr = arguments.existingBkMetadataServiceUri;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/MetadataMigrationBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/MetadataMigrationBase.java
new file mode 100644
index 00000000000..5ecca5f481f
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/MetadataMigrationBase.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.impl;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Response;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.admin.AdminResource;
+import org.apache.pulsar.broker.web.RestException;
+import org.apache.pulsar.common.migration.MigrationState;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.metadata.coordination.impl.MigrationCoordinator;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
+
+/**
+ * Admin resource for metadata store migration operations.
+ */
+@Slf4j
+public class MetadataMigrationBase extends AdminResource {
+
+ @GET
+ @Path("/status")
+ @ApiOperation(value = "Get current migration status", response = MigrationState.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "Migration status retrieved successfully"),
+ @ApiResponse(code = 500, message = "Internal server error")
+ })
+ public MigrationState getStatus() {
+ validateSuperUserAccess();
+
+ try {
+ var ogr = pulsar().getLocalMetadataStore().get(MigrationState.MIGRATION_FLAG_PATH).get();
+ if (ogr.isPresent()) {
+ return ObjectMapperFactory.getMapper().reader().readValue(ogr.get().getValue(), MigrationState.class);
+ } else {
+ return MigrationState.NOT_STARTED;
+ }
+ } catch (Exception e) {
+ log.error("Failed to get migration status", e);
+ throw new RestException(e);
+ }
+ }
+
+ @POST
+ @Path("/start")
+ @ApiOperation(value = "Start metadata store migration")
+ @ApiResponses(value = {
+ @ApiResponse(code = 204, message = "Migration started successfully"),
+ @ApiResponse(code = 400, message = "Invalid target URL"),
+ @ApiResponse(code = 409, message = "Migration already in progress"),
+ @ApiResponse(code = 500, message = "Internal server error")
+ })
+ public void startMigration(
+ @ApiParam(value = "Target metadata store URL", required = true)
+ @QueryParam("target")
+ String targetUrl) {
+ validateSuperUserAccess();
+
+ if (targetUrl == null || targetUrl.trim().isEmpty()) {
+ throw new RestException(Response.Status.BAD_REQUEST, "Target URL is required");
+ }
+
+ try {
+ // Check if metadata store is wrapped with DualMetadataStore
+ if (!(pulsar().getLocalMetadataStore() instanceof DualMetadataStore)) {
+ throw new RestException(Response.Status.BAD_REQUEST, "Metadata store is not configured for migration. "
+ + "Please ensure you're using a supported source metadata store (e.g., ZooKeeper).");
+ }
+
+ // Create coordinator
+ MigrationCoordinator coordinator = new MigrationCoordinator(pulsar().getLocalMetadataStore(), targetUrl);
+
+ // Start migration in background thread
+ pulsar().getExecutor().submit(() -> {
+ try {
+ log.info("Starting metadata migration to: {}", targetUrl);
+ coordinator.startMigration();
+ log.info("Metadata migration completed successfully");
+ } catch (Exception e) {
+ log.error("Metadata migration failed", e);
+ }
+ });
+
+ log.info("Migration initiated to target: {}", targetUrl);
+
+ } catch (RestException e) {
+ throw e;
+ } catch (Exception e) {
+ log.error("Failed to start migration", e);
+ throw new RestException(e);
+ }
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/MetadataMigration.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/MetadataMigration.java
new file mode 100644
index 00000000000..6b9a516fc80
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/MetadataMigration.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.v2;
+
+import io.swagger.annotations.Api;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import org.apache.pulsar.broker.admin.impl.MetadataMigrationBase;
+
+/**
+ * REST API for metadata store migration operations.
+ */
+@Path("/metadata/migration")
+@Api(value = "/metadata/migration", description = "Metadata store migration admin APIs", tags = "metadata-migration")
+@Produces(MediaType.APPLICATION_JSON)
+public class MetadataMigration extends MetadataMigrationBase {
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java
index f70fbbb45c9..7822f033846 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java
@@ -36,7 +36,7 @@ import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.MetadataStoreException;
import org.apache.pulsar.metadata.api.MetadataStoreTableView;
-import org.apache.pulsar.metadata.impl.AbstractMetadataStore;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
import org.apache.pulsar.metadata.tableview.impl.MetadataStoreTableViewImpl;
@Slf4j
@@ -65,7 +65,7 @@ public class ServiceUnitStateMetadataStoreTableViewImpl extends ServiceUnitState
init(pulsar);
conflictResolver = new ServiceUnitStateDataConflictResolver();
conflictResolver.setStorageType(MetadataStore);
- if (!(pulsar.getLocalMetadataStore() instanceof AbstractMetadataStore)
+ if (!(pulsar.getLocalMetadataStore() instanceof MetadataStoreExtended)
&& !MetadataSessionExpiredPolicy.shutdown.equals(pulsar.getConfig().getZookeeperSessionExpiredPolicy())) {
String errorMsg = String.format("Your current metadata store [%s] does not support the registration of "
+ "session event listeners. Please set \"zookeeperSessionExpiredPolicy\" to \"shutdown\";"
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java
index 96df607d7e4..8e642c0c47d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java
@@ -28,6 +28,7 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride;
import org.apache.pulsar.common.policies.data.TopicType;
import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper;
@@ -55,7 +56,9 @@ public class BrokerServiceChaosTest extends CanReconnectZKClientPulsarServiceBas
@Test
public void testFetchPartitionedTopicMetadataWithCacheRefresh() throws Exception {
final String configMetadataStoreConnectString =
- WhiteboxImpl.getInternalState(pulsar.getConfigurationMetadataStore(), "zkConnectString");
+ WhiteboxImpl.getInternalState(
+ ((DualMetadataStore) pulsar.getConfigurationMetadataStore()).getSourceStore(),
+ "zkConnectString");
@Cleanup
final ZooKeeper anotherZKCli = new ZooKeeper(configMetadataStoreConnectString, 5000, null);
// Set policy of auto create topic to PARTITIONED.
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
index d3a780633c0..a9ce5e15146 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
@@ -35,7 +35,9 @@ import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.TenantInfoImpl;
import org.apache.pulsar.common.policies.data.TopicType;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
import org.apache.pulsar.metadata.api.extended.SessionEvent;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.apache.pulsar.metadata.impl.ZKMetadataStore;
import org.apache.pulsar.tests.TestRetrySupport;
import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
@@ -88,9 +90,13 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetr
pulsar = new PulsarService(config);
pulsar.start();
broker = pulsar.getBrokerService();
- ZKMetadataStore zkMetadataStore = (ZKMetadataStore) pulsar.getLocalMetadataStore();
- localZkOfBroker = zkMetadataStore.getZkClient();
- zkMetadataStore.registerSessionListener(n -> {
+ MetadataStoreExtended store = pulsar.getLocalMetadataStore();
+ if (store instanceof DualMetadataStore dms) {
+ localZkOfBroker = ((ZKMetadataStore) dms.getSourceStore()).getZkClient();
+ } else if (store instanceof ZKMetadataStore zkStore) {
+ localZkOfBroker = zkStore.getZkClient();
+ }
+ store.registerSessionListener(n -> {
log.info("Received session event: {}", n);
sessionEvent = n;
});
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
index 8971e42f1d1..ab4af0c56ca 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
@@ -116,6 +116,7 @@ import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl
import org.apache.pulsar.common.schema.SchemaInfo;
import org.apache.pulsar.common.schema.SchemaType;
import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.awaitility.Awaitility;
import org.awaitility.reflect.WhiteboxImpl;
import org.glassfish.jersey.client.JerseyClient;
@@ -1547,8 +1548,9 @@ public class OneWayReplicatorTest extends OneWayReplicatorTestBase {
(PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get();
// We inject an error to make "start replicator" to fail.
+ DualMetadataStore dms = (DualMetadataStore) pulsar1.getConfigurationMetadataStore();
AsyncLoadingCache<String, Boolean> existsCache =
- WhiteboxImpl.getInternalState(pulsar1.getConfigurationMetadataStore(), "existsCache");
+ WhiteboxImpl.getInternalState(dms.getSourceStore(), "existsCache");
String path = "/admin/partitioned-topics/" + TopicName.get(topicName).getPersistenceNamingEncoding();
existsCache.put(path, CompletableFuture.completedFuture(true));
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/MetadataMigration.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/MetadataMigration.java
new file mode 100644
index 00000000000..a44fcb0c87a
--- /dev/null
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/MetadataMigration.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.admin;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.common.migration.MigrationState;
+
+/**
+ * Handle cluster metadata migrations.
+ */
+public interface MetadataMigration {
+
+ /**
+ * Start metadata store migration.
+ *
+ * @return
+ */
+ CompletableFuture<Void> start(String targetUrl);
+
+ /**
+ * Get current migration status.
+ *
+ * @return
+ */
+ CompletableFuture<MigrationState> status();
+}
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdmin.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdmin.java
index 12d225676dc..7f49a55d829 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdmin.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdmin.java
@@ -174,6 +174,8 @@ public interface PulsarAdmin extends Closeable {
*/
Transactions transactions();
+ MetadataMigration metadataMigration();
+
/**
* Close the PulsarAdminClient and release all the resources.
*
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/migration/MigrationPhase.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/migration/MigrationPhase.java
new file mode 100644
index 00000000000..9798c421488
--- /dev/null
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/migration/MigrationPhase.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.migration;
+
+/**
+ * Represents the different phases of metadata store migration.
+ */
+public enum MigrationPhase {
+ /**
+ * No migration in progress. Operating normally on source store only.
+ */
+ NOT_STARTED,
+
+ /**
+ * Migration preparation phase. All brokers and bookies are recreating
+ * their ephemeral nodes in the target store.
+ */
+ PREPARATION,
+
+ /**
+ * Data copy phase. The migration coordinator is copying persistent
+ * data from source to target store.
+ */
+ COPYING,
+
+ /**
+ * Migration completed. All services are using target store. Source
+ * store can be decommissioned after configuration update and restart.
+ */
+ COMPLETED,
+
+ /**
+ * Migration has failed. System has rolled-back to used the old metadata store.
+ */
+ FAILED,
+}
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/migration/MigrationState.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/migration/MigrationState.java
new file mode 100644
index 00000000000..e634c8c320c
--- /dev/null
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/migration/MigrationState.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.migration;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+/**
+ * Persistent state of an ongoing metadata store migration.
+ */
+@Data
+@AllArgsConstructor
+public class MigrationState {
+ /**
+ * Current migration phase.
+ */
+ private MigrationPhase phase;
+
+ /**
+ * Target metadata store URL (e.g., "oxia://host:port/namespace").
+ */
+ private String targetUrl;
+
+ public static final MigrationState NOT_STARTED = new MigrationState(MigrationPhase.NOT_STARTED, null);
+
+
+
+ public static final String MIGRATION_FLAG_PATH = "/pulsar/migration-coordinator/migration";
+ public static final String PARTICIPANTS_PATH = "/pulsar/migration-coordinator/participants";
+}
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/migration/package-info.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/migration/package-info.java
new file mode 100644
index 00000000000..87bb87c72b6
--- /dev/null
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/migration/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Implementation of policies.
+ */
+package org.apache.pulsar.common.migration;
\ No newline at end of file
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/MetadataMigrationImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/MetadataMigrationImpl.java
new file mode 100644
index 00000000000..8fa110e77a8
--- /dev/null
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/MetadataMigrationImpl.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.admin.internal;
+
+import java.util.concurrent.CompletableFuture;
+import javax.ws.rs.client.WebTarget;
+import org.apache.pulsar.client.admin.MetadataMigration;
+import org.apache.pulsar.client.api.Authentication;
+import org.apache.pulsar.common.migration.MigrationState;
+
+public class MetadataMigrationImpl extends BaseResource implements MetadataMigration {
+ private final WebTarget adminPath;
+
+ public MetadataMigrationImpl(WebTarget web, Authentication auth, long requestTimeoutMs) {
+ super(auth, requestTimeoutMs);
+ adminPath = web.path("/admin/v2/metadata/migration");
+ }
+
+ @Override
+ public CompletableFuture<Void> start(String targetUrl) {
+
+ return asyncPostRequest(adminPath.path("start")
+ .queryParam("target", targetUrl), null).thenApply(x -> null);
+ }
+
+ @Override
+ public CompletableFuture<MigrationState> status() {
+ return asyncGetRequest(adminPath.path("status"), new FutureCallback<>(){});
+ }
+}
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java
index acb53fede2e..730ba05ddfa 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java
@@ -35,6 +35,7 @@ import org.apache.pulsar.client.admin.Brokers;
import org.apache.pulsar.client.admin.Clusters;
import org.apache.pulsar.client.admin.Functions;
import org.apache.pulsar.client.admin.Lookup;
+import org.apache.pulsar.client.admin.MetadataMigration;
import org.apache.pulsar.client.admin.Namespaces;
import org.apache.pulsar.client.admin.NonPersistentTopics;
import org.apache.pulsar.client.admin.Packages;
@@ -105,6 +106,7 @@ public class PulsarAdminImpl implements PulsarAdmin {
private final Schemas schemas;
private final Packages packages;
private final Transactions transactions;
+ private final MetadataMigration metadataMigration;
protected final WebTarget root;
protected final Authentication auth;
@Getter
@@ -189,6 +191,7 @@ public class PulsarAdminImpl implements PulsarAdmin {
this.bookies = new BookiesImpl(root, auth, requestTimeoutMs);
this.packages = new PackagesImpl(root, auth, asyncHttpConnector, requestTimeoutMs);
this.transactions = new TransactionsImpl(root, auth, requestTimeoutMs);
+ this.metadataMigration = new MetadataMigrationImpl(root, auth, requestTimeoutMs);
if (originalCtxLoader != null) {
Thread.currentThread().setContextClassLoader(originalCtxLoader);
@@ -434,6 +437,11 @@ public class PulsarAdminImpl implements PulsarAdmin {
return transactions;
}
+ @Override
+ public MetadataMigration metadataMigration() {
+ return metadataMigration;
+ }
+
/**
* Close the Pulsar admin client to release all the resources.
*/
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdMetadataMigration.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdMetadataMigration.java
new file mode 100644
index 00000000000..005a2b73e49
--- /dev/null
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdMetadataMigration.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.admin.cli;
+
+import java.util.function.Supplier;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+/**
+ * CLI commands for metadata store migration operations.
+ */
+@Command(description = "Operations for metadata store migration")
+public class CmdMetadataMigration extends CmdBase {
+
+ public CmdMetadataMigration(Supplier<PulsarAdmin> admin) {
+ super("metadata-migration", admin);
+ addCommand("start", new CmdMetadataMigration.Start());
+ addCommand("status", new CmdMetadataMigration.Status());
+ }
+
+ @Command(description = "Start metadata store migration to target")
+ private class Start extends CliCommand {
+ @Option(names = {"--target"}, description = "Target metadata store URL (e.g., oxia://host:port/namespace)",
+ required = true)
+ private String targetUrl;
+
+ @Override
+ void run() throws Exception {
+ print("Starting metadata store migration");
+ print("Target: " + targetUrl);
+ print("");
+
+ // Extract store type from URL
+ getAdmin().metadataMigration().start(targetUrl).get();
+
+ print("");
+ print("✓ Migration started");
+ print("");
+ print("Monitor progress: pulsar-admin metadata-migration status");
+ }
+ }
+
+ @Command(description = "Check migration status")
+ private class Status extends CliCommand {
+ @Override
+ void run() throws Exception {
+ print(getAdmin().metadataMigration().status().get());
+ }
+ }
+}
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java
index cd79098f0c3..29530adf74a 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java
@@ -304,6 +304,8 @@ public class PulsarAdminTool implements CommandHook {
commandMap.put("packages", CmdPackages.class);
commandMap.put("transactions", CmdTransactions.class);
+ commandMap.put("migration", CmdMetadataMigration.class);
+
setupCommands(properties);
}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/extended/MetadataStoreExtended.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/extended/MetadataStoreExtended.java
index 182c14ef601..2bbd627c2fc 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/extended/MetadataStoreExtended.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/extended/MetadataStoreExtended.java
@@ -95,4 +95,12 @@ public interface MetadataStoreExtended extends MetadataStore {
default CompletableFuture<Void> handleMetadataEvent(MetadataEvent event) {
return CompletableFuture.completedFuture(null);
}
+
+ /**
+ * Force invalidation of cached entries for the specified paths.
+ *
+ * @param paths
+ */
+ default void invalidateCaches(String...paths) {
+ }
}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerIdGenerator.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerIdGenerator.java
index acac35057e5..86f9c0e235a 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerIdGenerator.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerIdGenerator.java
@@ -31,6 +31,7 @@ import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.MetadataStoreException;
import org.apache.pulsar.metadata.api.extended.CreateOption;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.apache.pulsar.metadata.impl.ZKMetadataStore;
@Slf4j
@@ -291,12 +292,14 @@ public class PulsarLedgerIdGenerator implements LedgerIdGenerator {
//If the config rootPath when use zk metadata store, it will append rootPath as the prefix of the path.
//So when we get the path from the stat, we should truncate the rootPath.
private String handleTheDeletePath(String path) {
- if (store instanceof ZKMetadataStore) {
- String rootPath = ((ZKMetadataStore) store).getRootPath();
- if (rootPath == null) {
- return path;
+ if (store instanceof DualMetadataStore dms) {
+ if (dms.getSourceStore() instanceof ZKMetadataStore zkStore) {
+ String rootPath = zkStore.getRootPath();
+ if (rootPath == null) {
+ return path;
+ }
+ return path.replaceFirst(rootPath, "");
}
- return path.replaceFirst(rootPath, "");
}
return path;
}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java
index 2673328b811..bfd9171a391 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java
@@ -66,6 +66,7 @@ import org.apache.pulsar.metadata.api.Notification;
import org.apache.pulsar.metadata.api.NotificationType;
import org.apache.pulsar.metadata.api.extended.CreateOption;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.apache.pulsar.metadata.impl.ZKMetadataStore;
import org.apache.zookeeper.KeeperException;
@@ -177,7 +178,13 @@ public class PulsarLedgerUnderreplicationManager implements LedgerUnderreplicati
if (!store.exists(layoutPath).join()) {
LedgerRereplicationLayoutFormat.Builder builder = LedgerRereplicationLayoutFormat.newBuilder();
builder.setType(LAYOUT).setVersion(LAYOUT_VERSION);
- store.put(layoutPath, builder.build().toString().getBytes(UTF_8), Optional.of(-1L)).join();
+ try {
+ store.put(layoutPath, builder.build().toString().getBytes(UTF_8), Optional.of(-1L)).get();
+ } catch (ExecutionException | InterruptedException e) {
+ if (!(e.getCause() instanceof MetadataStoreException.BadVersionException)) {
+ throw new RuntimeException(e);
+ }
+ }
} else {
byte[] layoutData = store.get(layoutPath).join().get().getValue();
@@ -424,7 +431,8 @@ public class PulsarLedgerUnderreplicationManager implements LedgerUnderreplicati
if (l != null) {
store.delete(getUrLedgerPath(ledgerId), Optional.of(l.getLedgerNodeVersion()))
.get(BLOCKING_CALL_TIMEOUT, MILLISECONDS);
- if (store instanceof ZKMetadataStore) {
+ if (store instanceof ZKMetadataStore
+ || store instanceof DualMetadataStore) {
try {
// clean up the hierarchy
String[] parts = getUrLedgerPath(ledgerId).split("/");
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java
index 89dbf2be990..1e838a658a1 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java
@@ -51,13 +51,13 @@ import org.apache.pulsar.metadata.api.CacheGetResult;
import org.apache.pulsar.metadata.api.MetadataCache;
import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.Notification;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
import org.apache.pulsar.metadata.api.extended.SessionEvent;
-import org.apache.pulsar.metadata.impl.AbstractMetadataStore;
@Slf4j
public class PulsarRegistrationClient implements RegistrationClient {
- private final AbstractMetadataStore store;
+ private final MetadataStoreExtended store;
private final String ledgersRootPath;
// registration paths
private final String bookieRegistrationPath;
@@ -74,7 +74,7 @@ public class PulsarRegistrationClient implements RegistrationClient {
public PulsarRegistrationClient(MetadataStore store,
String ledgersRootPath) {
- this.store = (AbstractMetadataStore) store;
+ this.store = (MetadataStoreExtended) store;
this.ledgersRootPath = ledgersRootPath;
this.bookieServiceInfoMetadataCache = store.getMetadataCache(BookieServiceInfoSerde.INSTANCE);
this.sequencer = Sequencer.create();
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
index b1f6f0a4f50..25890852775 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java
@@ -116,7 +116,8 @@ public class MetadataCacheImpl<T> implements MetadataCache<T>, Consumer<Notifica
String key,
Optional<CacheGetResult<T>> oldValue,
Executor executor) {
- if (store instanceof AbstractMetadataStore && ((AbstractMetadataStore) store).isConnected()) {
+ if (!(store instanceof AbstractMetadataStore)
+ || ((AbstractMetadataStore) store).isConnected()) {
if (log.isDebugEnabled()) {
log.debug("Reloading key {} into metadata cache {}", key, cacheName);
}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/MigrationCoordinator.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/MigrationCoordinator.java
new file mode 100644
index 00000000000..28f6184c397
--- /dev/null
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/MigrationCoordinator.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata.coordination.impl;
+
+import io.oxia.client.api.AsyncOxiaClient;
+import io.oxia.client.api.options.defs.OptionOverrideModificationsCount;
+import io.oxia.client.api.options.defs.OptionOverrideVersionId;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.common.migration.MigrationPhase;
+import org.apache.pulsar.common.migration.MigrationState;
+import org.apache.pulsar.common.util.Backoff;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.metadata.api.MetadataCache;
+import org.apache.pulsar.metadata.api.MetadataStore;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.extended.CreateOption;
+import org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStoreProvider;
+
+/**
+ * Coordinates metadata store migration process.
+ */
+@Slf4j
+public class MigrationCoordinator {
+ private final MetadataStore sourceStore;
+ private final String targetUrl;
+ private final AsyncOxiaClient oxiaClient;
+ private final MetadataCache<MigrationState> migrationStateCache;
+
+ private static final int MAX_PENDING_OPS = 1000;
+
+ public MigrationCoordinator(MetadataStore sourceStore, String targetUrl) throws MetadataStoreException {
+ this.sourceStore = sourceStore;
+ this.targetUrl = targetUrl;
+ this.migrationStateCache = sourceStore.getMetadataCache(MigrationState.class);
+
+ if (!targetUrl.startsWith("oxia://")) {
+ throw new MetadataStoreException("Expected target metadata store to be Oxia");
+ }
+
+ this.oxiaClient = new OxiaMetadataStoreProvider().getOxiaClient(targetUrl);
+ }
+
+ /**
+ * Start the migration process.
+ *
+ * @throws Exception if migration fails
+ */
+ public void startMigration() throws Exception {
+ log.info("=== Starting Migration ===");
+ log.info("Source: {} (current)", sourceStore.getClass().getSimpleName());
+ log.info("Target: {}", targetUrl);
+
+ try {
+ // 1. Create migration flag
+ setInitialMigrationPhase();
+
+ // 2. Wait for participants to prepare
+ waitForPreparation();
+
+ // 3. Copy persistent data
+ updatePhase(MigrationPhase.COPYING);
+ copyPersistentData();
+
+ // 4. Set state to completed
+ updatePhase(MigrationPhase.COMPLETED);
+
+ log.info("=== Migration Complete ===");
+ } catch (Exception e) {
+ log.error("Migration failed", e);
+ updatePhase(MigrationPhase.FAILED);
+ throw e;
+ }
+ }
+
+ private void setInitialMigrationPhase() throws MetadataStoreException {
+ try {
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer()
+ .writeValueAsBytes(new MigrationState(MigrationPhase.PREPARATION, targetUrl)),
+ Optional.of(-1L)).get();
+ } catch (Exception e) {
+ throw new MetadataStoreException(e);
+ }
+ }
+
+ private void updatePhase(MigrationPhase phase) throws MetadataStoreException {
+ try {
+ migrationStateCache.put(MigrationState.MIGRATION_FLAG_PATH,
+ new MigrationState(phase, targetUrl), EnumSet.noneOf(CreateOption.class)).get();
+ } catch (Exception e) {
+ throw new MetadataStoreException(e);
+ }
+ }
+
+ private void waitForPreparation() throws Exception {
+ log.info("Waiting for all participants to prepare...");
+
+ long startTime = System.currentTimeMillis();
+ Backoff backoff = new Backoff(100, TimeUnit.MILLISECONDS, 60, TimeUnit.SECONDS, 60,
+ TimeUnit.SECONDS);
+ while (System.currentTimeMillis() - startTime < 60_000) {
+ List<String> pending = sourceStore.getChildren(MigrationState.PARTICIPANTS_PATH).get();
+ if (pending.isEmpty()) {
+ log.info("All migration participants ready");
+ return;
+ }
+
+ log.info("Waiting for participants to prepare. pending: {}", pending);
+ Thread.sleep(backoff.next());
+ }
+
+ log.error("Failed to wait for all participants to prepare. pending participants: {}",
+ sourceStore.getChildren(MigrationState.PARTICIPANTS_PATH).get());
+ }
+
+ private void copyPersistentData() throws Exception {
+ log.info("Starting persistent data copy...");
+
+ AtomicLong copiedCount = new AtomicLong(0);
+ Semaphore semaphore = new Semaphore(MAX_PENDING_OPS);
+ AtomicReference<Throwable> exception = new AtomicReference<>();
+
+ // Bootstrap first level
+ BlockingQueue<String> workQueue = new LinkedBlockingQueue<>(getChildren("/").get());
+
+ while (true) {
+ String path = workQueue.poll(1, TimeUnit.SECONDS);
+ if (path == null) {
+ // Wait until all pending ops are done
+ if (semaphore.availablePermits() != MAX_PENDING_OPS) {
+ continue;
+ } else {
+ break;
+ }
+ }
+
+ semaphore.acquire();
+ copy(path).whenComplete((res, e) -> {
+ semaphore.release();
+ if (e != null) {
+ exception.compareAndSet(null, e);
+ }
+
+ copiedCount.incrementAndGet();
+ });
+
+ semaphore.acquire();
+ getChildren(path).whenComplete((res, e) -> {
+ if (e != null) {
+ exception.compareAndSet(null, e);
+ } else {
+ workQueue.addAll(res);
+ }
+
+ semaphore.release();
+ });
+
+ if (exception.get() != null) {
+ break;
+ }
+ }
+
+ if (exception.get() != null) {
+ throw new Exception(exception.get());
+ }
+
+ log.info("All data copied. total records={}", copiedCount.get());
+ }
+
+ private CompletableFuture<List<String>> getChildren(String parent) {
+ return sourceStore.getChildren(parent)
+ .thenApply(list -> list.stream().map(x -> {
+ if ("/".equals(parent)) {
+ return "/" + x;
+ } else {
+ return parent + "/" + x;
+ }
+ }).toList());
+ }
+
+ private CompletableFuture<Void> copy(String path) {
+ return sourceStore.get(path)
+ .thenCompose(ogr -> {
+ if (ogr.isPresent()) {
+ var gr = ogr.get();
+ if (gr.getStat().isEphemeral()) {
+ // Ignore ephemeral at this point
+ return CompletableFuture.completedFuture(null);
+ } else {
+ return oxiaClient.put(path, gr.getValue(),
+ Set.of(new OptionOverrideVersionId(gr.getStat().getVersion()),
+ new OptionOverrideModificationsCount(gr.getStat().getVersion())
+ )
+ ).thenRun(() -> log.debug("--- Copied {}", path));
+ }
+ } else {
+ return CompletableFuture.completedFuture(null);
+ }
+ }).thenApply(x -> null);
+ }
+}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/DualMetadataCache.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/DualMetadataCache.java
new file mode 100644
index 00000000000..2dc8e3ee863
--- /dev/null
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/DualMetadataCache.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata.impl;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import lombok.Getter;
+import org.apache.pulsar.metadata.api.CacheGetResult;
+import org.apache.pulsar.metadata.api.MetadataCache;
+import org.apache.pulsar.metadata.api.MetadataCacheConfig;
+import org.apache.pulsar.metadata.api.MetadataSerde;
+import org.apache.pulsar.metadata.api.extended.CreateOption;
+
+public class DualMetadataCache<T> implements MetadataCache<T> {
+ private final DualMetadataStore dualMetadataStore;
+ private final Class<T> clazz;
+ private final TypeReference<T> typeRef;
+ private final String cacheName;
+ private final MetadataSerde<T> serde;
+ private final MetadataCacheConfig cacheConfig;
+
+ @Getter
+ private final AtomicReference<MetadataCache<T>> metadataCache = new AtomicReference<>();
+
+ public DualMetadataCache(DualMetadataStore dualMetadataStore, Class<T> clazz, TypeReference<T> typeRef,
+ String cacheName, MetadataSerde<T> serde,
+ MetadataCacheConfig cacheConfig) {
+ this.dualMetadataStore = dualMetadataStore;
+ this.clazz = clazz;
+ this.typeRef = typeRef;
+ this.cacheName = cacheName;
+ this.serde = serde;
+ this.cacheConfig = cacheConfig;
+
+ var store = dualMetadataStore.targetStore != null
+ ? dualMetadataStore.targetStore : dualMetadataStore.sourceStore;
+
+ if (clazz != null) {
+ this.metadataCache.set(store.getMetadataCache(clazz, cacheConfig));
+ } else if (typeRef != null) {
+ this.metadataCache.set(store.getMetadataCache(typeRef, cacheConfig));
+ } else {
+ this.metadataCache.set(store.getMetadataCache(cacheName, serde, cacheConfig));
+ }
+ }
+
+ @Override
+ public CompletableFuture<Optional<T>> get(String path) {
+ return metadataCache.get().get(path);
+ }
+
+ @Override
+ public CompletableFuture<Optional<CacheGetResult<T>>> getWithStats(String path) {
+ return metadataCache.get().getWithStats(path);
+ }
+
+ @Override
+ public Optional<T> getIfCached(String path) {
+ return metadataCache.get().getIfCached(path);
+ }
+
+ @Override
+ public CompletableFuture<List<String>> getChildren(String path) {
+ return metadataCache.get().getChildren(path);
+ }
+
+ @Override
+ public CompletableFuture<Boolean> exists(String path) {
+ return metadataCache.get().exists(path);
+ }
+
+ @Override
+ public CompletableFuture<T> readModifyUpdateOrCreate(String path, Function<Optional<T>, T> modifyFunction) {
+ return metadataCache.get().readModifyUpdateOrCreate(path, modifyFunction);
+ }
+
+ @Override
+ public CompletableFuture<T> readModifyUpdate(String path, Function<T, T> modifyFunction) {
+ return metadataCache.get().readModifyUpdate(path, modifyFunction);
+ }
+
+ @Override
+ public CompletableFuture<Void> create(String path, T value) {
+ return metadataCache.get().create(path, value);
+ }
+
+ @Override
+ public CompletableFuture<Void> put(String path, T value, EnumSet<CreateOption> options) {
+ return metadataCache.get().put(path, value, options);
+ }
+
+ @Override
+ public CompletableFuture<Void> delete(String path) {
+ return metadataCache.get().delete(path);
+ }
+
+ @Override
+ public void invalidate(String path) {
+ metadataCache.get().invalidate(path);
+ }
+
+ @Override
+ public void invalidateAll() {
+ metadataCache.get().invalidateAll();
+ }
+
+ @Override
+ public void refresh(String path) {
+ metadataCache.get().refresh(path);
+ }
+
+ void handleSwitchToTargetStore() {
+ if (clazz != null) {
+ metadataCache.set(dualMetadataStore.targetStore.getMetadataCache(clazz, cacheConfig));
+ } else if (typeRef != null) {
+ metadataCache.set(dualMetadataStore.targetStore.getMetadataCache(typeRef, cacheConfig));
+ } else {
+ metadataCache.set(dualMetadataStore.targetStore.getMetadataCache(cacheName, serde, cacheConfig));
+ }
+ }
+}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/DualMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/DualMetadataStore.java
new file mode 100644
index 00000000000..972864a0321
--- /dev/null
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/DualMetadataStore.java
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata.impl;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.netty.util.concurrent.DefaultThreadFactory;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.common.migration.MigrationPhase;
+import org.apache.pulsar.common.migration.MigrationState;
+import org.apache.pulsar.common.util.Backoff;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.metadata.api.GetResult;
+import org.apache.pulsar.metadata.api.MetadataCache;
+import org.apache.pulsar.metadata.api.MetadataCacheConfig;
+import org.apache.pulsar.metadata.api.MetadataEvent;
+import org.apache.pulsar.metadata.api.MetadataEventSynchronizer;
+import org.apache.pulsar.metadata.api.MetadataSerde;
+import org.apache.pulsar.metadata.api.MetadataStore;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.MetadataStoreLifecycle;
+import org.apache.pulsar.metadata.api.Notification;
+import org.apache.pulsar.metadata.api.Stat;
+import org.apache.pulsar.metadata.api.extended.CreateOption;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.api.extended.SessionEvent;
+
+/**
+ * Wrapper around a metadata store that provides transparent migration capability.
+ *
+ * <p>When migration is not active, all operations are forwarded to the source store.
+ * When migration starts (detected via flag in source store), this wrapper:
+ * <ul>
+ * <li>Initializes connection to target store</li>
+ * <li>Recreates ephemeral nodes in target store</li>
+ * <li>Routes reads/writes based on migration phase</li>
+ * </ul>
+ */
+@Slf4j
+public class DualMetadataStore implements MetadataStoreExtended {
+
+ @Getter
+ final MetadataStoreExtended sourceStore;
+ volatile MetadataStoreExtended targetStore = null;
+
+ private volatile MigrationState migrationState = MigrationState.NOT_STARTED;
+
+ private final MetadataStoreConfig config;
+ private String participantId;
+ private final Set<String> localEphemeralPaths = ConcurrentHashMap.newKeySet();
+
+ private final ScheduledExecutorService executor;
+
+ private final MetadataCache<MigrationState> migrationStateCache;
+
+ private final Set<Consumer<Notification>> listeners = ConcurrentHashMap.newKeySet();
+ private final Set<Consumer<SessionEvent>> sessionListeners = ConcurrentHashMap.newKeySet();
+
+ private final AtomicInteger pendingSourceWrites = new AtomicInteger();
+
+ private final Set<DualMetadataCache<?>> caches = ConcurrentHashMap.newKeySet();
+
+ private static final IllegalStateException READ_ONLY_STATE_EXCEPTION =
+ new IllegalStateException("Write operations not allowed during migrations");
+
+ public DualMetadataStore(MetadataStore sourceStore, MetadataStoreConfig config) throws MetadataStoreException {
+ this.sourceStore = (MetadataStoreExtended) sourceStore;
+ this.config = config;
+ this.executor = new ScheduledThreadPoolExecutor(1,
+ new DefaultThreadFactory("pulsar-dual-metadata-store", true));
+ this.migrationStateCache = sourceStore.getMetadataCache(MigrationState.class);
+
+ if (sourceStore instanceof MetadataStoreLifecycle msl) {
+ msl.initializeCluster();
+ }
+
+ readCurrentState();
+ registerAsParticipant();
+
+ // Watch for migration events
+ watchForMigrationEvents();
+ }
+
+ private void readCurrentState() throws MetadataStoreException {
+ try {
+ // Read the current state, force ZK to sync to read the latest value
+ sourceStore.sync(MigrationState.MIGRATION_FLAG_PATH).get();
+ var initialState = migrationStateCache.get(MigrationState.MIGRATION_FLAG_PATH).get();
+ initialState.ifPresent(state -> this.migrationState = state);
+
+ if (migrationState.getPhase() == MigrationPhase.COMPLETED) {
+ initializeTargetStore(migrationState.getTargetUrl());
+ }
+
+ } catch (Exception e) {
+ throw new MetadataStoreException(e);
+ }
+ }
+
+ private void registerAsParticipant() throws MetadataStoreException {
+ try {
+ // Register ourselves as participant in an eventual migration
+ Stat stat = this.sourceStore.put(MigrationState.PARTICIPANTS_PATH + "/id-", new byte[0],
+ Optional.empty(), EnumSet.of(CreateOption.Sequential, CreateOption.Ephemeral)).get();
+ participantId = stat.getPath();
+ log.info("Participant metadata store created: {}", participantId);
+ } catch (Throwable e) {
+ throw new MetadataStoreException(e);
+ }
+ }
+
+ private void watchForMigrationEvents() {
+ // Register listener for migration-related paths
+ sourceStore.registerListener(notification -> {
+ if (!MigrationState.MIGRATION_FLAG_PATH.equals(notification.getPath())) {
+ return;
+ }
+
+ migrationStateCache.get(MigrationState.MIGRATION_FLAG_PATH)
+ .thenAccept(migrationState -> {
+ this.migrationState = migrationState.orElse(MigrationState.NOT_STARTED);
+
+ switch (this.migrationState.getPhase()) {
+ case PREPARATION -> executor.execute(this::handleMigrationStart);
+ case COMPLETED -> executor.execute(this::handleMigrationComplete);
+ case FAILED -> executor.execute(this::handleMigrationFailed);
+ default -> {
+ // no-op
+ }
+ }
+ });
+ });
+ }
+
+ private void handleMigrationStart() {
+ try {
+ log.info("=== Starting Metadata Migration Preparation ===");
+ log.info("Target metadata store URL: {}", migrationState.getTargetUrl());
+
+ long startTime = System.currentTimeMillis();
+ Backoff backoff = new Backoff(100, TimeUnit.MILLISECONDS, 60, TimeUnit.SECONDS, 60,
+ TimeUnit.SECONDS);
+ while (System.currentTimeMillis() - startTime < 60_000) {
+ int pending = pendingSourceWrites.get();
+ if (pending == 0) {
+ break;
+ } else {
+ log.info("Waiting for {} pending source writes to complete", pendingSourceWrites.get());
+ Thread.sleep(backoff.next());
+ }
+ }
+
+ if (pendingSourceWrites.get() != 0) {
+ log.error("Failed to wait for pending writes to complete");
+ return;
+ }
+
+ // Mark the session as lost so that all the component will avoid trying to make metadata writes
+ // for anything that can be deferred (eg: ledgers rollovers)
+ sessionListeners.forEach(listener -> listener.accept(SessionEvent.SessionLost));
+
+ // Initialize target store
+ initializeTargetStore(migrationState.getTargetUrl());
+
+ this.recreateEphemeralNodesInTarget();
+
+ // Acknowledge preparation by deleting the participant id
+ sourceStore.delete(participantId, Optional.empty()).get();
+
+ log.info("=== Migration Preparation Complete ===");
+
+ } catch (Exception e) {
+ log.error("Failed during migration preparation", e);
+ }
+ }
+
+ private void handleMigrationComplete() {
+ log.info("=== Metadata Migration Complete ===");
+
+ caches.forEach(DualMetadataCache::handleSwitchToTargetStore);
+ listeners.forEach(targetStore::registerListener);
+ sessionListeners.forEach(targetStore::registerSessionListener);
+
+ sessionListeners.forEach(listener -> listener.accept(SessionEvent.SessionReestablished));
+ }
+
+ private void handleMigrationFailed() {
+ log.info("=== Metadata Migration Failed ===");
+ sessionListeners.forEach(listener -> listener.accept(SessionEvent.SessionReestablished));
+ }
+
+
+ private synchronized void initializeTargetStore(String targetUrl) throws MetadataStoreException {
+ if (this.targetStore != null) {
+ return;
+ }
+
+ log.info("Initializing target metadata store: {}", targetUrl);
+ this.targetStore = (MetadataStoreExtended) MetadataStoreFactoryImpl.create(
+ targetUrl,
+ MetadataStoreConfig.builder()
+ .sessionTimeoutMillis(config.getSessionTimeoutMillis())
+ .batchingEnabled(config.isBatchingEnabled())
+ .batchingMaxDelayMillis(config.getBatchingMaxDelayMillis())
+ .batchingMaxOperations(config.getBatchingMaxOperations())
+ .batchingMaxSizeKb(config.getBatchingMaxSizeKb())
+ .build()
+ );
+
+ log.info("Target store initialized successfully");
+ }
+
+ private void recreateEphemeralNodesInTarget() throws Exception {
+ log.info("Found {} local ephemeral nodes to recreate", localEphemeralPaths.size());
+ var futures = localEphemeralPaths.stream()
+ .map(path ->
+ sourceStore.get(path)
+ .thenCompose(ogr ->
+ ogr.map(gr -> targetStore.put(path, gr.getValue(), Optional.empty(),
+ EnumSet.of(CreateOption.Ephemeral)))
+ .orElse(
+ CompletableFuture.completedFuture(null))
+ )
+ ).toList();
+
+ FutureUtil.waitForAll(futures).get();
+ }
+
+ @Override
+ public CompletableFuture<Optional<GetResult>> get(String path) {
+ return switch (migrationState.getPhase()) {
+ case NOT_STARTED, PREPARATION, COPYING, FAILED -> sourceStore.get(path);
+ case COMPLETED -> targetStore.get(path);
+ };
+ }
+
+ @Override
+ public CompletableFuture<List<String>> getChildren(String path) {
+ return switch (migrationState.getPhase()) {
+ case NOT_STARTED, PREPARATION, COPYING, FAILED -> sourceStore.getChildren(path);
+ case COMPLETED -> targetStore.getChildren(path);
+ };
+ }
+
+ @Override
+ public CompletableFuture<List<String>> getChildrenFromStore(String path) {
+ return switch (migrationState.getPhase()) {
+ case NOT_STARTED, PREPARATION, COPYING, FAILED -> sourceStore.getChildrenFromStore(path);
+ case COMPLETED -> targetStore.getChildrenFromStore(path);
+ };
+ }
+
+ @Override
+ public CompletableFuture<Boolean> exists(String path) {
+ return switch (migrationState.getPhase()) {
+ case NOT_STARTED, PREPARATION, COPYING, FAILED -> sourceStore.exists(path);
+ case COMPLETED -> targetStore.exists(path);
+ };
+ }
+
+ @Override
+ public CompletableFuture<Stat> put(String path, byte[] value, Optional<Long> expectedVersion) {
+ return put(path, value, expectedVersion, EnumSet.noneOf(CreateOption.class));
+ }
+
+ @Override
+ public CompletableFuture<Stat> put(String path, byte[] value, Optional<Long> expectedVersion,
+ EnumSet<CreateOption> options) {
+ switch (migrationState.getPhase()) {
+ case NOT_STARTED, FAILED -> {
+ // Track ephemeral nodes
+ if (options.contains(CreateOption.Ephemeral)) {
+ localEphemeralPaths.add(path);
+ }
+
+ // Track pending writes
+ pendingSourceWrites.incrementAndGet();
+ var future = sourceStore.put(path, value, expectedVersion, options);
+ future.whenComplete((result, e) -> pendingSourceWrites.decrementAndGet());
+ return future;
+ }
+
+ case PREPARATION, COPYING -> {
+ return CompletableFuture.failedFuture(READ_ONLY_STATE_EXCEPTION);
+ }
+
+ case COMPLETED -> {
+ return targetStore.put(path, value, expectedVersion, options);
+ }
+
+ default -> throw new IllegalStateException("Invalid phase " + migrationState.getPhase());
+ }
+ }
+
+ @Override
+ public CompletableFuture<Void> delete(String path, Optional<Long> expectedVersion) {
+ switch (migrationState.getPhase()) {
+ case NOT_STARTED, FAILED -> {
+ localEphemeralPaths.remove(path);
+
+ pendingSourceWrites.incrementAndGet();
+ var future = sourceStore.delete(path, expectedVersion);
+ future.whenComplete((result, e) -> pendingSourceWrites.decrementAndGet());
+ return future;
+ }
+
+ case PREPARATION, COPYING -> {
+ return CompletableFuture.failedFuture(READ_ONLY_STATE_EXCEPTION);
+ }
+
+ case COMPLETED -> {
+ return targetStore.delete(path, expectedVersion);
+ }
+
+ default -> throw new IllegalStateException("Invalid phase " + migrationState.getPhase());
+ }
+ }
+
+ @Override
+ public CompletableFuture<Void> deleteRecursive(String path) {
+ switch (migrationState.getPhase()) {
+ case NOT_STARTED, FAILED -> {
+ pendingSourceWrites.incrementAndGet();
+ var future = sourceStore.deleteRecursive(path);
+ future.whenComplete((result, e) -> pendingSourceWrites.decrementAndGet());
+ return future;
+ }
+ case PREPARATION, COPYING -> {
+ return CompletableFuture.failedFuture(READ_ONLY_STATE_EXCEPTION);
+ }
+ case COMPLETED -> {
+ return targetStore.deleteRecursive(path);
+ }
+
+ default -> throw new IllegalStateException("Invalid phase " + migrationState.getPhase());
+ }
+ }
+
+ @Override
+ public void registerListener(Consumer<Notification> listener) {
+ switch (migrationState.getPhase()) {
+ case NOT_STARTED, PREPARATION, COPYING, FAILED -> {
+ listeners.add(listener);
+ sourceStore.registerListener(listener);
+ }
+
+ case COMPLETED -> targetStore.registerListener(listener);
+ }
+ }
+
+ @Override
+ public void registerSessionListener(Consumer<SessionEvent> listener) {
+ switch (migrationState.getPhase()) {
+ case NOT_STARTED, PREPARATION, COPYING, FAILED -> {
+ sessionListeners.add(listener);
+ sourceStore.registerSessionListener(listener);
+ }
+
+ case COMPLETED -> targetStore.registerSessionListener(listener);
+ }
+ }
+
+ @Override
+ public <T> MetadataCache<T> getMetadataCache(Class<T> clazz, MetadataCacheConfig cacheConfig) {
+ var cache = new DualMetadataCache<>(this, clazz, null, null, null, cacheConfig);
+ caches.add(cache);
+ return cache;
+ }
+
+ @Override
+ public <T> MetadataCache<T> getMetadataCache(TypeReference<T> typeRef, MetadataCacheConfig cacheConfig) {
+ var cache = new DualMetadataCache<>(this, null, typeRef, null, null, cacheConfig);
+ caches.add(cache);
+ return cache;
+ }
+
+ @Override
+ public <T> MetadataCache<T> getMetadataCache(String cacheName, MetadataSerde<T> serde,
+ MetadataCacheConfig cacheConfig) {
+ var cache = new DualMetadataCache<>(this, null, null, cacheName, serde, cacheConfig);
+ caches.add(cache);
+ return cache;
+ }
+
+ @Override
+ public Optional<MetadataEventSynchronizer> getMetadataEventSynchronizer() {
+ return sourceStore.getMetadataEventSynchronizer();
+ }
+
+ @Override
+ public void updateMetadataEventSynchronizer(MetadataEventSynchronizer synchronizer) {
+ sourceStore.updateMetadataEventSynchronizer(synchronizer);
+ }
+
+ @Override
+ public CompletableFuture<Void> handleMetadataEvent(MetadataEvent event) {
+ return sourceStore.handleMetadataEvent(event);
+ }
+
+ @Override
+ public void close() throws Exception {
+ log.info("Closing DualMetadataStore");
+
+ // Close target store first (if exists)
+ if (targetStore != null) {
+ try {
+ targetStore.close();
+ log.info("Target store closed");
+ } catch (Exception e) {
+ log.error("Error closing target store", e);
+ }
+ }
+
+ // Close source store
+ try {
+ sourceStore.close();
+ log.info("Source store closed");
+ } catch (Exception e) {
+ log.error("Error closing source store", e);
+ }
+
+ executor.shutdownNow();
+ executor.awaitTermination(5, TimeUnit.SECONDS);
+ }
+}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
index f56d6c6941f..011508567e5 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
@@ -685,6 +685,11 @@ class ZkMetadataStoreProvider implements MetadataStoreProvider {
if (metadataURL.startsWith(ZKMetadataStore.ZK_SCHEME_IDENTIFIER)) {
metadataURL = metadataURL.substring(ZKMetadataStore.ZK_SCHEME_IDENTIFIER.length());
}
- return new ZKMetadataStore(metadataURL, metadataStoreConfig, enableSessionWatcher);
+
+ // Create the ZK metadata store
+ ZKMetadataStore zkStore = new ZKMetadataStore(metadataURL, metadataStoreConfig, enableSessionWatcher);
+
+ // Wrap with DualMetadataStore to enable migration capability
+ return new DualMetadataStore(zkStore, metadataStoreConfig);
}
}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStoreProvider.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStoreProvider.java
index a4c52134a8a..2b7dbfce728 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStoreProvider.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStoreProvider.java
@@ -18,6 +18,10 @@
*/
package org.apache.pulsar.metadata.impl.oxia;
+import io.oxia.client.api.AsyncOxiaClient;
+import io.oxia.client.api.OxiaClientBuilder;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
import lombok.NonNull;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.metadata.api.MetadataStore;
@@ -29,7 +33,7 @@ public class OxiaMetadataStoreProvider implements MetadataStoreProvider {
// declare the specific namespace to avoid any changes in the future.
public static final String DefaultNamespace = "default";
- public static final String OXIA_SCHEME = "oxia";
+ public static final String OXIA_SCHEME = "oxia";
public static final String OXIA_SCHEME_IDENTIFIER = OXIA_SCHEME + ":";
@Override
@@ -72,4 +76,16 @@ public class OxiaMetadataStoreProvider implements MetadataStoreProvider {
}
return Pair.of(split[0], split[1]);
}
-}
+
+ public AsyncOxiaClient getOxiaClient(String metadataURL) throws MetadataStoreException {
+ var pair = getServiceAddressAndNamespace(metadataURL);
+ try {
+ return OxiaClientBuilder.create(pair.getLeft())
+ .namespace(pair.getRight())
+ .batchLinger(Duration.of(100, ChronoUnit.MILLIS))
+ .asyncClient().get();
+ } catch (Exception e) {
+ throw new MetadataStoreException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/MetadataStoreTableViewImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/MetadataStoreTableViewImpl.java
index c06fbe3cc07..37a2e77084f 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/MetadataStoreTableViewImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/MetadataStoreTableViewImpl.java
@@ -52,8 +52,8 @@ import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.MetadataStoreException;
import org.apache.pulsar.metadata.api.MetadataStoreTableView;
import org.apache.pulsar.metadata.api.NotificationType;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
import org.apache.pulsar.metadata.api.extended.SessionEvent;
-import org.apache.pulsar.metadata.impl.AbstractMetadataStore;
import org.jspecify.annotations.Nullable;
@Slf4j
@@ -161,8 +161,8 @@ public class MetadataStoreTableViewImpl<T> implements MetadataStoreTableView<T>
.asyncReloadConsumer(this::consumeAsyncReload)
.build());
store.registerListener(this::handleNotification);
- if (store instanceof AbstractMetadataStore abstractMetadataStore) {
- abstractMetadataStore.registerSessionListener(this::handleSessionEvent);
+ if (store instanceof MetadataStoreExtended metadataStoreExtended) {
+ metadataStoreExtended.registerSessionListener(this::handleSessionEvent);
} else {
// Since ServiceUnitStateMetadataStoreTableViewImpl has checked the configuration that named
// "zookeeperSessionExpiredPolicy", skip to print the duplicated log here.
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java
index 9e8c5a54a5d..ae7de3a47e9 100644
--- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java
@@ -99,7 +99,7 @@ public class AuditorPeriodicBookieCheckTest extends BookKeeperClusterTestCase {
getBookie(0),
getBookie(1))));
long underReplicatedLedger = -1;
- for (int i = 0; i < 10; i++) {
+ for (int i = 0; i < 20; i++) {
underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
if (underReplicatedLedger != -1) {
break;
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java
index 1d741c551dd..146e4f0dd58 100644
--- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java
@@ -30,6 +30,7 @@ import org.apache.bookkeeper.net.BookieId;
import org.apache.bookkeeper.util.TestUtils;
import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory;
import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.apache.pulsar.metadata.impl.ZKMetadataStore;
import org.apache.zookeeper.ZooKeeper;
import org.awaitility.Awaitility;
@@ -242,7 +243,9 @@ public class AutoRecoveryMainTest extends BookKeeperClusterTestCase {
(PulsarLedgerManagerFactory) pulsarMetadataClientDriver.getLedgerManagerFactory();
Field field = pulsarLedgerManagerFactory.getClass().getDeclaredField("store");
field.setAccessible(true);
- ZKMetadataStore zkMetadataStore = (ZKMetadataStore) field.get(pulsarLedgerManagerFactory);
+
+ DualMetadataStore store = (DualMetadataStore) field.get(pulsarLedgerManagerFactory);
+ ZKMetadataStore zkMetadataStore = (ZKMetadataStore) store.getSourceStore();
return zkMetadataStore.getZkClient();
}
}
diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java
index d78bc4c3a46..008817195cc 100644
--- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java
+++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java
@@ -87,6 +87,7 @@ import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory;
import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.apache.pulsar.metadata.impl.ZKMetadataStore;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooKeeper;
@@ -1242,7 +1243,8 @@ public class TestReplicationWorker extends BookKeeperClusterTestCase {
(PulsarLedgerManagerFactory) pulsarMetadataClientDriver.getLedgerManagerFactory();
Field field = pulsarLedgerManagerFactory.getClass().getDeclaredField("store");
field.setAccessible(true);
- ZKMetadataStore zkMetadataStore = (ZKMetadataStore) field.get(pulsarLedgerManagerFactory);
+ DualMetadataStore store = (DualMetadataStore) field.get(pulsarLedgerManagerFactory);
+ ZKMetadataStore zkMetadataStore = (ZKMetadataStore) store.getSourceStore();
return zkMetadataStore.getZkClient();
}
}
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/DualMetadataCacheTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/DualMetadataCacheTest.java
new file mode 100644
index 00000000000..88890466b78
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/DualMetadataCacheTest.java
@@ -0,0 +1,531 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import java.util.EnumSet;
+import java.util.Optional;
+import java.util.UUID;
+import lombok.AllArgsConstructor;
+import lombok.Cleanup;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.common.migration.MigrationPhase;
+import org.apache.pulsar.common.migration.MigrationState;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.metadata.api.CacheGetResult;
+import org.apache.pulsar.metadata.api.MetadataCache;
+import org.apache.pulsar.metadata.api.MetadataCacheConfig;
+import org.apache.pulsar.metadata.api.MetadataStore;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreFactory;
+import org.apache.pulsar.metadata.api.extended.CreateOption;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Slf4j
+public class DualMetadataCacheTest extends BaseMetadataStoreTest {
+
+ @Data
+ @AllArgsConstructor
+ @NoArgsConstructor
+ static class TestObject {
+ String name;
+ int value;
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ @AfterMethod(alwaysRun = true)
+ @Override
+ public void cleanup() throws Exception {
+ super.cleanup();
+ }
+
+ @Test
+ public void testCacheGetInNotStartedPhase() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ // Create object via source store
+ String path = prefix + "/test-obj";
+ TestObject obj = new TestObject("test", 42);
+ cache.create(path, obj).join();
+
+ // Read via cache
+ Optional<TestObject> result = cache.get(path).join();
+ assertTrue(result.isPresent());
+ assertEquals(result.get().getName(), "test");
+ assertEquals(result.get().getValue(), 42);
+ }
+
+ @Test
+ public void testCacheGetWithStatsInNotStartedPhase() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ // Create object
+ String path = prefix + "/test-obj";
+ TestObject obj = new TestObject("test", 42);
+ cache.create(path, obj).join();
+
+ // Read with stats
+ Optional<CacheGetResult<TestObject>> result = cache.getWithStats(path).join();
+ assertTrue(result.isPresent());
+ assertEquals(result.get().getValue().getName(), "test");
+ assertEquals(result.get().getValue().getValue(), 42);
+ assertNotNull(result.get().getStat());
+ }
+
+ @Test
+ public void testCacheGetIfCached() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ String path = prefix + "/test-obj";
+
+ // Initially not cached
+ Optional<TestObject> cached = cache.getIfCached(path);
+ assertEquals(cached, Optional.empty());
+
+ // Create object
+ TestObject obj = new TestObject("test", 42);
+ cache.create(path, obj).join();
+
+ // Now should be cached
+ Optional<TestObject> cachedAfter = cache.getIfCached(path);
+ assertTrue(cachedAfter.isPresent());
+ assertEquals(cachedAfter.get().getName(), "test");
+ }
+
+ @Test
+ public void testCacheGetChildren() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ // Create multiple objects
+ cache.create(prefix + "/child1", new TestObject("obj1", 1)).join();
+ cache.create(prefix + "/child2", new TestObject("obj2", 2)).join();
+ cache.create(prefix + "/child3", new TestObject("obj3", 3)).join();
+
+ // Get children
+ var children = cache.getChildren(prefix).join();
+ assertEquals(children.size(), 3);
+ assertTrue(children.contains("child1"));
+ assertTrue(children.contains("child2"));
+ assertTrue(children.contains("child3"));
+ }
+
+ @Test
+ public void testCacheExists() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ String path = prefix + "/test-obj";
+
+ // Initially doesn't exist
+ assertFalse(cache.exists(path).join());
+
+ // Create object
+ cache.create(path, new TestObject("test", 42)).join();
+
+ // Now exists
+ assertTrue(cache.exists(path).join());
+ }
+
+ @Test
+ public void testCacheReadModifyUpdateOrCreate() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ String path = prefix + "/test-obj";
+
+ // Create new object via readModifyUpdateOrCreate
+ TestObject result1 = cache.readModifyUpdateOrCreate(path, optObj -> {
+ assertFalse(optObj.isPresent());
+ return new TestObject("created", 100);
+ }).join();
+
+ assertEquals(result1.getName(), "created");
+ assertEquals(result1.getValue(), 100);
+
+ // Modify existing object
+ TestObject result2 = cache.readModifyUpdateOrCreate(path, optObj -> {
+ assertTrue(optObj.isPresent());
+ TestObject existing = optObj.get();
+ return new TestObject(existing.getName() + "-modified", existing.getValue() + 1);
+ }).join();
+
+ assertEquals(result2.getName(), "created-modified");
+ assertEquals(result2.getValue(), 101);
+ }
+
+ @Test
+ public void testCacheReadModifyUpdate() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ String path = prefix + "/test-obj";
+
+ // Create initial object
+ cache.create(path, new TestObject("initial", 1)).join();
+
+ // Modify it
+ TestObject result = cache.readModifyUpdate(path, obj -> {
+ return new TestObject(obj.getName() + "-updated", obj.getValue() * 2);
+ }).join();
+
+ assertEquals(result.getName(), "initial-updated");
+ assertEquals(result.getValue(), 2);
+
+ // Verify persisted
+ Optional<TestObject> verified = cache.get(path).join();
+ assertTrue(verified.isPresent());
+ assertEquals(verified.get().getName(), "initial-updated");
+ assertEquals(verified.get().getValue(), 2);
+ }
+
+ @Test
+ public void testCachePutWithOptions() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ String path = prefix + "/test-obj";
+ TestObject obj = new TestObject("test", 42);
+
+ // Put without options (creates or updates)
+ cache.put(path, obj, EnumSet.noneOf(CreateOption.class)).join();
+
+ // Verify
+ Optional<TestObject> result = cache.get(path).join();
+ assertTrue(result.isPresent());
+ assertEquals(result.get().getName(), "test");
+ }
+
+ @Test
+ public void testCacheDelete() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ String path = prefix + "/test-obj";
+
+ // Create object
+ cache.create(path, new TestObject("test", 42)).join();
+ assertTrue(cache.exists(path).join());
+
+ // Delete
+ cache.delete(path).join();
+ assertFalse(cache.exists(path).join());
+ }
+
+ @Test
+ public void testCacheInvalidate() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ String path = prefix + "/test-obj";
+
+ // Create and cache object
+ cache.create(path, new TestObject("test", 42)).join();
+ cache.get(path).join(); // Ensure it's cached
+
+ // Verify cached
+ Optional<TestObject> cached = cache.getIfCached(path);
+ assertTrue(cached.isPresent());
+
+ // Invalidate
+ cache.invalidate(path);
+
+ // Should not be in cache anymore (but still in store)
+ Optional<TestObject> cachedAfterInvalidate = cache.getIfCached(path);
+ assertEquals(cachedAfterInvalidate, Optional.empty());
+
+ // But should still exist in store
+ Optional<TestObject> fromStore = cache.get(path).join();
+ assertTrue(fromStore.isPresent());
+ }
+
+ @Test
+ public void testCacheInvalidateAll() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ // Create multiple objects
+ cache.create(prefix + "/obj1", new TestObject("test1", 1)).join();
+ cache.create(prefix + "/obj2", new TestObject("test2", 2)).join();
+ cache.create(prefix + "/obj3", new TestObject("test3", 3)).join();
+
+ // Load into cache
+ cache.get(prefix + "/obj1").join();
+ cache.get(prefix + "/obj2").join();
+ cache.get(prefix + "/obj3").join();
+
+ // Invalidate all
+ cache.invalidateAll();
+
+ // None should be cached
+ assertEquals(cache.getIfCached(prefix + "/obj1"), Optional.empty());
+ assertEquals(cache.getIfCached(prefix + "/obj2"), Optional.empty());
+ assertEquals(cache.getIfCached(prefix + "/obj3"), Optional.empty());
+ }
+
+ @Test
+ public void testCacheRefresh() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ String path = prefix + "/test-obj";
+
+ // Create object
+ cache.create(path, new TestObject("test", 42)).join();
+ cache.get(path).join(); // Ensure it's cached
+
+ // Modify directly in store (bypassing cache)
+ sourceStore.put(path,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(new TestObject("modified", 100)),
+ Optional.empty()).join();
+
+ // Refresh cache
+ cache.refresh(path);
+
+ // Wait a bit for refresh to complete
+ Thread.sleep(200);
+
+ // Should get updated value
+ Optional<TestObject> result = cache.get(path).join();
+ assertTrue(result.isPresent());
+ assertEquals(result.get().getName(), "modified");
+ assertEquals(result.get().getValue(), 100);
+ }
+
+ @Test
+ public void testCacheSwitchToTargetStore() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ String targetUrl = "memory:" + UUID.randomUUID();
+ @Cleanup
+ MetadataStore targetStore = MetadataStoreFactory.create(targetUrl,
+ MetadataStoreConfig.builder().build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class);
+
+ // Create object in source
+ String path = prefix + "/test-obj";
+ cache.create(path, new TestObject("source-obj", 1)).join();
+
+ // Verify it exists in source
+ assertTrue(cache.exists(path).join());
+
+ // Trigger migration - first PREPARATION
+ MigrationState preparationState = new MigrationState(MigrationPhase.PREPARATION, targetUrl);
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(preparationState),
+ Optional.empty()).join();
+
+ // Then COMPLETED phase
+ MigrationState completedState = new MigrationState(MigrationPhase.COMPLETED, targetUrl);
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(completedState),
+ Optional.empty()).join();
+
+ // Wait for dual store to switch and caches to update
+ Thread.sleep(1000);
+
+ // Create object in target via cache (should use target store now)
+ String targetPath = prefix + "/target-obj";
+ cache.create(targetPath, new TestObject("target-obj", 2)).join();
+
+ // Verify it exists in target store directly
+ Optional<byte[]> targetResult = targetStore.get(targetPath).join()
+ .map(gr -> gr.getValue());
+ assertTrue(targetResult.isPresent());
+ TestObject targetObj = ObjectMapperFactory.getMapper().reader().readValue(targetResult.get(), TestObject.class);
+ assertEquals(targetObj.getName(), "target-obj");
+ assertEquals(targetObj.getValue(), 2);
+ }
+
+ @Test
+ public void testMultipleCachesWithDifferentTypes() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ // Create caches for different types
+ MetadataCache<TestObject> objCache = dualStore.getMetadataCache(TestObject.class);
+ MetadataCache<String> strCache = dualStore.getMetadataCache(String.class);
+
+ // Use both caches
+ objCache.create(prefix + "/obj", new TestObject("test", 42)).join();
+ strCache.create(prefix + "/str", "test-string").join();
+
+ // Verify both work
+ Optional<TestObject> objResult = objCache.get(prefix + "/obj").join();
+ assertTrue(objResult.isPresent());
+ assertEquals(objResult.get().getName(), "test");
+
+ Optional<String> strResult = strCache.get(prefix + "/str").join();
+ assertTrue(strResult.isPresent());
+ assertEquals(strResult.get(), "test-string");
+ }
+
+ @Test
+ public void testCacheWithCustomConfig() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ // Create cache with custom config
+ MetadataCacheConfig cacheConfig = MetadataCacheConfig.builder()
+ .refreshAfterWriteMillis(1000)
+ .build();
+
+ MetadataCache<TestObject> cache = dualStore.getMetadataCache(TestObject.class, cacheConfig);
+
+ // Use the cache
+ String path = prefix + "/test-obj";
+ cache.create(path, new TestObject("test", 42)).join();
+
+ Optional<TestObject> result = cache.get(path).join();
+ assertTrue(result.isPresent());
+ assertEquals(result.get().getName(), "test");
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/DualMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/DualMetadataStoreTest.java
new file mode 100644
index 00000000000..65306b2155e
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/DualMetadataStoreTest.java
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.common.migration.MigrationPhase;
+import org.apache.pulsar.common.migration.MigrationState;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.metadata.api.GetResult;
+import org.apache.pulsar.metadata.api.MetadataStore;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreFactory;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.api.extended.SessionEvent;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
+import org.apache.pulsar.metadata.impl.ZKMetadataStore;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Slf4j
+public class DualMetadataStoreTest extends BaseMetadataStoreTest {
+
+
+ @BeforeMethod(alwaysRun = true)
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ @AfterMethod(alwaysRun = true)
+ @Override
+ public void cleanup() throws Exception {
+ super.cleanup();
+ }
+
+ @Test
+ public void testNotStartedPhaseRoutesToSource() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ // Write should go to source store
+ String path = prefix + "/test-key";
+ byte[] data = "test-data".getBytes(StandardCharsets.UTF_8);
+ dualStore.put(path, data, Optional.empty()).join();
+
+ // Verify data in source store
+ Optional<GetResult> result = sourceStore.get(path).join();
+ assertTrue(result.isPresent());
+ assertEquals(new String(result.get().getValue(), StandardCharsets.UTF_8), "test-data");
+
+ // Read should come from source store
+ Optional<GetResult> readResult = dualStore.get(path).join();
+ assertTrue(readResult.isPresent());
+ assertEquals(new String(readResult.get().getValue(), StandardCharsets.UTF_8), "test-data");
+ }
+
+ @Test
+ public void testPreparationPhaseBlocksWrites() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ // Set migration state to PREPARATION
+ MigrationState preparationState = new MigrationState(MigrationPhase.PREPARATION,
+ "memory:" + UUID.randomUUID());
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(preparationState),
+ Optional.empty()).join();
+
+ // Wait for dual store to detect migration
+ Thread.sleep(500);
+
+ // Writes should be blocked
+ String path = prefix + "/test-key";
+ byte[] data = "test-data".getBytes(StandardCharsets.UTF_8);
+ try {
+ dualStore.put(path, data, Optional.empty()).join();
+ fail("Should have thrown IllegalStateException");
+ } catch (CompletionException e) {
+ assertTrue(e.getCause() instanceof IllegalStateException);
+ assertTrue(e.getCause().getMessage().contains("Write operations not allowed during migrations"));
+ }
+
+ // Reads should still work
+ Optional<GetResult> result = dualStore.get(path).join();
+ assertFalse(result.isPresent());
+ }
+
+ @Test
+ public void testCopyingPhaseBlocksWrites() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ // Set migration state to COPYING
+ MigrationState copyingState = new MigrationState(MigrationPhase.COPYING,
+ "memory:" + UUID.randomUUID());
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(copyingState),
+ Optional.empty()).join();
+
+ // Wait for dual store to detect migration
+ Thread.sleep(500);
+
+ // Writes should be blocked
+ String path = prefix + "/test-key";
+ byte[] data = "test-data".getBytes(StandardCharsets.UTF_8);
+ try {
+ dualStore.put(path, data, Optional.empty()).join();
+ fail("Should have thrown IllegalStateException");
+ } catch (CompletionException e) {
+ assertTrue(e.getCause() instanceof IllegalStateException);
+ assertTrue(e.getCause().getMessage().contains("Write operations not allowed during migration"));
+ }
+ }
+
+ @Test
+ public void testCompletedPhaseRoutesToTarget() throws Exception {
+ String prefix = newKey();
+
+ @Cleanup
+ MetadataStore store =
+ MetadataStoreFactory.create(zks.getConnectionString(), MetadataStoreConfig.builder().build());
+
+ String oxiaService = "oxia://" + getOxiaServerConnectString();
+
+ @Cleanup
+ MetadataStore targetStore = MetadataStoreFactory.create(oxiaService,
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+
+ // Set migration state to PREPARATION
+ MigrationState preparationState = new MigrationState(MigrationPhase.PREPARATION, oxiaService);
+ store.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(preparationState),
+ Optional.empty()).join();
+
+ // Set migration state to COMPLETED
+ MigrationState completedState = new MigrationState(MigrationPhase.COMPLETED, oxiaService);
+ store.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(completedState),
+ Optional.empty()).join();
+
+ // Wait for dual store to detect migration and initialize target
+ Thread.sleep(1000);
+
+ // Write should go to target store
+ String path = prefix + "/test-key";
+ byte[] data = "test-data".getBytes(StandardCharsets.UTF_8);
+ store.put(path, data, Optional.empty()).join();
+
+ // Verify data in target store
+ Optional<GetResult> targetResult = targetStore.get(path).join();
+ assertTrue(targetResult.isPresent());
+ assertEquals(new String(targetResult.get().getValue(), StandardCharsets.UTF_8), "test-data");
+
+ // Read should come from target store
+ Optional<GetResult> readResult = store.get(path).join();
+ assertTrue(readResult.isPresent());
+ assertEquals(new String(readResult.get().getValue(), StandardCharsets.UTF_8), "test-data");
+ }
+
+ @Test
+ public void testFailedPhaseRoutesToSource() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ // Set migration state to FAILED
+ MigrationState failedState = new MigrationState(MigrationPhase.FAILED,
+ "memory:" + UUID.randomUUID());
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(failedState),
+ Optional.empty()).join();
+
+ // Wait for dual store to detect migration
+ Thread.sleep(500);
+
+ // Write should go to source store (migration failed)
+ String path = prefix + "/test-key";
+ byte[] data = "test-data".getBytes(StandardCharsets.UTF_8);
+ dualStore.put(path, data, Optional.empty()).join();
+
+ // Verify data in source store
+ Optional<GetResult> result = sourceStore.get(path).join();
+ assertTrue(result.isPresent());
+ assertEquals(new String(result.get().getValue(), StandardCharsets.UTF_8), "test-data");
+ }
+
+ @Test
+ public void testSessionLostEventDuringPreparation() throws Exception {
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ CountDownLatch sessionLostLatch = new CountDownLatch(1);
+ AtomicReference<SessionEvent> receivedEvent = new AtomicReference<>();
+
+ dualStore.registerSessionListener(event -> {
+ receivedEvent.set(event);
+ if (event == SessionEvent.SessionLost) {
+ sessionLostLatch.countDown();
+ }
+ });
+
+ // Set migration state to PREPARATION to trigger SessionLost
+ MigrationState preparationState = new MigrationState(MigrationPhase.PREPARATION,
+ "memory:" + UUID.randomUUID());
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(preparationState),
+ Optional.empty()).join();
+
+ // Wait for SessionLost event
+ assertTrue(sessionLostLatch.await(5, TimeUnit.SECONDS));
+ assertEquals(receivedEvent.get(), SessionEvent.SessionLost);
+ }
+
+ @Test
+ public void testSessionReestablishedEventOnCompletion() throws Exception {
+ @Cleanup
+ MetadataStore sourceStore =
+ new ZKMetadataStore(zks.getConnectionString(), MetadataStoreConfig.builder().build(), true);
+
+ String targetUrl = "memory:" + UUID.randomUUID();
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ CountDownLatch sessionReestablishedLatch = new CountDownLatch(1);
+ List<SessionEvent> receivedEvents = new ArrayList<>();
+
+ dualStore.registerSessionListener(event -> {
+ receivedEvents.add(event);
+ if (event == SessionEvent.SessionReestablished) {
+ sessionReestablishedLatch.countDown();
+ }
+ });
+
+ // First trigger PREPARATION (SessionLost)
+ MigrationState preparationState = new MigrationState(MigrationPhase.PREPARATION, targetUrl);
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(preparationState),
+ Optional.empty()).join();
+
+ Thread.sleep(500);
+
+ // Then trigger COMPLETED (SessionReestablished)
+ MigrationState completedState = new MigrationState(MigrationPhase.COMPLETED, targetUrl);
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(completedState),
+ Optional.empty()).join();
+
+ // Wait for SessionReestablished event
+ assertTrue(sessionReestablishedLatch.await(5, TimeUnit.SECONDS));
+ assertTrue(receivedEvents.contains(SessionEvent.SessionLost));
+ assertTrue(receivedEvents.contains(SessionEvent.SessionReestablished));
+ }
+
+ @Test
+ public void testSessionReestablishedEventOnFailure() throws Exception {
+ MetadataStore zkStore =
+ new ZKMetadataStore(zks.getConnectionString(), MetadataStoreConfig.builder().build(), true);
+
+ @Cleanup
+ MetadataStoreExtended dualStore = new DualMetadataStore(zkStore, MetadataStoreConfig.builder().build());
+
+ CountDownLatch sessionReestablishedLatch = new CountDownLatch(1);
+ List<SessionEvent> receivedEvents = new ArrayList<>();
+
+ dualStore.registerSessionListener(event -> {
+ receivedEvents.add(event);
+ if (event == SessionEvent.SessionReestablished) {
+ sessionReestablishedLatch.countDown();
+ }
+ });
+
+ // First trigger PREPARATION (SessionLost)
+ MigrationState preparationState = new MigrationState(MigrationPhase.PREPARATION,
+ "memory:" + UUID.randomUUID());
+ zkStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(preparationState),
+ Optional.empty()).join();
+
+ Thread.sleep(500);
+
+ // Then trigger FAILED (SessionReestablished)
+ MigrationState failedState = new MigrationState(MigrationPhase.FAILED,
+ "memory:" + UUID.randomUUID());
+ zkStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(failedState),
+ Optional.empty()).join();
+
+ // Wait for SessionReestablished event
+ assertTrue(sessionReestablishedLatch.await(5, TimeUnit.SECONDS));
+ assertTrue(receivedEvents.contains(SessionEvent.SessionLost));
+ assertTrue(receivedEvents.contains(SessionEvent.SessionReestablished));
+ }
+
+ @Test
+ public void testParticipantRegistration() throws Exception {
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ // Verify participant registration node exists
+ List<String> participants = sourceStore.getChildren(MigrationState.PARTICIPANTS_PATH).join();
+ log.info("participants: {}", participants);
+ assertEquals(participants.size(), 1);
+ assertTrue(participants.get(0).startsWith("id-"));
+ }
+
+ @Test
+ public void testDeleteOperationRouting() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore dualStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+ assertEquals(dualStore.getClass(), DualMetadataStore.class);
+
+ // Create a key in NOT_STARTED phase
+ String path = prefix + "/test-key";
+ byte[] data = "test-data".getBytes(StandardCharsets.UTF_8);
+ dualStore.put(path, data, Optional.empty()).join();
+
+ // Delete should work in NOT_STARTED phase
+ dualStore.delete(path, Optional.empty()).join();
+ assertFalse(dualStore.exists(path).join());
+ }
+
+ @Test
+ public void testParticipantRegistrationWithChroot() throws Exception {
+ // Test with chroot path to ensure participant registration works
+ String chrootPath = "/test-chroot-" + UUID.randomUUID().toString().substring(0, 8);
+ String zkConnectString = zks.getConnectionString() + chrootPath;
+
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zkConnectString,
+ MetadataStoreConfig.builder().build());
+
+ // Verify participant registration node exists under chroot
+ List<String> participants = sourceStore.getChildren(MigrationState.PARTICIPANTS_PATH).join();
+ log.info("Participants in chroot {}: {}", chrootPath, participants);
+ assertEquals(participants.size(), 1);
+ assertTrue(participants.get(0).startsWith("id-"));
+
+ // Verify the parent path was created
+ assertTrue(sourceStore.exists(MigrationState.PARTICIPANTS_PATH).join());
+ }
+
+ @Test
+ public void testExistsOperationRouting() throws Exception {
+ String prefix = newKey();
+ @Cleanup
+ MetadataStore sourceStore = new ZKMetadataStore(zks.getConnectionString(),
+ MetadataStoreConfig.builder().build(), false);
+
+ String targetUrl = "memory:" + UUID.randomUUID();
+ @Cleanup
+ MetadataStore targetStore = MetadataStoreFactory.create(targetUrl,
+ MetadataStoreConfig.builder().build());
+
+ @Cleanup
+ DualMetadataStore dualStore = new DualMetadataStore(sourceStore,
+ MetadataStoreConfig.builder().build());
+
+ String path = prefix + "/test-key";
+ byte[] data = "test-data".getBytes(StandardCharsets.UTF_8);
+
+ // Create in source
+ sourceStore.put(path, data, Optional.empty()).join();
+
+ // Exists should check source in NOT_STARTED phase
+ assertTrue(dualStore.exists(path).join());
+
+ // First trigger PREPARATION (SessionLost)
+ MigrationState preparationState = new MigrationState(MigrationPhase.PREPARATION, targetUrl);
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(preparationState),
+ Optional.empty()).join();
+
+ // Switch to COMPLETED phase
+ MigrationState completedState = new MigrationState(MigrationPhase.COMPLETED, targetUrl);
+ sourceStore.put(MigrationState.MIGRATION_FLAG_PATH,
+ ObjectMapperFactory.getMapper().writer().writeValueAsBytes(completedState),
+ Optional.empty()).join();
+
+ Thread.sleep(1000);
+
+ // Create in target
+ String targetPath = prefix + "/target-key";
+ targetStore.put(targetPath, data, Optional.empty()).join();
+
+ // Exists should check target in COMPLETED phase
+ assertTrue(dualStore.exists(targetPath).join());
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java
index c3ae22aeb54..d188b63ba15 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java
@@ -69,7 +69,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreFactory;
import org.apache.pulsar.metadata.api.NotificationType;
import org.apache.pulsar.metadata.api.Stat;
import org.apache.pulsar.metadata.api.extended.CreateOption;
-import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl;
+import org.apache.pulsar.metadata.impl.DualMetadataCache;
import org.apache.pulsar.metadata.impl.LocalMemoryMetadataStore;
import org.awaitility.Awaitility;
import org.mockito.stubbing.Answer;
@@ -188,9 +188,9 @@ public class MetadataCacheTest extends BaseMetadataStoreTest {
@Cleanup
MetadataStore store2 = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().build());
- MetadataCacheImpl<MyClass> objCache1 = (MetadataCacheImpl<MyClass>) store1.getMetadataCache(MyClass.class);
+ MetadataCache<MyClass> objCache1 = store1.getMetadataCache(MyClass.class);
- MetadataCacheImpl<MyClass> objCache2 = (MetadataCacheImpl<MyClass>) store2.getMetadataCache(MyClass.class);
+ MetadataCache<MyClass> objCache2 = store2.getMetadataCache(MyClass.class);
AtomicReference<MyClass> storeObj = new AtomicReference<MyClass>();
store2.registerListener(n -> {
if (n.getType() == NotificationType.Modified) {
@@ -529,13 +529,18 @@ public class MetadataCacheTest extends BaseMetadataStoreTest {
.setMax(1, TimeUnit.SECONDS)
.setMandatoryStop(3, TimeUnit.SECONDS)).build());
- Field metadataCacheField = cache.getClass().getDeclaredField("objCache");
+ MetadataCache<MyClass> cacheRef = cache;
+ if (cache instanceof DualMetadataCache dc) {
+ cacheRef = (MetadataCache<MyClass>) dc.getMetadataCache().get();
+ }
+
+ Field metadataCacheField = cacheRef.getClass().getDeclaredField("objCache");
metadataCacheField.setAccessible(true);
- var objCache = metadataCacheField.get(cache);
+ var objCache = metadataCacheField.get(cacheRef);
var spyObjCache = (AsyncLoadingCache<?, ?>) spy(objCache);
doAnswer((Answer<CompletableFuture<MyClass>>) invocation -> CompletableFuture.failedFuture(
new MetadataStoreException.BadVersionException(""))).when(spyObjCache).get(any());
- metadataCacheField.set(cache, spyObjCache);
+ metadataCacheField.set(cacheRef, spyObjCache);
// Test three times to ensure that the retry works each time.
for (int i = 0; i < 3; i++) {
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
index 9bd2ddd5e8f..061bccad931 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java
@@ -59,6 +59,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreFactory;
import org.apache.pulsar.metadata.api.Notification;
import org.apache.pulsar.metadata.api.NotificationType;
import org.apache.pulsar.metadata.api.Stat;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.apache.pulsar.metadata.impl.PulsarZooKeeperClient;
import org.apache.pulsar.metadata.impl.ZKMetadataStore;
import org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStore;
@@ -471,8 +472,8 @@ public class MetadataStoreTest extends BaseMetadataStoreTest {
}
MetadataStoreConfig config = builder.build();
@Cleanup
- ZKMetadataStore store = (ZKMetadataStore) MetadataStoreFactory.create(zks.getConnectionString(), config);
- ZooKeeper zkClient = store.getZkClient();
+ DualMetadataStore store = (DualMetadataStore) MetadataStoreFactory.create(zks.getConnectionString(), config);
+ ZooKeeper zkClient = ((ZKMetadataStore) store.getSourceStore()).getZkClient();
assertTrue(zkClient.getClientConfig().isSaslClientEnabled());
final Runnable verify = () -> {
String currentThreadName = Thread.currentThread().getName();
@@ -526,9 +527,9 @@ public class MetadataStoreTest extends BaseMetadataStoreTest {
builder.configFilePath("src/test/resources/zk_client_disabled_sasl.conf");
MetadataStoreConfig config = builder.build();
@Cleanup
- ZKMetadataStore store = (ZKMetadataStore) MetadataStoreFactory.create(zks.getConnectionString(), config);
-
- PulsarZooKeeperClient zkClient = (PulsarZooKeeperClient) store.getZkClient();
+ DualMetadataStore store = (DualMetadataStore) MetadataStoreFactory.create(zks.getConnectionString(), config);
+ PulsarZooKeeperClient zkClient =
+ (PulsarZooKeeperClient) ((ZKMetadataStore) store.getSourceStore()).getZkClient();
assertFalse(zkClient.getClientConfig().isSaslClientEnabled());
zkClient.process(new WatchedEvent(Watcher.Event.EventType.None, Watcher.Event.KeeperState.Expired, null));
@@ -669,6 +670,7 @@ public class MetadataStoreTest extends BaseMetadataStoreTest {
List<String> subPaths = new ArrayList<>(store.getChildren("/").get());
subPaths.remove("zookeeper"); // ignored
+ subPaths.remove("pulsar"); // ignored
assertThat(subPaths).containsExactlyInAnyOrderElementsOf(Set.of("a", "b"));
List<String> subPaths2 = store.getChildren("/a").get();
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MigrationCoordinatorTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MigrationCoordinatorTest.java
new file mode 100644
index 00000000000..466a3b2aadb
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MigrationCoordinatorTest.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import java.nio.charset.StandardCharsets;
+import java.util.EnumSet;
+import java.util.Optional;
+import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.common.migration.MigrationPhase;
+import org.apache.pulsar.common.migration.MigrationState;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.metadata.api.GetResult;
+import org.apache.pulsar.metadata.api.MetadataStore;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreFactory;
+import org.apache.pulsar.metadata.api.extended.CreateOption;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.coordination.impl.MigrationCoordinator;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Slf4j
+public class MigrationCoordinatorTest extends BaseMetadataStoreTest {
+
+ protected String getOxiaServerConnectString() {
+ return "oxia://" + super.getOxiaServerConnectString();
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ @AfterMethod(alwaysRun = true)
+ @Override
+ public void cleanup() throws Exception {
+ super.cleanup();
+ }
+
+ @Test
+ public void testPersistentDataCopy() throws Exception {
+ String prefix = newKey();
+
+ @Cleanup
+ MetadataStoreExtended sourceStore =
+ (MetadataStoreExtended) MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().build());
+
+ String targetUrl = getOxiaServerConnectString();
+
+ @Cleanup
+ MetadataStore targetStore = MetadataStoreFactory.create(targetUrl, MetadataStoreConfig.builder().build());
+
+ // Create persistent nodes
+ String key1 = prefix + "/persistent/key1";
+ String key2 = prefix + "/persistent/key2";
+ String key3 = prefix + "/persistent/nested/key3";
+
+ sourceStore.put(key1, "value1".getBytes(StandardCharsets.UTF_8), Optional.empty()).join();
+ sourceStore.put(key2, "value2".getBytes(StandardCharsets.UTF_8), Optional.empty()).join();
+ sourceStore.put(key3, "value3".getBytes(StandardCharsets.UTF_8), Optional.empty()).join();
+
+ // Create ephemeral node (should NOT be copied)
+ String ephemeralKey = prefix + "/ephemeral/key";
+ sourceStore.put(ephemeralKey, "ephemeral-value".getBytes(StandardCharsets.UTF_8),
+ Optional.empty(), EnumSet.of(CreateOption.Ephemeral)).join();
+
+ // Start migration
+ MigrationCoordinator coordinator = new MigrationCoordinator(sourceStore, targetUrl);
+ coordinator.startMigration();
+
+ Optional<GetResult> result = sourceStore.get(MigrationState.MIGRATION_FLAG_PATH).join();
+ assertTrue(result.isPresent());
+ MigrationState state = ObjectMapperFactory.getMapper().reader()
+ .readValue(result.get().getValue(), MigrationState.class);
+ assertEquals(state.getPhase(), MigrationPhase.COMPLETED);
+
+ // Verify persistent nodes were copied
+ Optional<GetResult> target1 = targetStore.get(key1).join();
+ assertTrue(target1.isPresent());
+ assertEquals(new String(target1.get().getValue(), StandardCharsets.UTF_8), "value1");
+
+ Optional<GetResult> target2 = targetStore.get(key2).join();
+ assertTrue(target2.isPresent());
+ assertEquals(new String(target2.get().getValue(), StandardCharsets.UTF_8), "value2");
+
+ Optional<GetResult> target3 = targetStore.get(key3).join();
+ assertTrue(target3.isPresent());
+ assertEquals(new String(target3.get().getValue(), StandardCharsets.UTF_8), "value3");
+
+ // Verify ephemeral node is in the target store
+ Optional<GetResult> targetEphemeral = targetStore.get(ephemeralKey).join();
+ assertTrue(targetEphemeral.isPresent());
+ assertEquals(new String(targetEphemeral.get().getValue(), StandardCharsets.UTF_8), "ephemeral-value");
+ }
+
+ @Test
+ public void testVersionPreservation() throws Exception {
+ String prefix = newKey();
+
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ String targetUrl = getOxiaServerConnectString();
+
+ @Cleanup
+ MetadataStore targetStore = MetadataStoreFactory.create(targetUrl,
+ MetadataStoreConfig.builder().build());
+
+ // Create a node and update it multiple times to get a specific version
+ String key = prefix + "/versioned-key";
+ sourceStore.put(key, "v1".getBytes(StandardCharsets.UTF_8), Optional.empty()).join();
+ sourceStore.put(key, "v2".getBytes(StandardCharsets.UTF_8), Optional.empty()).join();
+ sourceStore.put(key, "v3".getBytes(StandardCharsets.UTF_8), Optional.empty()).join();
+
+ // Get the version from source
+ Optional<GetResult> sourceResult = sourceStore.get(key).join();
+ assertTrue(sourceResult.isPresent());
+ long sourceVersion = sourceResult.get().getStat().getVersion();
+
+ // Start migration
+ MigrationCoordinator coordinator = new MigrationCoordinator(sourceStore, targetUrl);
+ coordinator.startMigration();
+
+ Optional<GetResult> result = sourceStore.get(MigrationState.MIGRATION_FLAG_PATH).join();
+ assertTrue(result.isPresent());
+ MigrationState state = ObjectMapperFactory.getMapper().reader()
+ .readValue(result.get().getValue(), MigrationState.class);
+ assertEquals(state.getPhase(), MigrationPhase.COMPLETED);
+
+ // Verify version and modification count were preserved in target
+ Optional<GetResult> targetResult = targetStore.get(key).join();
+ assertTrue(targetResult.isPresent());
+ assertEquals(targetResult.get().getStat().getVersion(), sourceVersion);
+ assertEquals(new String(targetResult.get().getValue(), StandardCharsets.UTF_8), "v3");
+ }
+
+ @Test
+ public void testEmptyMetadataMigration() throws Exception {
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().build());
+
+ String targetUrl = getOxiaServerConnectString();
+
+ // Start migration with empty metadata
+ MigrationCoordinator coordinator = new MigrationCoordinator(sourceStore, targetUrl);
+ coordinator.startMigration();
+
+ Optional<GetResult> result = sourceStore.get(MigrationState.MIGRATION_FLAG_PATH).join();
+ assertTrue(result.isPresent());
+ MigrationState state = ObjectMapperFactory.getMapper().reader()
+ .readValue(result.get().getValue(), MigrationState.class);
+ assertEquals(state.getPhase(), MigrationPhase.COMPLETED);
+ }
+
+ @Test
+ public void testLargeDatasetMigration() throws Exception {
+ String prefix = newKey();
+
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ String targetUrl = getOxiaServerConnectString();
+
+ @Cleanup
+ MetadataStore targetStore = MetadataStoreFactory.create(targetUrl,
+ MetadataStoreConfig.builder().build());
+
+ // Create a larger dataset (100 nodes)
+ int nodeCount = 100;
+ for (int i = 0; i < nodeCount; i++) {
+ String key = prefix + "/data/node-" + i;
+ String value = "value-" + i;
+ sourceStore.put(key, value.getBytes(StandardCharsets.UTF_8), Optional.empty()).join();
+ }
+
+ long startTime = System.currentTimeMillis();
+
+ // Start migration
+ MigrationCoordinator coordinator = new MigrationCoordinator(sourceStore, targetUrl);
+ coordinator.startMigration();
+
+ Optional<GetResult> result = sourceStore.get(MigrationState.MIGRATION_FLAG_PATH).join();
+ assertTrue(result.isPresent());
+ MigrationState state = ObjectMapperFactory.getMapper().reader()
+ .readValue(result.get().getValue(), MigrationState.class);
+ assertEquals(state.getPhase(), MigrationPhase.COMPLETED);
+
+ long duration = System.currentTimeMillis() - startTime;
+ log.info("Migration of {} nodes completed in {} ms", nodeCount, duration);
+
+ // Verify all nodes were copied
+ for (int i = 0; i < nodeCount; i++) {
+ String key = prefix + "/data/node-" + i;
+ Optional<GetResult> targetResult = targetStore.get(key).join();
+ assertTrue(targetResult.isPresent(), "Node " + key + " should exist in target");
+ assertEquals(new String(targetResult.get().getValue(), StandardCharsets.UTF_8),
+ "value-" + i);
+ }
+ }
+
+ @Test
+ public void testNestedPathMigration() throws Exception {
+ String prefix = newKey();
+
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ String targetUrl = getOxiaServerConnectString();
+
+ @Cleanup
+ MetadataStore targetStore = MetadataStoreFactory.create(targetUrl,
+ MetadataStoreConfig.builder().build());
+
+ // Create nested paths
+ sourceStore.put(prefix + "/level1/key1", "value1".getBytes(StandardCharsets.UTF_8),
+ Optional.empty()).join();
+ sourceStore.put(prefix + "/level1/level2/key2", "value2".getBytes(StandardCharsets.UTF_8),
+ Optional.empty()).join();
+ sourceStore.put(prefix + "/level1/level2/level3/key3",
+ "value3".getBytes(StandardCharsets.UTF_8), Optional.empty()).join();
+
+ // Start migration
+ MigrationCoordinator coordinator = new MigrationCoordinator(sourceStore, targetUrl);
+ coordinator.startMigration();
+
+ Optional<GetResult> result = sourceStore.get(MigrationState.MIGRATION_FLAG_PATH).join();
+ assertTrue(result.isPresent());
+ MigrationState state = ObjectMapperFactory.getMapper().reader()
+ .readValue(result.get().getValue(), MigrationState.class);
+ assertEquals(state.getPhase(), MigrationPhase.COMPLETED);
+
+ // Verify all nested paths were copied
+ Optional<GetResult> target1 = targetStore.get(prefix + "/level1/key1").join();
+ assertTrue(target1.isPresent());
+ assertEquals(new String(target1.get().getValue(), StandardCharsets.UTF_8), "value1");
+
+ Optional<GetResult> target2 = targetStore.get(prefix + "/level1/level2/key2").join();
+ assertTrue(target2.isPresent());
+ assertEquals(new String(target2.get().getValue(), StandardCharsets.UTF_8), "value2");
+
+ Optional<GetResult> target3 = targetStore.get(prefix + "/level1/level2/level3/key3").join();
+ assertTrue(target3.isPresent());
+ assertEquals(new String(target3.get().getValue(), StandardCharsets.UTF_8), "value3");
+ }
+
+ @Test
+ public void testMigrationStateStructure() throws Exception {
+ @Cleanup
+ MetadataStore sourceStore = MetadataStoreFactory.create(zks.getConnectionString(),
+ MetadataStoreConfig.builder().fsyncEnable(false).build());
+
+ String targetUrl = getOxiaServerConnectString();
+
+ // Start migration
+ MigrationCoordinator coordinator = new MigrationCoordinator(sourceStore, targetUrl);
+ coordinator.startMigration();
+
+ // Verify migration state structure
+ Optional<GetResult> result = sourceStore.get(MigrationState.MIGRATION_FLAG_PATH).join();
+ assertTrue(result.isPresent());
+
+ MigrationState state = ObjectMapperFactory.getMapper().reader()
+ .readValue(result.get().getValue(), MigrationState.class);
+
+ assertNotNull(state.getPhase());
+ assertNotNull(state.getTargetUrl());
+ assertEquals(state.getTargetUrl(), targetUrl);
+
+ // Phase should be PREPARATION or COPYING or COMPLETED
+ assertTrue(state.getPhase() == MigrationPhase.PREPARATION
+ || state.getPhase() == MigrationPhase.COPYING
+ || state.getPhase() == MigrationPhase.COMPLETED);
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java
index cf8000ebef6..e9c41766dbc 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java
@@ -41,6 +41,7 @@ import org.apache.pulsar.metadata.api.coordination.ResourceLock;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
import org.apache.pulsar.metadata.api.extended.SessionEvent;
import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
import org.apache.pulsar.metadata.impl.ZKMetadataStore;
import org.awaitility.Awaitility;
import org.testng.annotations.Test;
@@ -168,8 +169,8 @@ public class ZKSessionTest extends BaseMetadataStoreTest {
ResourceLock<String> lock = lm1.acquireLock(path, "value-1").join();
-
- zks.expireSession(((ZKMetadataStore) store).getZkSessionId());
+ var zkStore = (ZKMetadataStore) ((DualMetadataStore) store).getSourceStore();
+ zks.expireSession(zkStore.getZkSessionId());
SessionEvent e = sessionEvents.poll(5, TimeUnit.SECONDS);
assertEquals(e, SessionEvent.ConnectionLost);
@@ -212,7 +213,8 @@ public class ZKSessionTest extends BaseMetadataStoreTest {
LeaderElectionState les = leaderElectionEvents.poll(5, TimeUnit.SECONDS);
assertEquals(les, LeaderElectionState.Leading);
// --- expire session
- zks.expireSession(((ZKMetadataStore) store).getZkSessionId());
+ var zkStore = (ZKMetadataStore) ((DualMetadataStore) store).getSourceStore();
+ zks.expireSession(zkStore.getZkSessionId());
SessionEvent e = sessionEvents.poll(5, TimeUnit.SECONDS);
assertEquals(e, SessionEvent.ConnectionLost);