You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by GitBox <gi...@apache.org> on 2022/10/26 16:01:49 UTC

[GitHub] [bookkeeper] hangc0276 commented on a diff in pull request #3457: BP-56: Support data migration

hangc0276 commented on code in PR #3457:
URL: https://github.com/apache/bookkeeper/pull/3457#discussion_r1005877921


##########
bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookies/MigrationReplicasCommand.java:
##########
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.tools.cli.commands.bookies;
+
+import com.beust.jcommander.Parameter;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.discover.BookieServiceInfo;
+import org.apache.bookkeeper.http.HttpRouter;
+import org.apache.bookkeeper.http.HttpServer;
+import org.apache.bookkeeper.meta.MigrationManager;
+import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.tools.cli.helpers.BookieCommand;
+import org.apache.bookkeeper.tools.framework.CliFlags;
+import org.apache.bookkeeper.tools.framework.CliSpec;
+import org.apache.bookkeeper.util.HttpClientUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Migrate the data of the specified replica to other bookie nodes.
+ * */
+public class MigrationReplicasCommand extends BookieCommand<MigrationReplicasCommand.MigrationReplicasFlags> {
+    static final Logger LOG = LoggerFactory.getLogger(MigrationReplicasCommand.class);
+    private static final String NAME = "migrationReplicas";
+    private static final String DESC = "Migrate the data of the specified replica to other bookie nodes";
+    public MigrationReplicasCommand() {
+        this(new MigrationReplicasFlags());
+    }
+    private MigrationReplicasCommand(MigrationReplicasFlags flags) {
+        super(CliSpec.<MigrationReplicasFlags>newBuilder()
+                .withName(NAME)
+                .withDescription(DESC)
+                .withFlags(flags)
+                .build());
+    }
+
+    /**
+     * Flags for migrationReplicas command.
+     */
+    @Accessors(fluent = true)
+    @Setter
+    public static class MigrationReplicasFlags extends CliFlags {
+
+        @Parameter(names = { "-b", "--bookieIds" }, description = "Bookies corresponding to the migrated replica, "
+                + "eg: bookieId1,bookieId2,bookieId3")
+        private List<String> bookieIdsOfMigratedReplica;
+
+        @Parameter(names = { "-l", "--ledgerIds" }, description = "The ledgerIds corresponding to the migrated replica,"
+                + "eg: ledgerId1,ledgerId2,ledgerId3. The default is empty, which means that all replica data on these "
+                + "bookie nodes will be migrated to other bookie nodes")
+        private List<Long> ledgerIdsOfMigratedReplica = Collections.emptyList();
+
+        @Parameter(names = { "-r", "--readOnly" }, description = "Whether to set the bookie nodes of the migrated data"
+                + " to readOnly, the default is false")
+        private boolean readOnly = false;
+    }
+
+    @Override
+    public boolean apply(ServerConfiguration conf, MigrationReplicasFlags cmdFlags) {
+        try (BookKeeperAdmin admin = new BookKeeperAdmin(new ClientConfiguration(conf))) {
+            if (cmdFlags.readOnly) {
+                if (!conf.isHttpServerEnabled()) {
+                    throw new RuntimeException("Please enable http service first, config httpServerEnabled is false!");
+                }
+                Set<BookieId> toSwitchBookieIdsSet = new HashSet<>();
+                for (String bookieIdStr : cmdFlags.bookieIdsOfMigratedReplica) {
+                    toSwitchBookieIdsSet.add(BookieId.parse(bookieIdStr));
+                }
+
+                Collection<BookieId> readOnlyBookies = admin.getReadOnlyBookies();
+                toSwitchBookieIdsSet.removeAll(readOnlyBookies);
+                Set<BookieId> switchedBookies = new HashSet<>();
+                if (!switchToReadonly(admin, toSwitchBookieIdsSet, switchedBookies, true)) {
+                    LOG.warn("Some bookie nodes that fail to set readonly, "
+                            + "and the successful bookies fall back to the previous state!");
+                    Set<BookieId> fallbackSuccessfulBookies = new HashSet<>();
+                    if (!switchToReadonly(admin, switchedBookies, fallbackSuccessfulBookies, false)) {
+                        switchedBookies.removeAll(fallbackSuccessfulBookies);
+                        LOG.error(String.format("Fallback failed! Failed nodes:%s", switchedBookies));
+                    }
+                    return false;
+                }
+            }
+
+            return migrationReplicas(admin, conf, cmdFlags);
+        } catch (Exception e) {
+            throw new UncheckedExecutionException(e.getMessage(), e);
+        }
+    }
+
+    private boolean switchToReadonly(BookKeeperAdmin admin,

Review Comment:
   The method can switch bookie's mode between readWrite and readOnly, please rename this method.



##########
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java:
##########
@@ -1951,6 +1953,77 @@ public int runCmd(CommandLine cmdLine) throws Exception {
         }
     }
 
+    /**
+     * Migrate the specified ledger data on some bookie to other bookies.
+     * */
+    class MigrationReplicasCmd extends MyCommand {
+
+        MigrationReplicasCmd() {
+            super(CMD_MIGRATIONREPLICAS);
+            Option bookiesOpt = new Option("b", "bookieIds", true,
+                    "Bookies corresponding to the migrated replica");
+            Option ledgersOpt = new Option("l", "ledgerIds", true,
+                    "The ledgerIds corresponding to the migrated replica");
+            Option readOnlyOpt = new Option("r", "readOnly", true,
+                    "Whether to set the bookie nodes of the migrated data to readOnly, the default is true");

Review Comment:
   Default is `true`? Not sync with `MigrationReplicasFlags` default value.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@bookkeeper.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org