You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2020/09/29 16:41:59 UTC

[hadoop-ozone] branch HDDS-3698-upgrade updated: HDDS-4227. Implement a 'Prepare For Upgrade' step in OM that applies all committed Ratis transactions. (#1430)

This is an automated email from the ASF dual-hosted git repository.

avijayan pushed a commit to branch HDDS-3698-upgrade
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git


The following commit(s) were added to refs/heads/HDDS-3698-upgrade by this push:
     new a2788b7  HDDS-4227. Implement a 'Prepare For Upgrade' step in OM that applies all committed Ratis transactions. (#1430)
a2788b7 is described below

commit a2788b72cf068ab1723cd1c64670acca4db78e8f
Author: avijayanhwx <14...@users.noreply.github.com>
AuthorDate: Tue Sep 29 09:41:43 2020 -0700

    HDDS-4227. Implement a 'Prepare For Upgrade' step in OM that applies all committed Ratis transactions. (#1430)
---
 hadoop-hdds/common/pom.xml                         |  8 ++
 .../hadoop/hdds/ratis/RatisUpgradeUtils.java       | 96 +++++++++++++++++++++
 .../hadoop/hdds/ratis/TestRatisUpgradeUtils.java   | 97 ++++++++++++++++++++++
 .../org/apache/hadoop/ozone/om/OMConfigKeys.java   |  4 +
 .../apache/hadoop/ozone/om/OMStarterInterface.java |  2 +
 .../org/apache/hadoop/ozone/om/OzoneManager.java   | 64 ++++++++++++--
 .../hadoop/ozone/om/OzoneManagerStarter.java       | 38 +++++++++
 .../ozone/om/ratis/OzoneManagerDoubleBuffer.java   |  2 +-
 .../ozone/om/ratis/OzoneManagerRatisServer.java    |  4 +
 .../hadoop/ozone/om/TestOzoneManagerStarter.java   | 11 +++
 10 files changed, 319 insertions(+), 7 deletions(-)

diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index cc171f1..d7abf0e 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -188,6 +188,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>hamcrest-all</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.codahale.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>3.0.2</version>
+      <scope>test</scope>
+      <!-- Needed for mocking RaftServerImpl -->
+    </dependency>
+
   </dependencies>
 
   <build>
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisUpgradeUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisUpgradeUtils.java
new file mode 100644
index 0000000..796668d
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisUpgradeUtils.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.ratis;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.ratis.protocol.RaftGroup;
+import org.apache.ratis.server.impl.RaftServerImpl;
+import org.apache.ratis.server.impl.RaftServerProxy;
+import org.apache.ratis.statemachine.StateMachine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Ratis utility functions.
+ */
+public final class RatisUpgradeUtils {
+
+  private RatisUpgradeUtils() {
+  }
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(RatisUpgradeUtils.class);
+
+  /**
+   * Flush all committed transactions in a given Raft Server for a given group.
+   * @param stateMachine state machine to use
+   * @param raftGroup raft group
+   * @param server Raft server proxy instance.
+   * @param maxTimeToWaitSeconds Max time to wait before declaring failure.
+   * @throws InterruptedException when interrupted
+   * @throws IOException on error while waiting
+   */
+  public static void waitForAllTxnsApplied(
+      StateMachine stateMachine,
+      RaftGroup raftGroup,
+      RaftServerProxy server,
+      long maxTimeToWaitSeconds,
+      long timeBetweenRetryInSeconds)
+      throws InterruptedException, IOException {
+
+    long intervalTime = TimeUnit.SECONDS.toMillis(timeBetweenRetryInSeconds);
+    long endTime = System.currentTimeMillis() +
+        TimeUnit.SECONDS.toMillis(maxTimeToWaitSeconds);
+    boolean success = false;
+    while (System.currentTimeMillis() < endTime) {
+      success = checkIfAllTransactionsApplied(stateMachine, server, raftGroup);
+      if (success) {
+        break;
+      }
+      Thread.sleep(intervalTime);
+    }
+
+    if (!success) {
+      throw new IOException(String.format("After waiting for %d seconds, " +
+          "State Machine has not applied  all the transactions.",
+          maxTimeToWaitSeconds));
+    }
+
+    long snapshotIndex = stateMachine.takeSnapshot();
+    if (snapshotIndex != stateMachine.getLastAppliedTermIndex().getIndex()) {
+      throw new IOException("Index from Snapshot does not match last applied " +
+          "Index");
+    }
+  }
+
+  private static boolean checkIfAllTransactionsApplied(
+      StateMachine stateMachine,
+      RaftServerProxy serverProxy,
+      RaftGroup raftGroup) throws IOException {
+    LOG.info("Checking for pending transactions to be applied.");
+    RaftServerImpl impl = serverProxy.getImpl(raftGroup.getGroupId());
+    long lastCommittedIndex = impl.getState().getLog().getLastCommittedIndex();
+    long appliedIndex = stateMachine.getLastAppliedTermIndex().getIndex();
+    LOG.info("lastCommittedIndex = {}, appliedIndex = {}",
+        lastCommittedIndex, appliedIndex);
+    return (lastCommittedIndex == appliedIndex);
+  }
+
+}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ratis/TestRatisUpgradeUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ratis/TestRatisUpgradeUtils.java
new file mode 100644
index 0000000..078bbb5
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ratis/TestRatisUpgradeUtils.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.ratis;
+
+import static org.apache.hadoop.hdds.ratis.RatisUpgradeUtils.waitForAllTxnsApplied;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.ratis.protocol.RaftGroup;
+import org.apache.ratis.server.impl.RaftServerImpl;
+import org.apache.ratis.server.impl.RaftServerProxy;
+import org.apache.ratis.server.impl.ServerState;
+import org.apache.ratis.server.protocol.TermIndex;
+import org.apache.ratis.server.raftlog.RaftLog;
+import org.apache.ratis.statemachine.StateMachine;
+import org.junit.Test;
+
+/**
+ * Testing util methods in TestRatisUpgradeUtils.
+ */
+public class TestRatisUpgradeUtils {
+
+  @Test
+  public void testWaitForAllTxnsApplied() throws IOException,
+      InterruptedException {
+
+    StateMachine stateMachine = mock(StateMachine.class);
+    RaftGroup raftGroup = RaftGroup.emptyGroup();
+    RaftServerProxy raftServerProxy = mock(RaftServerProxy.class);
+    RaftServerImpl raftServer = mock(RaftServerImpl.class);
+    ServerState serverState = mock(ServerState.class);
+    RaftLog raftLog = mock(RaftLog.class);
+
+    when(raftServerProxy.getImpl(
+        raftGroup.getGroupId())).thenReturn(raftServer);
+    when(raftServer.getState()).thenReturn(serverState);
+    when(serverState.getLog()).thenReturn(raftLog);
+    when(raftLog.getLastCommittedIndex()).thenReturn(1L);
+
+    TermIndex termIndex = mock(TermIndex.class);
+    when(termIndex.getIndex()).thenReturn(0L).thenReturn(0L).thenReturn(1L);
+    when(stateMachine.getLastAppliedTermIndex()).thenReturn(termIndex);
+    when(stateMachine.takeSnapshot()).thenReturn(1L);
+
+    waitForAllTxnsApplied(stateMachine, raftGroup, raftServerProxy, 10, 2);
+    verify(stateMachine.getLastAppliedTermIndex(),
+        times(4)); // 3 checks + 1 after snapshot
+  }
+
+  @Test
+  public void testWaitForAllTxnsAppliedTimeOut() throws Exception {
+
+    StateMachine stateMachine = mock(StateMachine.class);
+    RaftGroup raftGroup = RaftGroup.emptyGroup();
+    RaftServerProxy raftServerProxy = mock(RaftServerProxy.class);
+    RaftServerImpl raftServer = mock(RaftServerImpl.class);
+    ServerState serverState = mock(ServerState.class);
+    RaftLog raftLog = mock(RaftLog.class);
+
+    when(raftServerProxy.getImpl(
+        raftGroup.getGroupId())).thenReturn(raftServer);
+    when(raftServer.getState()).thenReturn(serverState);
+    when(serverState.getLog()).thenReturn(raftLog);
+    when(raftLog.getLastCommittedIndex()).thenReturn(1L);
+
+    TermIndex termIndex = mock(TermIndex.class);
+    when(termIndex.getIndex()).thenReturn(0L);
+    when(stateMachine.getLastAppliedTermIndex()).thenReturn(termIndex);
+    when(stateMachine.takeSnapshot()).thenReturn(1L);
+
+    LambdaTestUtils.intercept(IOException.class, "State Machine has not " +
+        "applied  all the transactions", () ->
+        waitForAllTxnsApplied(stateMachine, raftGroup, raftServerProxy,
+            10, 2));
+  }
+
+}
\ No newline at end of file
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index f16679a..fba6dcc 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -246,4 +246,8 @@ public final class OMConfigKeys {
       "ozone.om.enable.filesystem.paths";
   public static final boolean OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT =
       false;
+
+  public static final long OZONE_OM_MAX_TIME_TO_WAIT_FLUSH_TXNS =
+      TimeUnit.MINUTES.toSeconds(5);
+  public static final long OZONE_OM_FLUSH_TXNS_RETRY_INTERVAL_SECONDS = 5L;
 }
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStarterInterface.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStarterInterface.java
index f632ad1..14252a7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStarterInterface.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStarterInterface.java
@@ -30,4 +30,6 @@ public interface OMStarterInterface {
       AuthenticationException;
   boolean init(OzoneConfiguration conf) throws IOException,
       AuthenticationException;
+  boolean prepareForUpgrade(OzoneConfiguration conf) throws IOException,
+      AuthenticationException;
 }
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a0e0067..3a8a7f3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -192,6 +192,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT;
 import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
 import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
+import static org.apache.hadoop.hdds.ratis.RatisUpgradeUtils.waitForAllTxnsApplied;
 import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString;
 import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName;
 import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
@@ -214,10 +215,12 @@ import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FLUSH_TXNS_RETRY_INTERVAL_SECONDS;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MAX_TIME_TO_WAIT_FLUSH_TXNS;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME;
@@ -233,6 +236,7 @@ import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.
 
 import org.apache.hadoop.util.Time;
 import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
+import org.apache.ratis.server.impl.RaftServerProxy;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.ratis.util.ExitUtils;
 import org.apache.ratis.util.FileUtils;
@@ -326,20 +330,22 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   private final boolean useRatisForReplication;
 
   private boolean isNativeAuthorizerEnabled;
+  private boolean prepareForUpgrade;
 
   private ExitManager exitManager;
 
   private enum State {
     INITIALIZED,
     RUNNING,
+    PREPARING_FOR_UPGRADE,
     STOPPED
   }
 
   // Used in MiniOzoneCluster testing
   private State omState;
 
-  private OzoneManager(OzoneConfiguration conf) throws IOException,
-      AuthenticationException {
+  private OzoneManager(OzoneConfiguration conf, boolean forUpgrade)
+      throws IOException, AuthenticationException {
     super(OzoneVersionInfo.OZONE_VERSION_INFO);
     Preconditions.checkNotNull(conf);
     configuration = conf;
@@ -485,6 +491,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     };
     ShutdownHookManager.get().addShutdownHook(shutdownHook,
         SHUTDOWN_HOOK_PRIORITY);
+    this.prepareForUpgrade = forUpgrade;
     omState = State.INITIALIZED;
   }
 
@@ -918,7 +925,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
    */
   public static OzoneManager createOm(OzoneConfiguration conf)
       throws IOException, AuthenticationException {
-    return new OzoneManager(conf);
+    return new OzoneManager(conf, false);
+  }
+
+  public static OzoneManager createOmUpgradeMode(OzoneConfiguration conf)
+      throws IOException, AuthenticationException {
+    return new OzoneManager(conf, true);
   }
 
   /**
@@ -994,6 +1006,39 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     }
   }
 
+  public boolean applyAllPendingTransactions()
+      throws InterruptedException, IOException {
+
+    if (!isRatisEnabled) {
+      LOG.info("Ratis not enabled. Nothing to do.");
+      return true;
+    }
+
+    waitForAllTxnsApplied(omRatisServer.getOmStateMachine(),
+        omRatisServer.getRaftGroup(),
+        (RaftServerProxy) omRatisServer.getServer(),
+        OZONE_OM_MAX_TIME_TO_WAIT_FLUSH_TXNS,
+        OZONE_OM_FLUSH_TXNS_RETRY_INTERVAL_SECONDS);
+
+    long appliedIndexFromRatis =
+        omRatisServer.getOmStateMachine().getLastAppliedTermIndex().getIndex();
+    OMTransactionInfo omTransactionInfo =
+        OMTransactionInfo.readTransactionInfo(metadataManager);
+    long index = omTransactionInfo.getTermIndex().getIndex();
+    if (index != appliedIndexFromRatis) {
+      throw new IllegalStateException(
+          String.format("Cannot prepare OM for Upgrade " +
+          "since transaction info table index %d does not match ratis %s",
+              index, appliedIndexFromRatis));
+    }
+
+    LOG.info("OM has been prepared for upgrade. All transactions " +
+        "upto {} have been flushed to the state machine, " +
+        "and a snapshot has been taken.",
+        omRatisServer.getOmStateMachine().getLastAppliedTermIndex().getIndex());
+    return true;
+  }
+
   /**
    * Initializes secure OzoneManager.
    */
@@ -1179,15 +1224,22 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
       // Allow OM to start as Http Server failure is not fatal.
       LOG.error("OM HttpServer failed to start.", ex);
     }
-    omRpcServer.start();
-    isOmRpcServerRunning = true;
 
+    if (!prepareForUpgrade) {
+      omRpcServer.start();
+      isOmRpcServerRunning = true;
+    }
     registerMXBean();
 
     startJVMPauseMonitor();
     setStartTime();
-    omState = State.RUNNING;
 
+    if (!prepareForUpgrade) {
+      omState = State.RUNNING;
+    } else {
+      omState = State.PREPARING_FOR_UPGRADE;
+      LOG.info("Started OM services in upgrade mode.");
+    }
   }
 
   /**
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
index 6dc4aea..936980b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
@@ -98,6 +98,28 @@ public class OzoneManagerStarter extends GenericCli {
     }
   }
 
+
+  /**
+   * This function implements a sub-command to allow the OM to be
+   * "prepared for upgrade".
+   */
+  @CommandLine.Command(name = "--prepareForUpgrade",
+      aliases = {"--prepareForDowngrade", "--flushTransactions"},
+      customSynopsis = "ozone om [global options] --prepareForUpgrade",
+      hidden = false,
+      description = "Prepare the OM for upgrade/downgrade. (Flush Raft log " +
+          "transactions.)",
+      mixinStandardHelpOptions = true,
+      versionProvider = HddsVersionProvider.class)
+  public void prepareOmForUpgrade() throws Exception {
+    commonInit();
+    boolean result = receiver.prepareForUpgrade(conf);
+    if (!result) {
+      throw new Exception("Prepare OM For Upgrade failed.");
+    }
+    System.exit(0);
+  }
+
   /**
    * This function should be called by each command to ensure the configuration
    * is set and print the startup banner message.
@@ -130,6 +152,22 @@ public class OzoneManagerStarter extends GenericCli {
         AuthenticationException {
       return OzoneManager.omInit(conf);
     }
+
+    public boolean prepareForUpgrade(OzoneConfiguration conf)
+        throws IOException, AuthenticationException {
+      try (OzoneManager om = OzoneManager.createOmUpgradeMode(conf)) {
+        om.start();
+        boolean success = false;
+        try {
+          LOG.info("Preparing OM for upgrade.");
+          success = om.applyAllPendingTransactions();
+        } catch (InterruptedException e) {
+          LOG.error("Error preparing OM for upgrade.", e);
+          Thread.currentThread().interrupt();
+        }
+        return success;
+      }
+    }
   }
 
 }
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
index 68d359e..dcf7984 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
@@ -319,7 +319,7 @@ public final class OzoneManagerDoubleBuffer {
 
           if (LOG.isDebugEnabled()) {
             LOG.debug("Sync Iteration {} flushed transactions in this " +
-                    "iteration{}", flushIterations.get(),
+                    "iteration {}", flushIterations.get(),
                 flushedTransactionsSize);
           }
 
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
index d6d2be6..fc3efea 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
@@ -698,4 +698,8 @@ public final class OzoneManagerRatisServer {
   public TermIndex getLastAppliedTermIndex() {
     return omStateMachine.getLastAppliedTermIndex();
   }
+
+  public RaftServer getServer() {
+    return server;
+  }
 }
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerStarter.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerStarter.java
index 8028169..c137991 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerStarter.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerStarter.java
@@ -133,6 +133,8 @@ public class TestOzoneManagerStarter {
     private boolean initStatus = true;
     private boolean throwOnStart = false;
     private boolean throwOnInit = false;
+    private boolean prepareUpgradeCalled = false;
+    private boolean throwOnPrepareUpgrade = false;
 
     public void start(OzoneConfiguration conf) throws IOException,
         AuthenticationException {
@@ -150,5 +152,14 @@ public class TestOzoneManagerStarter {
       }
       return initStatus;
     }
+
+    public boolean prepareForUpgrade(OzoneConfiguration conf)
+        throws IOException, AuthenticationException {
+      prepareUpgradeCalled = true;
+      if (throwOnPrepareUpgrade) {
+        throw new IOException("Simulated Exception");
+      }
+      return true;
+    }
   }
 }
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org