You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sh...@apache.org on 2021/03/08 03:42:56 UTC
[ozone] branch HDDS-2823 updated: HDDS-4890. SCM Ratis
enable/disable switch (#1981)
This is an automated email from the ASF dual-hosted git repository.
shashikant pushed a commit to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-2823 by this push:
new faeb3d5 HDDS-4890. SCM Ratis enable/disable switch (#1981)
faeb3d5 is described below
commit faeb3d521e34a5fc99703acc27fe7171e87628c9
Author: bshashikant <sh...@apache.org>
AuthorDate: Mon Mar 8 09:12:43 2021 +0530
HDDS-4890. SCM Ratis enable/disable switch (#1981)
---
.../hdds/scm/metadata/DBTransactionBuffer.java | 35 ++++++++++++++++
.../scm/metadata/SCMDBTransactionBufferImpl.java | 49 ++++++++++++++++++++++
.../hdds/scm/block/DeletedBlockLogImplV2.java | 2 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 36 +++++++++-------
.../scm/container/ContainerStateManagerImpl.java | 24 ++++-------
.../scm/container/states/ContainerStateMap.java | 5 +++
.../hadoop/hdds/scm/ha/InterSCMGrpcService.java | 2 +-
...ffer.java => MockSCMHADBTransactionBuffer.java} | 24 ++++++++---
.../hadoop/hdds/scm/ha/MockSCMHAManager.java | 8 +++-
...onBuffer.java => SCMHADBTransactionBuffer.java} | 8 ++--
...ffer.java => SCMHADBTransactionBufferImpl.java} | 19 +++++++--
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 12 ++++--
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 8 ++++
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 45 ++++++++++++++------
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 4 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 7 +++-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 5 ++-
.../scm/pipeline/PipelineStateManagerV2Impl.java | 25 +++++++----
.../hdds/scm/server/SCMClientProtocolServer.java | 22 +++++++---
.../hdds/scm/server/StorageContainerManager.java | 33 ++++++++-------
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 4 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 27 ++++++------
.../hdds/scm/ha/TestSequenceIDGenerator.java | 2 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 17 ++++----
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 3 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +--
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 13 +++++-
.../hadoop/ozone/TestStorageContainerManager.java | 17 +++++---
.../client/rpc/TestDeleteWithSlowFollower.java | 7 +++-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 3 +-
.../scm/ReconStorageContainerManagerFacade.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 4 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 4 +-
34 files changed, 345 insertions(+), 141 deletions(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/DBTransactionBuffer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/DBTransactionBuffer.java
new file mode 100644
index 0000000..85f5b40
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/DBTransactionBuffer.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.metadata;
+
+import org.apache.hadoop.hdds.utils.db.Table;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * DB transaction that abstracts the updates to the underlying datastore.
+ */
+public interface DBTransactionBuffer<KEY, VALUE> extends Closeable {
+
+ void addToBuffer(Table<KEY, VALUE> table, KEY key, VALUE value) throws
+ IOException;
+
+ void removeFromBuffer(Table<KEY, VALUE> table, KEY key) throws
+ IOException;
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBTransactionBufferImpl.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBTransactionBufferImpl.java
new file mode 100644
index 0000000..9937a49
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBTransactionBufferImpl.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.metadata;
+
+import org.apache.hadoop.hdds.utils.db.Table;
+
+import java.io.IOException;
+
+/**
+ * Default implementation for DBTransactionBuffer for SCM without Ratis.
+ */
+public class SCMDBTransactionBufferImpl implements DBTransactionBuffer {
+
+ public SCMDBTransactionBufferImpl() {
+
+ }
+
+ @Override
+ public void addToBuffer(Table table, Object key, Object value)
+ throws IOException {
+ table.put(key, value);
+ }
+
+ @Override
+ public void removeFromBuffer(Table table, Object key)
+ throws IOException {
+ table.delete(key);
+ }
+
+ @Override
+ public void close() {
+ }
+
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImplV2.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImplV2.java
index ab8fb77..d5e5d93 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImplV2.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImplV2.java
@@ -41,10 +41,10 @@ import org.apache.hadoop.hdds.scm.container.ContainerManagerV2;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
-import org.apache.hadoop.hdds.scm.ha.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
+import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.utils.db.Table;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java
index ac278e7..d2c55a0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java
@@ -21,9 +21,10 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.hdds.scm.ha.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler;
+import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
+import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.hdds.utils.db.TypedTable;
@@ -60,8 +61,10 @@ public class DeletedBlockLogStateManagerImpl
OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT);
this.deletedTable = deletedTable;
this.transactionBuffer = txBuffer;
- this.deletingTxIDs = ConcurrentHashMap.newKeySet();
- this.skippingRetryTxIDs = ConcurrentHashMap.newKeySet();
+ final boolean isRatisEnabled = SCMHAUtils.isSCMHAEnabled(conf);
+ this.deletingTxIDs = isRatisEnabled ? ConcurrentHashMap.newKeySet() : null;
+ this.skippingRetryTxIDs =
+ isRatisEnabled ? ConcurrentHashMap.newKeySet() : null;
}
public TableIterator<Long, TypedTable.KeyValue<Long,
@@ -89,8 +92,9 @@ public class DeletedBlockLogStateManagerImpl
throw new IllegalStateException("");
}
- if (!deletingTxIDs.contains(txID) &&
- !skippingRetryTxIDs.contains(txID)) {
+ if ((deletingTxIDs == null || !deletingTxIDs.contains(txID)) && (
+ skippingRetryTxIDs == null || !skippingRetryTxIDs
+ .contains(txID))) {
nextTx = next;
if (LOG.isTraceEnabled()) {
LOG.trace("DeletedBlocksTransaction matching txID:{}",
@@ -160,18 +164,18 @@ public class DeletedBlockLogStateManagerImpl
public void addTransactionsToDB(ArrayList<DeletedBlocksTransaction> txs)
throws IOException {
for (DeletedBlocksTransaction tx : txs) {
- deletedTable.putWithBatch(
- transactionBuffer.getCurrentBatchOperation(), tx.getTxID(), tx);
+ transactionBuffer.addToBuffer(deletedTable, tx.getTxID(), tx);
}
}
@Override
public void removeTransactionsFromDB(ArrayList<Long> txIDs)
throws IOException {
- deletingTxIDs.addAll(txIDs);
+ if (deletingTxIDs != null) {
+ deletingTxIDs.addAll(txIDs);
+ }
for (Long txID : txIDs) {
- deletedTable.deleteWithBatch(
- transactionBuffer.getCurrentBatchOperation(), txID);
+ transactionBuffer.removeFromBuffer(deletedTable, txID);
}
}
@@ -194,14 +198,17 @@ public class DeletedBlockLogStateManagerImpl
// then set the retry value to -1, stop retrying, admins can
// analyze those blocks and purge them manually by SCMCli.
DeletedBlocksTransaction.Builder builder = block.toBuilder().setCount(-1);
- deletedTable.putWithBatch(
- transactionBuffer.getCurrentBatchOperation(), txID, builder.build());
- skippingRetryTxIDs.add(txID);
+ transactionBuffer.addToBuffer(deletedTable, txID, builder.build());
+ if (skippingRetryTxIDs != null) {
+ skippingRetryTxIDs.add(txID);
+ }
}
}
-
public void onFlush() {
+ // onFlush() can be invoked only when ratis is enabled.
+ Preconditions.checkNotNull(deletingTxIDs);
+ Preconditions.checkNotNull(skippingRetryTxIDs);
deletingTxIDs.clear();
skippingRetryTxIDs.clear();
}
@@ -255,7 +262,6 @@ public class DeletedBlockLogStateManagerImpl
public DeletedBlockLogStateManager build() {
Preconditions.checkNotNull(conf);
- Preconditions.checkNotNull(scmRatisServer);
Preconditions.checkNotNull(table);
final DeletedBlockLogStateManager impl =
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
index dedfe8c..2fc71bc 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
@@ -41,10 +41,10 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.states.ContainerState;
import org.apache.hadoop.hdds.scm.container.states.ContainerStateMap;
import org.apache.hadoop.hdds.scm.ha.CheckedConsumer;
-import org.apache.hadoop.hdds.scm.ha.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.ha.ExecutionUtil;
import org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
+import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
@@ -303,16 +303,12 @@ public final class ContainerStateManagerImpl
try {
if (!containers.contains(containerID)) {
ExecutionUtil.create(() -> {
- containerStore.putWithBatch(
- transactionBuffer.getCurrentBatchOperation(),
- containerID, container);
+ transactionBuffer.addToBuffer(containerStore, containerID, container);
containers.addContainer(container);
pipelineManager.addContainerToPipeline(pipelineID, containerID);
}).onException(() -> {
containers.removeContainer(containerID);
- containerStore.deleteWithBatch(
- transactionBuffer.getCurrentBatchOperation(),
- containerID);
+ transactionBuffer.removeFromBuffer(containerStore, containerID);
}).execute();
}
} finally {
@@ -348,13 +344,10 @@ public final class ContainerStateManagerImpl
if (newState.getNumber() > oldState.getNumber()) {
ExecutionUtil.create(() -> {
containers.updateState(id, oldState, newState);
- containerStore.putWithBatch(
- transactionBuffer.getCurrentBatchOperation(),
- id, containers.getContainerInfo(id));
+ transactionBuffer.addToBuffer(containerStore, id,
+ containers.getContainerInfo(id));
}).onException(() -> {
- containerStore.putWithBatch(
- transactionBuffer.getCurrentBatchOperation(),
- id, oldInfo);
+ transactionBuffer.addToBuffer(containerStore, id, oldInfo);
containers.updateState(id, newState, oldState);
}).execute();
containerStateChangeActions.getOrDefault(event, info -> {
@@ -488,9 +481,7 @@ public final class ContainerStateManagerImpl
final ContainerID cid = ContainerID.getFromProtobuf(id);
final ContainerInfo containerInfo = containers.getContainerInfo(cid);
ExecutionUtil.create(() -> {
- containerStore.deleteWithBatch(
- transactionBuffer.getCurrentBatchOperation(),
- cid);
+ transactionBuffer.removeFromBuffer(containerStore, cid);
containers.removeContainer(cid);
}).onException(() -> containerStore.put(cid, containerInfo)).execute();
} finally {
@@ -564,7 +555,6 @@ public final class ContainerStateManagerImpl
public ContainerStateManagerV2 build() throws IOException {
Preconditions.checkNotNull(conf);
Preconditions.checkNotNull(pipelineMgr);
- Preconditions.checkNotNull(scmRatisServer);
Preconditions.checkNotNull(table);
final ContainerStateManagerV2 csm = new ContainerStateManagerImpl(
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java
index ff2ca66..a76c851 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java
@@ -120,6 +120,11 @@ public class ContainerStateMap {
factorMap.insert(info.getReplicationFactor(), id);
typeMap.insert(info.getReplicationType(), id);
replicaMap.put(id, ConcurrentHashMap.newKeySet());
+
+ // Flush the cache of this container type, will be added later when
+ // get container queries are executed.
+ flushCache(info);
+ LOG.trace("Container {} added to ContainerStateMap.", id);
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcService.java
index 61d132e..e962ded 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcService.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcService.java
@@ -61,7 +61,7 @@ public class InterSCMGrpcService extends
public void download(CopyDBCheckpointRequestProto request,
StreamObserver<CopyDBCheckpointResponseProto> responseObserver) {
try {
- scm.getScmHAManager().getDBTransactionBuffer().flush();
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
Table<String, TransactionInfo> transactionInfoTable =
Arrays.stream(new SCMDBDefinition().getColumnFamilies())
.filter(t -> t.getValueType() == TransactionInfo.class)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockDBTransactionBuffer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockSCMHADBTransactionBuffer.java
similarity index 76%
rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockDBTransactionBuffer.java
rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockSCMHADBTransactionBuffer.java
index e5a67e2..b4ce677 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockDBTransactionBuffer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockSCMHADBTransactionBuffer.java
@@ -20,23 +20,23 @@ import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.RDBBatchOperation;
+import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.ratis.statemachine.SnapshotInfo;
import java.io.IOException;
-public class MockDBTransactionBuffer implements DBTransactionBuffer {
+public class MockSCMHADBTransactionBuffer implements SCMHADBTransactionBuffer {
private DBStore dbStore;
private BatchOperation currentBatchOperation;
- public MockDBTransactionBuffer() {
+ public MockSCMHADBTransactionBuffer() {
}
- public MockDBTransactionBuffer(DBStore store) {
+ public MockSCMHADBTransactionBuffer(DBStore store) {
this.dbStore = store;
}
- @Override
- public BatchOperation getCurrentBatchOperation() {
+ private BatchOperation getCurrentBatchOperation() {
if (currentBatchOperation == null) {
if (dbStore != null) {
currentBatchOperation = dbStore.initBatchOperation();
@@ -48,6 +48,18 @@ public class MockDBTransactionBuffer implements DBTransactionBuffer {
}
@Override
+ public void addToBuffer(Table table, Object key, Object value)
+ throws IOException {
+ table.putWithBatch(getCurrentBatchOperation(), key, value);
+ }
+
+ @Override
+ public void removeFromBuffer(Table table, Object key)
+ throws IOException {
+ table.deleteWithBatch(getCurrentBatchOperation(), key);
+ }
+
+ @Override
public void updateLatestTrxInfo(TransactionInfo info) {
}
@@ -70,7 +82,7 @@ public class MockDBTransactionBuffer implements DBTransactionBuffer {
@Override
public void flush() throws IOException {
if (dbStore != null) {
- dbStore.commitBatchOperation(currentBatchOperation);
+ dbStore.commitBatchOperation(getCurrentBatchOperation());
currentBatchOperation.close();
currentBatchOperation = null;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockSCMHAManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockSCMHAManager.java
index 4926e71..91c4307 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockSCMHAManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockSCMHAManager.java
@@ -27,6 +27,7 @@ import java.util.Map;
import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
import org.apache.hadoop.hdds.scm.AddSCMRequest;
+import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.ratis.protocol.ClientId;
import org.apache.ratis.protocol.Message;
import org.apache.ratis.protocol.RaftClientReply;
@@ -61,7 +62,7 @@ public final class MockSCMHAManager implements SCMHAManager {
* Creates MockSCMHAManager instance.
*/
private MockSCMHAManager(boolean isLeader) {
- this(isLeader, new MockDBTransactionBuffer());
+ this(isLeader, new MockSCMHADBTransactionBuffer());
}
private MockSCMHAManager(boolean isLeader, DBTransactionBuffer buffer) {
@@ -100,6 +101,11 @@ public final class MockSCMHAManager implements SCMHAManager {
}
@Override
+ public SCMHADBTransactionBuffer asSCMHADBTransactionBuffer() {
+ return null;
+ }
+
+ @Override
public SCMSnapshotProvider getSCMSnapshotProvider() {
return null;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/DBTransactionBuffer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHADBTransactionBuffer.java
similarity index 88%
rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/DBTransactionBuffer.java
rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHADBTransactionBuffer.java
index 8ec70a0..f3f9f97 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/DBTransactionBuffer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHADBTransactionBuffer.java
@@ -16,11 +16,10 @@
*/
package org.apache.hadoop.hdds.scm.ha;
+import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.utils.TransactionInfo;
-import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.ratis.statemachine.SnapshotInfo;
-import java.io.Closeable;
import java.io.IOException;
/**
@@ -28,9 +27,8 @@ import java.io.IOException;
* to flush a batch into SCM DB. This buffer also maintains a latest transaction
* info to indicate the information of the latest transaction in the buffer.
*/
-public interface DBTransactionBuffer extends Closeable {
-
- BatchOperation getCurrentBatchOperation();
+public interface SCMHADBTransactionBuffer
+ extends DBTransactionBuffer {
void updateLatestTrxInfo(TransactionInfo info);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMDBTransactionBuffer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHADBTransactionBufferImpl.java
similarity index 89%
rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMDBTransactionBuffer.java
rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHADBTransactionBufferImpl.java
index 748b5e7..4ee2ae3 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMDBTransactionBuffer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHADBTransactionBufferImpl.java
@@ -36,25 +36,36 @@ import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY;
* written into DB to indicate the term and transaction index for the latest
* operation in DB.
*/
-public class SCMDBTransactionBuffer implements DBTransactionBuffer {
+public class SCMHADBTransactionBufferImpl implements SCMHADBTransactionBuffer {
private final StorageContainerManager scm;
private SCMMetadataStore metadataStore;
private BatchOperation currentBatchOperation;
private TransactionInfo latestTrxInfo;
private SnapshotInfo latestSnapshot;
- public SCMDBTransactionBuffer(StorageContainerManager scm)
+ public SCMHADBTransactionBufferImpl(StorageContainerManager scm)
throws IOException {
this.scm = scm;
init();
}
- @Override
- public BatchOperation getCurrentBatchOperation() {
+ private BatchOperation getCurrentBatchOperation() {
return currentBatchOperation;
}
@Override
+ public void addToBuffer(Table table, Object key, Object value)
+ throws IOException {
+ table.putWithBatch(getCurrentBatchOperation(), key, value);
+ }
+
+ @Override
+ public void removeFromBuffer(Table table, Object key)
+ throws IOException {
+ table.deleteWithBatch(getCurrentBatchOperation(), key);
+ }
+
+ @Override
public void updateLatestTrxInfo(TransactionInfo info) {
if (info.compareTo(this.latestTrxInfo) <= 0) {
throw new IllegalArgumentException(
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
index c3259bc..6dc0f15 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
@@ -21,6 +21,7 @@ import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
import org.apache.hadoop.hdds.scm.metadata.Replicate;
import org.apache.hadoop.util.Time;
@@ -50,7 +51,9 @@ public class SCMHAInvocationHandler implements InvocationHandler {
this.requestType = requestType;
this.localHandler = localHandler;
this.ratisHandler = ratisHandler;
- ratisHandler.registerStateMachineHandler(requestType, localHandler);
+ if (ratisHandler != null) {
+ ratisHandler.registerStateMachineHandler(requestType, localHandler);
+ }
}
@Override
@@ -58,8 +61,10 @@ public class SCMHAInvocationHandler implements InvocationHandler {
final Object[] args) throws Throwable {
try {
long startTime = Time.monotonicNow();
- final Object result = method.isAnnotationPresent(Replicate.class) ?
- invokeRatis(method, args) : invokeLocal(method, args);
+ final Object result =
+ ratisHandler != null && method.isAnnotationPresent(Replicate.class) ?
+ invokeRatis(method, args) :
+ invokeLocal(method, args);
LOG.debug("Call: {} took {} ms", method, Time.monotonicNow() - startTime);
return result;
} catch(InvocationTargetException iEx) {
@@ -83,6 +88,7 @@ public class SCMHAInvocationHandler implements InvocationHandler {
private Object invokeRatis(Method method, Object[] args)
throws Exception {
long startTime = Time.monotonicNowNanos();
+ Preconditions.checkNotNull(ratisHandler);
final SCMRatisResponse response = ratisHandler.submitRequest(
SCMRatisRequest.of(requestType, method.getName(), args));
LOG.info("Invoking method {} on target {}, cost {}us",
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java
index 4745eb1..c6dfd9c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.ha;
import org.apache.hadoop.hdds.scm.AddSCMRequest;
+import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.ratis.server.protocol.TermIndex;
import java.io.IOException;
@@ -48,6 +49,13 @@ public interface SCMHAManager {
DBTransactionBuffer getDBTransactionBuffer();
/**
+ * Returns the DBTransactionBuffer as SCMHADBTransactionBuffer if its
+ * valid.
+ * @return
+ */
+ SCMHADBTransactionBuffer asSCMHADBTransactionBuffer();
+
+ /**
* Stops the HA service.
*/
void shutdown() throws IOException;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
index c46560d..3df5f7b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
@@ -21,6 +21,8 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.AddSCMRequest;
+import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBTransactionBufferImpl;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.utils.HAUtils;
@@ -56,7 +58,7 @@ public class SCMHAManagerImpl implements SCMHAManager {
private final SCMRatisServer ratisServer;
private final ConfigurationSource conf;
- private final SCMDBTransactionBuffer transactionBuffer;
+ private final DBTransactionBuffer transactionBuffer;
private final SCMSnapshotProvider scmSnapshotProvider;
private final StorageContainerManager scm;
private ExitManager exitManager;
@@ -71,12 +73,19 @@ public class SCMHAManagerImpl implements SCMHAManager {
final StorageContainerManager scm) throws IOException {
this.conf = conf;
this.scm = scm;
- this.transactionBuffer =
- new SCMDBTransactionBuffer(scm);
- this.ratisServer = new SCMRatisServerImpl(conf, scm, transactionBuffer);
- this.scmSnapshotProvider = new SCMSnapshotProvider(conf,
- scm.getSCMHANodeDetails().getPeerNodeDetails());
- grpcServer = new InterSCMGrpcProtocolService(conf, scm);
+ if (SCMHAUtils.isSCMHAEnabled(conf)) {
+ this.transactionBuffer = new SCMHADBTransactionBufferImpl(scm);
+ this.ratisServer = new SCMRatisServerImpl(conf, scm,
+ (SCMHADBTransactionBuffer) transactionBuffer);
+ this.scmSnapshotProvider = new SCMSnapshotProvider(conf,
+ scm.getSCMHANodeDetails().getPeerNodeDetails());
+ grpcServer = new InterSCMGrpcProtocolService(conf, scm);
+ } else {
+ this.transactionBuffer = new SCMDBTransactionBufferImpl();
+ this.scmSnapshotProvider = null;
+ this.grpcServer = null;
+ this.ratisServer = null;
+ }
}
@@ -85,14 +94,17 @@ public class SCMHAManagerImpl implements SCMHAManager {
*/
@Override
public void start() throws IOException {
+ if (ratisServer == null) {
+ return;
+ }
ratisServer.start();
if (ratisServer.getDivision().getGroup().getPeers().isEmpty()) {
// this is a bootstrapped node
// It will first try to add itself to existing ring
boolean success = HAUtils.addSCM(OzoneConfiguration.of(conf),
new AddSCMRequest.Builder().setClusterId(scm.getClusterId())
- .setScmId(scm.getScmId()).setRatisAddr(
- scm.getSCMHANodeDetails().getLocalNodeDetails()
+ .setScmId(scm.getScmId())
+ .setRatisAddr(scm.getSCMHANodeDetails().getLocalNodeDetails()
// TODO : Should we use IP instead of hostname??
.getRatisHostPortStr()).build(), scm.getSCMNodeId());
if (!success) {
@@ -120,6 +132,13 @@ public class SCMHAManagerImpl implements SCMHAManager {
return scmSnapshotProvider;
}
+ @Override
+ public SCMHADBTransactionBuffer asSCMHADBTransactionBuffer() {
+ Preconditions
+ .checkArgument(transactionBuffer instanceof SCMHADBTransactionBuffer);
+ return (SCMHADBTransactionBuffer)transactionBuffer;
+
+ }
/**
* Download and install latest checkpoint from leader SCM.
*
@@ -287,9 +306,11 @@ public class SCMHAManagerImpl implements SCMHAManager {
*/
@Override
public void shutdown() throws IOException {
- ratisServer.stop();
- ratisServer.getSCMStateMachine().stop();
- grpcServer.stop();
+ if (ratisServer != null) {
+ ratisServer.stop();
+ ratisServer.getSCMStateMachine().stop();
+ grpcServer.stop();
+ }
}
@Override
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
index 0d61bca..3e6b57c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
@@ -71,7 +71,7 @@ public class SCMRatisServerImpl implements SCMRatisServer {
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
SCMRatisServerImpl(final ConfigurationSource conf,
- final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ final StorageContainerManager scm, final SCMHADBTransactionBuffer buffer)
throws IOException {
this.scm = scm;
this.stateMachine = new SCMStateMachine(scm, this, buffer);
@@ -176,7 +176,7 @@ public class SCMRatisServerImpl implements SCMRatisServer {
final RaftClientReply raftClientReply =
server.submitClientRequestAsync(raftClientRequest).get();
if (LOG.isDebugEnabled()) {
- LOG.debug("request {} Reply {}", raftClientRequest, raftClientReply);
+ LOG.info("request {} Reply {}", raftClientRequest, raftClientReply);
}
return SCMRatisResponse.decode(raftClientReply);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
index a86668a..65569b9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
@@ -44,11 +44,13 @@ import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.statemachine.SnapshotInfo;
+import org.apache.ratis.statemachine.StateMachine;
import org.apache.ratis.statemachine.TransactionContext;
import org.apache.ratis.statemachine.impl.BaseStateMachine;
import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
+import org.apache.ratis.util.ExitUtils;
import org.apache.ratis.util.LifeCycle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,14 +67,14 @@ public class SCMStateMachine extends BaseStateMachine {
private StorageContainerManager scm;
private SCMRatisServer ratisServer;
private Map<RequestType, Object> handlers;
- private DBTransactionBuffer transactionBuffer;
+ private SCMHADBTransactionBuffer transactionBuffer;
private final SimpleStateMachineStorage storage =
new SimpleStateMachineStorage();
private final AtomicBoolean isInitialized;
private ExecutorService installSnapshotExecutor;
public SCMStateMachine(final StorageContainerManager scm,
- final SCMRatisServer ratisServer, DBTransactionBuffer buffer)
+ final SCMRatisServer ratisServer, SCMHADBTransactionBuffer buffer)
throws SCMException {
this.scm = scm;
this.ratisServer = ratisServer;
@@ -124,6 +126,7 @@ public class SCMStateMachine extends BaseStateMachine {
.build());
} catch (Exception ex) {
applyTransactionFuture.completeExceptionally(ex);
+ ExitUtils.terminate(1, ex.getMessage(), ex, StateMachine.LOG);
}
return applyTransactionFuture;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java
index 5f3dbb5..9fbb35a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.ha;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.metadata.Replicate;
import org.apache.hadoop.hdds.utils.db.Table;
import org.slf4j.Logger;
@@ -202,8 +203,8 @@ public class SequenceIdGenerator {
}
try {
- sequenceIdTable.putWithBatch(transactionBuffer
- .getCurrentBatchOperation(), sequenceIdName, newLastId);
+ transactionBuffer
+ .addToBuffer(sequenceIdTable, sequenceIdName, newLastId);
} catch (IOException ioe) {
throw new RuntimeException("Failed to put lastId to Batch", ioe);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManagerV2Impl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManagerV2Impl.java
index 2130c4d..c84a852 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManagerV2Impl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManagerV2Impl.java
@@ -22,9 +22,9 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.ha.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
+import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
@@ -93,8 +93,8 @@ public class PipelineStateManagerV2Impl implements StateManager {
try {
Pipeline pipeline = Pipeline.getFromProtobuf(pipelineProto);
if (pipelineStore != null) {
- pipelineStore.putWithBatch(transactionBuffer.getCurrentBatchOperation(),
- pipeline.getId(), pipeline);
+ transactionBuffer
+ .addToBuffer(pipelineStore, pipeline.getId(), pipeline);
pipelineStateMap.addPipeline(pipeline);
nodeManager.addPipeline(pipeline);
LOG.info("Created pipeline {}.", pipeline);
@@ -223,8 +223,7 @@ public class PipelineStateManagerV2Impl implements StateManager {
try {
PipelineID pipelineID = PipelineID.getFromProtobuf(pipelineIDProto);
if (pipelineStore != null) {
- pipelineStore.deleteWithBatch(
- transactionBuffer.getCurrentBatchOperation(), pipelineID);
+ transactionBuffer.removeFromBuffer(pipelineStore, pipelineID);
}
Pipeline pipeline = pipelineStateMap.removePipeline(pipelineID);
nodeManager.removePipeline(pipeline);
@@ -240,7 +239,19 @@ public class PipelineStateManagerV2Impl implements StateManager {
PipelineID pipelineID, ContainerID containerID) throws IOException {
lock.writeLock().lock();
try {
+ // Typica;;y, SCM can send a pipeline close Action to datanode and receive
+ // pipelineCloseAction to close the pipeline which will remove the
+ // pipelineId both from the piplineStateMap as well as
+ // pipeline2containerMap Subsequently, close container handler event can
+ // also try to close the container as a part of which , it will also
+ // try to remove the container from the pipeline2container Map which will
+ // fail with PipelineNotFoundException. These are executed over ratis, and
+ // if the exception is propagated to SCMStateMachine., it will bring down
+ // the SCM. Ignoring it here.
pipelineStateMap.removeContainerFromPipeline(pipelineID, containerID);
+ } catch (PipelineNotFoundException pnfe) {
+ LOG.info("Pipeline {} is not found in the pipeline2ContainerMap. Pipeline"
+ + " may have been closed already.", pipelineID);
} finally {
lock.writeLock().unlock();
}
@@ -261,8 +272,8 @@ public class PipelineStateManagerV2Impl implements StateManager {
if (pipelineStore != null) {
pipelineStateMap.updatePipelineState(pipelineID,
Pipeline.PipelineState.fromProtobuf(newState));
- pipelineStore.putWithBatch(transactionBuffer.getCurrentBatchOperation(),
- pipelineID, getPipeline(pipelineID));
+ transactionBuffer
+ .addToBuffer(pipelineStore, pipelineID, getPipeline(pipelineID));
}
} catch (IOException ex) {
LOG.warn("Pipeline {} state update failed", pipelineID);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 7d789f6..5d3614e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -68,12 +68,13 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+import java.util.Collections;
+import java.util.ArrayList;
import java.util.TreeSet;
+import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StorageContainerLocationProtocolService.newReflectiveBlockingService;
@@ -532,9 +533,18 @@ public class SCMClientProtocolServer implements
ScmInfo.Builder builder =
new ScmInfo.Builder()
.setClusterId(scm.getScmStorageConfig().getClusterID())
- .setScmId(scm.getScmStorageConfig().getScmId())
- .setRatisPeerRoles(
- scm.getScmHAManager().getRatisServer().getRatisRoles());
+ .setScmId(scm.getScmStorageConfig().getScmId());
+ if (scm.getScmHAManager().getRatisServer() != null) {
+ builder.setRatisPeerRoles(
+ scm.getScmHAManager().getRatisServer().getRatisRoles());
+ } else {
+ // In case, there is no ratis, there is no ratis role.
+ // This will just print the hostname with ratis port as the default
+ // behaviour.
+ String adddress = scm.getSCMHANodeDetails().getLocalNodeDetails()
+ .getRatisHostPortStr();
+ builder.setRatisPeerRoles(Arrays.asList(adddress));
+ }
return builder.build();
} catch (Exception ex) {
auditSuccess = false;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index f66f601..f289698 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -460,10 +460,13 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
if (configurator.getScmContext() != null) {
scmContext = configurator.getScmContext();
} else {
+ // When term equals SCMContext.INVALID_TERM, the isLeader() check
+ // and getTermOfLeader() will always pass.
+ long term = SCMHAUtils.isSCMHAEnabled(conf) ? 0 : SCMContext.INVALID_TERM;
// non-leader of term 0, in safe mode, preCheck not completed.
scmContext = new SCMContext.Builder()
.setLeader(false)
- .setTerm(0)
+ .setTerm(term)
.setIsInSafeMode(true)
.setIsPreCheckComplete(false)
.setSCM(this)
@@ -794,17 +797,11 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmStorageConfig.setClusterId(clusterId);
}
scmStorageConfig.initialize();
- // TODO: Removing the HA enabled check right now as
- // when the SCM starts up , it always spins up the ratis
- // server irrespective of the check. If the ratis server is not
- // initialized here and starts up during the regular start,
- // it won't be starting a leader election and hence won't work. The
- // check will be re-introduced one we have clear segregation path with
- // ratis enable/disable switch.
- // if (SCMHAUtils.isSCMHAEnabled(conf)) {
- SCMRatisServerImpl.initialize(scmStorageConfig.getClusterID(),
- scmStorageConfig.getScmId(), haDetails.getLocalNodeDetails(), conf);
- // }
+ if (SCMHAUtils.isSCMHAEnabled(conf)) {
+ SCMRatisServerImpl.initialize(scmStorageConfig.getClusterID(),
+ scmStorageConfig.getScmId(), haDetails.getLocalNodeDetails(),
+ conf);
+ }
LOG.info("SCM initialization succeeded. Current cluster id for sd={}"
+ "; cid={}; layoutVersion={}; scmId={}",
scmStorageConfig.getStorageDir(), scmStorageConfig.getClusterID(),
@@ -1220,8 +1217,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
* @return - if the current scm is the leader and is ready.
*/
public boolean checkLeader() {
- return scmContext.isLeader() && getScmHAManager().getRatisServer()
- .getDivision().getInfo().isLeaderReady();
+ // For NON-HA setup, the node will always be the leader
+ if (!SCMHAUtils.isSCMHAEnabled(configuration)) {
+ Preconditions.checkArgument(scmContext.isLeader());
+ return true;
+ } else {
+ // FOR HA setup, the node has to be the leader and ready to serve
+ // requests.
+ return scmContext.isLeader() && getScmHAManager().getRatisServer()
+ .getDivision().getInfo().isLeaderReady();
+ }
}
public void checkAdminAccess(String remoteUser) throws IOException {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
index 5bfa11a..6aa4cfe 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
@@ -476,7 +476,9 @@ public final class TestUtils {
public static StorageContainerManager getScmSimple(OzoneConfiguration conf)
throws IOException, AuthenticationException {
SCMConfigurator configurator = new SCMConfigurator();
- conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+ // The default behaviour whether ratis will be enabled or not
+ // in SCM will be inferred from ozone-default.xml.
+ // conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
return StorageContainerManager.createSCM(conf, configurator);
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
index 5b9348a..edf8e0c 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
@@ -22,13 +22,14 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerManagerV2;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.ha.DBTransactionBuffer;
-import org.apache.hadoop.hdds.scm.ha.MockDBTransactionBuffer;
+import org.apache.hadoop.hdds.scm.ha.SCMHADBTransactionBuffer;
+import org.apache.hadoop.hdds.scm.ha.MockSCMHADBTransactionBuffer;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -83,24 +84,25 @@ public class TestDeletedBlockLog {
private ContainerManagerV2 containerManager;
private StorageContainerManager scm;
private List<DatanodeDetails> dnList;
- private DBTransactionBuffer dbTransactionBuffer;
+ private SCMHADBTransactionBuffer scmHADBTransactionBuffer;
@Before
public void setup() throws Exception {
testDir = GenericTestUtils.getTestDir(
TestDeletedBlockLog.class.getSimpleName());
conf = new OzoneConfiguration();
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
conf.setInt(OZONE_SCM_BLOCK_DELETION_MAX_RETRY, 20);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
scm = TestUtils.getScm(conf);
containerManager = Mockito.mock(ContainerManagerV2.class);
- dbTransactionBuffer =
- new MockDBTransactionBuffer(scm.getScmMetadataStore().getStore());
+ scmHADBTransactionBuffer =
+ new MockSCMHADBTransactionBuffer(scm.getScmMetadataStore().getStore());
deletedBlockLog = new DeletedBlockLogImplV2(conf,
containerManager,
scm.getScmHAManager().getRatisServer(),
scm.getScmMetadataStore().getDeletedBlocksTXTable(),
- dbTransactionBuffer,
+ scmHADBTransactionBuffer,
scm.getScmContext(),
scm.getSequenceIdGen());
dnList = new ArrayList<>(3);
@@ -164,26 +166,23 @@ public class TestDeletedBlockLog {
private void addTransactions(Map<Long, List<Long>> containerBlocksMap)
throws IOException {
- dbTransactionBuffer.getCurrentBatchOperation();
deletedBlockLog.addTransactions(containerBlocksMap);
- dbTransactionBuffer.flush();
+ scmHADBTransactionBuffer.flush();
}
private void incrementCount(List<Long> txIDs) throws IOException {
- dbTransactionBuffer.getCurrentBatchOperation();
deletedBlockLog.incrementCount(txIDs);
- dbTransactionBuffer.flush();
+ scmHADBTransactionBuffer.flush();
}
private void commitTransactions(
List<DeleteBlockTransactionResult> transactionResults,
DatanodeDetails... dns) throws IOException {
- dbTransactionBuffer.getCurrentBatchOperation();
for (DatanodeDetails dnDetails : dns) {
deletedBlockLog
.commitTransactions(transactionResults, dnDetails.getUuid());
}
- dbTransactionBuffer.flush();
+ scmHADBTransactionBuffer.flush();
}
private void commitTransactions(
@@ -333,7 +332,7 @@ public class TestDeletedBlockLog {
containerManager,
scm.getScmHAManager().getRatisServer(),
scm.getScmMetadataStore().getDeletedBlocksTXTable(),
- dbTransactionBuffer,
+ scmHADBTransactionBuffer,
scm.getScmContext(),
scm.getSequenceIdGen());
List<DeletedBlocksTransaction> blocks =
@@ -351,7 +350,7 @@ public class TestDeletedBlockLog {
containerManager,
scm.getScmHAManager().getRatisServer(),
scm.getScmMetadataStore().getDeletedBlocksTXTable(),
- dbTransactionBuffer,
+ scmHADBTransactionBuffer,
scm.getScmContext(),
scm.getSequenceIdGen());
blocks = getTransactions(BLOCKS_PER_TXN * 40);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIDGenerator.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIDGenerator.java
index e1fc850..9264669 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIDGenerator.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIDGenerator.java
@@ -30,7 +30,7 @@ public class TestSequenceIDGenerator {
@Test
public void testSequenceIDGenUponNonRatis() throws Exception {
OzoneConfiguration conf = SCMTestUtils.getConf();
-
+ conf.setBoolean(OZONE_SCM_HA_ENABLE_KEY, false);
SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(conf);
scmMetadataStore.start(conf);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
index fc74800..06dbc0b 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
@@ -27,8 +27,8 @@ import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.ha.DBTransactionBuffer;
-import org.apache.hadoop.hdds.scm.ha.MockDBTransactionBuffer;
+import org.apache.hadoop.hdds.scm.ha.SCMHADBTransactionBuffer;
+import org.apache.hadoop.hdds.scm.ha.MockSCMHADBTransactionBuffer;
import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMServiceManager;
@@ -113,7 +113,7 @@ public class TestPipelineManagerImpl {
}
private PipelineManagerV2Impl createPipelineManager(
- boolean isLeader, DBTransactionBuffer buffer) throws IOException {
+ boolean isLeader, SCMHADBTransactionBuffer buffer) throws IOException {
return PipelineManagerV2Impl.newPipelineManager(conf,
MockSCMHAManager.getInstance(isLeader, buffer),
new MockNodeManager(true, 20),
@@ -125,7 +125,8 @@ public class TestPipelineManagerImpl {
@Test
public void testCreatePipeline() throws Exception {
- DBTransactionBuffer buffer1 = new MockDBTransactionBuffer(dbStore);
+ SCMHADBTransactionBuffer buffer1 =
+ new MockSCMHADBTransactionBuffer(dbStore);
PipelineManagerV2Impl pipelineManager =
createPipelineManager(true, buffer1);
Assert.assertTrue(pipelineManager.getPipelines().isEmpty());
@@ -141,7 +142,8 @@ public class TestPipelineManagerImpl {
buffer1.close();
pipelineManager.close();
- DBTransactionBuffer buffer2 = new MockDBTransactionBuffer(dbStore);
+ SCMHADBTransactionBuffer buffer2 =
+ new MockSCMHADBTransactionBuffer(dbStore);
PipelineManagerV2Impl pipelineManager2 =
createPipelineManager(true, buffer2);
// Should be able to load previous pipelines.
@@ -173,7 +175,7 @@ public class TestPipelineManagerImpl {
@Test
public void testUpdatePipelineStates() throws Exception {
- DBTransactionBuffer buffer = new MockDBTransactionBuffer(dbStore);
+ SCMHADBTransactionBuffer buffer = new MockSCMHADBTransactionBuffer(dbStore);
PipelineManagerV2Impl pipelineManager =
createPipelineManager(true, buffer);
Table<PipelineID, Pipeline> pipelineStore =
@@ -450,7 +452,8 @@ public class TestPipelineManagerImpl {
@Test
public void testPipelineOpenOnlyWhenLeaderReported() throws Exception {
- DBTransactionBuffer buffer1 = new MockDBTransactionBuffer(dbStore);
+ SCMHADBTransactionBuffer buffer1 =
+ new MockSCMHADBTransactionBuffer(dbStore);
PipelineManagerV2Impl pipelineManager =
createPipelineManager(true, buffer1);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
index ecf0e13..f3682ed 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
@@ -61,6 +61,7 @@ public class TestSCMInstallSnapshot {
@BeforeClass
public static void setup() throws Exception {
conf = new OzoneConfiguration();
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
SCMHAConfiguration scmhaConfiguration = conf.getObject(
SCMHAConfiguration.class);
scmhaConfiguration.setRatisSnapshotThreshold(1L);
@@ -152,7 +153,7 @@ public class TestSCMInstallSnapshot {
getTransactionInfoTable().get(OzoneConsts.TRANSACTION_INFO_KEY)
.getTransactionIndex());
Assert.assertEquals(100,
- scm.getScmHAManager().getDBTransactionBuffer().getLatestTrxInfo()
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().getLatestTrxInfo()
.getTermIndex().getIndex());
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java
index 463d4b3..140b010 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java
@@ -62,7 +62,7 @@ public class TestSCMSnapshot {
@Test
public void testSnapshot() throws Exception {
StorageContainerManager scm = cluster.getStorageContainerManager();
- long snapshotInfo1 = scm.getScmHAManager().getDBTransactionBuffer()
+ long snapshotInfo1 = scm.getScmHAManager().asSCMHADBTransactionBuffer()
.getLatestTrxInfo().getTransactionIndex();
ContainerManagerV2 containerManager = scm.getContainerManager();
PipelineManager pipelineManager = scm.getPipelineManager();
@@ -74,7 +74,7 @@ public class TestSCMSnapshot {
containerManager.allocateContainer(
RATIS, ONE, "Owner2").getPipelineID());
pipelineManager.openPipeline(ratisPipeline2.getId());
- long snapshotInfo2 = scm.getScmHAManager().getDBTransactionBuffer()
+ long snapshotInfo2 = scm.getScmHAManager().asSCMHADBTransactionBuffer()
.getLatestTrxInfo().getTransactionIndex();
Assert.assertTrue(
@@ -93,7 +93,7 @@ public class TestSCMSnapshot {
cluster.restartStorageContainerManager(false);
TransactionInfo trxInfoAfterRestart =
- scm.getScmHAManager().getDBTransactionBuffer().getLatestTrxInfo();
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().getLatestTrxInfo();
Assert.assertTrue(
trxInfoAfterRestart.getTransactionIndex() >= snapshotInfo2);
try {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 07ae25a..b0ce55d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -631,7 +631,7 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
Path metaDir = Paths.get(path, "ozone-meta");
Files.createDirectories(metaDir);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
- conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+ // conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
if (!chunkSize.isPresent()) {
//set it to 1MB by default in tests
chunkSize = Optional.of(1);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 76a4f82..65c70b8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -91,6 +91,8 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_D
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY;
import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.net.ServerSocketUtil.getPort;
@@ -101,6 +103,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEY
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import org.apache.ratis.protocol.ClientId;
@@ -171,6 +174,11 @@ public final class TestSecureOzoneCluster {
getPort(OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT, 100));
conf.setInt(OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
getPort(OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT, 100));
+ // use the same base ports as MiniOzoneHACluster
+ conf.setInt(OZONE_SCM_RATIS_PORT_KEY, getPort(1200, 100));
+ conf.setInt(OZONE_SCM_GRPC_PORT_KEY, getPort(1201, 100));
+ conf.set(OZONE_OM_ADDRESS_KEY, "localhost:1202");
+
DefaultMetricsSystem.setMiniClusterMode(true);
final String path = folder.newFolder().toString();
@@ -329,7 +337,7 @@ public final class TestSecureOzoneCluster {
scmStore.setScmId(scmId);
// writes the version file properties
scmStore.initialize();
- if (!SCMHAUtils.isSCMHAEnabled(conf)) {
+ if (SCMHAUtils.isSCMHAEnabled(conf)) {
SCMRatisServerImpl.initialize(clusterId, scmId,
SCMHANodeDetails.loadSCMHAConfig(conf).getLocalNodeDetails(), conf);
}
@@ -705,6 +713,9 @@ public final class TestSecureOzoneCluster {
if (scm != null) {
scm.stop();
}
+ if (om != null) {
+ om.stop();
+ }
IOUtils.closeQuietly(om);
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index fce4e57..a6bdee5 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.NodeManager;
@@ -298,8 +299,10 @@ public class TestStorageContainerManager {
// empty again.
GenericTestUtils.waitFor(() -> {
try {
- cluster.getStorageContainerManager().getScmHAManager()
- .getDBTransactionBuffer().flush();
+ if (SCMHAUtils.isSCMHAEnabled(cluster.getConf())) {
+ cluster.getStorageContainerManager().getScmHAManager()
+ .asSCMHADBTransactionBuffer().flush();
+ }
return delLog.getNumOfValidTransactions() == 0;
} catch (IOException e) {
return false;
@@ -326,8 +329,10 @@ public class TestStorageContainerManager {
// eventually these TX will success.
GenericTestUtils.waitFor(() -> {
try {
- cluster.getStorageContainerManager().getScmHAManager()
- .getDBTransactionBuffer().flush();
+ if (SCMHAUtils.isSCMHAEnabled(cluster.getConf())) {
+ cluster.getStorageContainerManager().getScmHAManager()
+ .asSCMHADBTransactionBuffer().flush();
+ }
return delLog.getFailedTransactions().size() == 0;
} catch (IOException e) {
return false;
@@ -699,7 +704,9 @@ public class TestStorageContainerManager {
Map<Long, List<Long>> containerBlocksMap)
throws IOException {
delLog.addTransactions(containerBlocksMap);
- scm.getScmHAManager().getDBTransactionBuffer().flush();
+ if (SCMHAUtils.isSCMHAEnabled(scm.getConfiguration())) {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
+ }
}
@SuppressWarnings("visibilitymodifier")
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
index eb16b80..dfd2692 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -283,8 +284,10 @@ public class TestDeleteWithSlowFollower {
deleteKey("ratis");
GenericTestUtils.waitFor(() -> {
try {
- cluster.getStorageContainerManager().getScmHAManager()
- .getDBTransactionBuffer().flush();
+ if (SCMHAUtils.isSCMHAEnabled(cluster.getConf())) {
+ cluster.getStorageContainerManager().getScmHAManager()
+ .asSCMHADBTransactionBuffer().flush();
+ }
return
dnStateMachine.getCommandDispatcher()
.getDeleteBlocksCommandHandler().getInvocationCount() >= 1;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
index 2ffddd7..c242d61 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
@@ -138,7 +138,8 @@ public class TestSCMInstallSnapshotWithHA {
// Get the latest db checkpoint from the leader SCM.
TransactionInfo transactionInfo =
- leaderSCM.getScmHAManager().getDBTransactionBuffer().getLatestTrxInfo();
+ leaderSCM.getScmHAManager().asSCMHADBTransactionBuffer()
+ .getLatestTrxInfo();
TermIndex leaderTermIndex =
TermIndex.valueOf(transactionInfo.getTerm(),
transactionInfo.getTransactionIndex());
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index df38a57..9ebd3d0 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@ -37,13 +37,13 @@ import org.apache.hadoop.hdds.scm.container.ReplicationManager;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicyFactory;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementMetrics;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.ha.MockDBTransactionBuffer;
import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMHANodeDetails;
import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails;
import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBTransactionBufferImpl;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
import org.apache.hadoop.hdds.scm.node.DeadNodeHandler;
@@ -119,7 +119,7 @@ public class ReconStorageContainerManagerFacade
this.dbStore = DBStoreBuilder
.createDBStore(ozoneConfiguration, new ReconSCMDBDefinition());
this.scmhaManager = MockSCMHAManager.getInstance(
- true, new MockDBTransactionBuffer(dbStore));
+ true, new SCMDBTransactionBufferImpl());
this.sequenceIdGen = new SequenceIdGenerator(
conf, scmhaManager, ReconSCMDBDefinition.SEQUENCE_ID.getTable(dbStore));
this.nodeManager =
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java
index 3155756..b591bb6 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.ha.MockDBTransactionBuffer;
+import org.apache.hadoop.hdds.scm.ha.MockSCMHADBTransactionBuffer;
import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
@@ -83,7 +83,7 @@ public class AbstractReconContainerManagerTest {
conf.set(OZONE_SCM_NAMES, "localhost");
store = DBStoreBuilder.createDBStore(conf, new ReconSCMDBDefinition());
scmhaManager = MockSCMHAManager.getInstance(
- true, new MockDBTransactionBuffer(store));
+ true, new MockSCMHADBTransactionBuffer(store));
sequenceIdGen = new SequenceIdGenerator(
conf, scmhaManager, ReconSCMDBDefinition.SEQUENCE_ID.getTable(store));
scmContext = SCMContext.emptyContext();
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java
index 1de004b..9cbf2d2 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.ha.MockDBTransactionBuffer;
+import org.apache.hadoop.hdds.scm.ha.MockSCMHADBTransactionBuffer;
import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
@@ -81,7 +81,7 @@ public class TestReconPipelineManager {
scmStorageConfig = new ReconStorageConfig(conf);
store = DBStoreBuilder.createDBStore(conf, new ReconSCMDBDefinition());
scmhaManager = MockSCMHAManager.getInstance(
- true, new MockDBTransactionBuffer(store));
+ true, new MockSCMHADBTransactionBuffer(store));
scmContext = SCMContext.emptyContext();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org