You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sk...@apache.org on 2021/07/22 13:24:02 UTC
[ignite-3] branch main updated: IGNITE-14848 replace raft
implementation. Fixes #197
This is an automated email from the ASF dual-hosted git repository.
sk0x50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new b8d0737 IGNITE-14848 replace raft implementation. Fixes #197
b8d0737 is described below
commit b8d07376afb4c5637e5cdbeaab2c807a67a59164
Author: Mirza Aliev <al...@gmail.com>
AuthorDate: Thu Jul 22 16:23:31 2021 +0300
IGNITE-14848 replace raft implementation. Fixes #197
Signed-off-by: Slava Koptilin <sl...@gmail.com>
---
modules/metastorage-client/pom.xml | 7 ++++
.../client/ITMetaStorageServiceTest.java | 11 +++--
.../internal/metastorage/MetaStorageManager.java | 2 +-
modules/raft/pom.xml | 19 +++++++++
.../raft/server/ITSimpleCounterServerTest.java | 13 +++++-
.../java/org/apache/ignite/internal/raft/Loza.java | 49 ++++++++++++----------
.../internal/raft/server/impl/RaftServerImpl.java | 2 +-
.../apache/ignite/internal/app/IgnitionImpl.java | 2 +-
modules/table/pom.xml | 7 ++++
.../ignite/distributed/ITDistributedTableTest.java | 15 +++++--
.../internal/table/distributed/TableManager.java | 2 +-
parent/pom.xml | 7 ++++
12 files changed, 101 insertions(+), 35 deletions(-)
diff --git a/modules/metastorage-client/pom.xml b/modules/metastorage-client/pom.xml
index 0041b99..10ca77b 100644
--- a/modules/metastorage-client/pom.xml
+++ b/modules/metastorage-client/pom.xml
@@ -93,5 +93,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-raft</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/modules/metastorage-client/src/integrationTest/java/org/apache/ignite/internal/metastorage/client/ITMetaStorageServiceTest.java b/modules/metastorage-client/src/integrationTest/java/org/apache/ignite/internal/metastorage/client/ITMetaStorageServiceTest.java
index a30efac..b4b210d 100644
--- a/modules/metastorage-client/src/integrationTest/java/org/apache/ignite/internal/metastorage/client/ITMetaStorageServiceTest.java
+++ b/modules/metastorage-client/src/integrationTest/java/org/apache/ignite/internal/metastorage/client/ITMetaStorageServiceTest.java
@@ -36,6 +36,8 @@ import org.apache.ignite.internal.metastorage.server.KeyValueStorage;
import org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener;
import org.apache.ignite.internal.raft.server.RaftServer;
import org.apache.ignite.internal.raft.server.impl.RaftServerImpl;
+import org.apache.ignite.internal.testframework.WorkDirectory;
+import org.apache.ignite.internal.testframework.WorkDirectoryExtension;
import org.apache.ignite.internal.util.Cursor;
import org.apache.ignite.lang.ByteArray;
import org.apache.ignite.lang.IgniteLogger;
@@ -59,6 +61,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
import static java.util.stream.Collectors.toList;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
@@ -75,6 +78,7 @@ import static org.mockito.Mockito.verify;
* Meta storage client tests.
*/
@SuppressWarnings("WeakerAccess")
+@ExtendWith(WorkDirectoryExtension.class)
public class ITMetaStorageServiceTest {
/** The logger. */
private static final IgniteLogger LOG = IgniteLogger.forClass(ITMetaStorageServiceTest.class);
@@ -88,9 +92,6 @@ public class ITMetaStorageServiceTest {
/** */
private static final String METASTORAGE_RAFT_GROUP_NAME = "METASTORAGE_RAFT_GROUP";
- /** */
- public static final int LATEST_REVISION = -1;
-
/** Factory. */
private static final RaftClientMessagesFactory FACTORY = new RaftClientMessagesFactory();
@@ -135,6 +136,10 @@ public class ITMetaStorageServiceTest {
/** Meta storage raft server. */
private RaftServer metaStorageRaftSrv;
+ /** */
+ @WorkDirectory
+ private Path dataPath;
+
static {
EXPECTED_RESULT_MAP = new TreeMap<>();
diff --git a/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/MetaStorageManager.java b/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/MetaStorageManager.java
index 21a6699..9dc383d 100644
--- a/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/MetaStorageManager.java
+++ b/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/MetaStorageManager.java
@@ -158,7 +158,7 @@ public class MetaStorageManager {
metaStorageNodesOnStart = true;
this.metaStorageSvcFut = CompletableFuture.completedFuture(new MetaStorageServiceImpl(
- raftMgr.startRaftGroup(
+ raftMgr.prepareRaftGroup(
METASTORAGE_RAFT_GROUP_NAME,
clusterNetSvc.topologyService().allMembers().stream().filter(
metaStorageNodesContainsLocPred).
diff --git a/modules/raft/pom.xml b/modules/raft/pom.xml
index a158299..7686702 100644
--- a/modules/raft/pom.xml
+++ b/modules/raft/pom.xml
@@ -122,6 +122,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
@@ -139,6 +146,18 @@
<skip>true</skip>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-testJar</id>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/modules/raft/src/integrationTest/java/org/apache/ignite/raft/server/ITSimpleCounterServerTest.java b/modules/raft/src/integrationTest/java/org/apache/ignite/raft/server/ITSimpleCounterServerTest.java
index 38aec61..1f87528 100644
--- a/modules/raft/src/integrationTest/java/org/apache/ignite/raft/server/ITSimpleCounterServerTest.java
+++ b/modules/raft/src/integrationTest/java/org/apache/ignite/raft/server/ITSimpleCounterServerTest.java
@@ -17,9 +17,12 @@
package org.apache.ignite.raft.server;
+import java.nio.file.Path;
import java.util.List;
import org.apache.ignite.internal.raft.server.RaftServer;
-import org.apache.ignite.internal.raft.server.impl.RaftServerImpl;
+import org.apache.ignite.internal.raft.server.impl.JRaftServerImpl;
+import org.apache.ignite.internal.testframework.WorkDirectory;
+import org.apache.ignite.internal.testframework.WorkDirectoryExtension;
import org.apache.ignite.network.ClusterNode;
import org.apache.ignite.network.ClusterService;
import org.apache.ignite.network.NetworkAddress;
@@ -30,6 +33,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.extension.ExtendWith;
import static org.apache.ignite.raft.jraft.test.TestUtils.waitForTopology;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -40,6 +44,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Single node raft server.
*/
+@ExtendWith(WorkDirectoryExtension.class)
class ITSimpleCounterServerTest extends RaftServerAbstractTest {
/**
* The server implementation.
@@ -66,6 +71,10 @@ class ITSimpleCounterServerTest extends RaftServerAbstractTest {
*/
private RaftGroupService client2;
+ /** */
+ @WorkDirectory
+ private Path dataPath;
+
/**
* @param testInfo Test info.
*/
@@ -77,7 +86,7 @@ class ITSimpleCounterServerTest extends RaftServerAbstractTest {
ClusterService service = clusterService(addr.toString(), PORT, List.of(), true);
- server = new RaftServerImpl(service, FACTORY) {
+ server = new JRaftServerImpl(service, dataPath.toString()) {
@Override public synchronized void shutdown() throws Exception {
super.shutdown();
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/Loza.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/Loza.java
index 625153a..638c2ab 100644
--- a/modules/raft/src/main/java/org/apache/ignite/internal/raft/Loza.java
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/Loza.java
@@ -20,7 +20,7 @@ package org.apache.ignite.internal.raft;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.ignite.internal.raft.server.RaftServer;
-import org.apache.ignite.internal.raft.server.impl.RaftServerImpl;
+import org.apache.ignite.internal.raft.server.impl.JRaftServerImpl;
import org.apache.ignite.network.ClusterNode;
import org.apache.ignite.network.ClusterService;
import org.apache.ignite.raft.client.Peer;
@@ -53,51 +53,54 @@ public class Loza {
*
* @param clusterNetSvc Cluster network service.
*/
- public Loza(ClusterService clusterNetSvc) {
+ public Loza(ClusterService clusterNetSvc, String dataPath) {
this.clusterNetSvc = clusterNetSvc;
- this.raftServer = new RaftServerImpl(clusterNetSvc, FACTORY);
+ this.raftServer = new JRaftServerImpl(clusterNetSvc, dataPath);
}
/**
- * Creates a RAFT group.
+ * Creates a raft group service.
+ * If {@code nodes} contains the current node, then raft group starts on the current node.
*
- * @param groupId RAFT group id.
- * @param peers Group peers.
- * @param lsnr Group listener.
- * @return A RAFT group client.
+ * @param groupId Raft group id.
+ * @param nodes Raft group nodes.
+ * @param lsnr Raft group listener.
+ * @return A service providing operations on a raft group.
*/
- public RaftGroupService startRaftGroup(String groupId, List<ClusterNode> peers, RaftGroupListener lsnr) {
- assert !peers.isEmpty();
+ public RaftGroupService prepareRaftGroup(String groupId, List<ClusterNode> nodes, RaftGroupListener lsnr) {
+ assert !nodes.isEmpty();
- //Now we are using only one node in a raft group.
- //TODO: IGNITE-13885 Investigate jraft implementation for replication framework based on RAFT protocol.
- if (peers.get(0).name().equals(clusterNetSvc.topologyService().localMember().name()))
- raftServer.startRaftGroup(groupId, lsnr, List.of(new Peer(peers.get(0).address())));
+ List<Peer> peers = nodes.stream().map(n -> new Peer(n.address())).collect(Collectors.toList());
+
+ String locNodeName = clusterNetSvc.topologyService().localMember().name();
+
+ if (nodes.stream().map(ClusterNode::name).collect(Collectors.toSet()).contains(locNodeName))
+ raftServer.startRaftGroup(groupId, lsnr, peers);
return new RaftGroupServiceImpl(
groupId,
clusterNetSvc,
FACTORY,
TIMEOUT,
- peers.stream().map(i -> new Peer(i.address())).collect(Collectors.toList()),
+ peers,
true,
DELAY
);
}
/**
- * Stops a RAFT group.
+ * Stops a raft group on the current node if {@code nodes} contains the current node.
*
- * @param groupId RAFT group id.
- * @param peers Group peers.
+ * @param groupId Raft group id.
+ * @param nodes Raft group nodes.
*/
- public void stopRaftGroup(String groupId, List<ClusterNode> peers) {
- assert !peers.isEmpty();
+ public void stopRaftGroup(String groupId, List<ClusterNode> nodes) {
+ assert !nodes.isEmpty();
+
+ String locNodeName = clusterNetSvc.topologyService().localMember().name();
- //Now we are using only one node in a raft group.
- //TODO: IGNITE-13885 Investigate jraft implementation for replication framework based on RAFT protocol.
- if (peers.get(0).name().equals(clusterNetSvc.topologyService().localMember().name()))
+ if (nodes.stream().map(ClusterNode::name).collect(Collectors.toSet()).contains(locNodeName))
raftServer.stopRaftGroup(groupId);
}
}
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/server/impl/RaftServerImpl.java b/modules/raft/src/test/java/org/apache/ignite/internal/raft/server/impl/RaftServerImpl.java
similarity index 98%
rename from modules/raft/src/main/java/org/apache/ignite/internal/raft/server/impl/RaftServerImpl.java
rename to modules/raft/src/test/java/org/apache/ignite/internal/raft/server/impl/RaftServerImpl.java
index 699bdd4..767bddc 100644
--- a/modules/raft/src/main/java/org/apache/ignite/internal/raft/server/impl/RaftServerImpl.java
+++ b/modules/raft/src/test/java/org/apache/ignite/internal/raft/server/impl/RaftServerImpl.java
@@ -47,7 +47,7 @@ import org.jetbrains.annotations.Nullable;
/**
* A single node service implementation.
- * @deprecated TODO asch Replace with jraft server https://issues.apache.org/jira/browse/IGNITE-14848
+ * Only for test purposes.
*/
public class RaftServerImpl implements RaftServer {
/** */
diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgnitionImpl.java b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgnitionImpl.java
index ed5a75d..11e5aac 100644
--- a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgnitionImpl.java
+++ b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgnitionImpl.java
@@ -192,7 +192,7 @@ public class IgnitionImpl implements Ignition {
clusterNetSvc.start();
// Raft Component startup.
- Loza raftMgr = new Loza(clusterNetSvc);
+ Loza raftMgr = new Loza(clusterNetSvc, workDir.toString());
// Meta storage Component startup.
MetaStorageManager metaStorageMgr = new MetaStorageManager(
diff --git a/modules/table/pom.xml b/modules/table/pom.xml
index 7614a1e..4adb93a 100644
--- a/modules/table/pom.xml
+++ b/modules/table/pom.xml
@@ -121,6 +121,13 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
<!-- Benchmarks dependencies -->
<dependency>
<groupId>org.openjdk.jmh</groupId>
diff --git a/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ITDistributedTableTest.java b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ITDistributedTableTest.java
index fc6284a..524d4a5 100644
--- a/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ITDistributedTableTest.java
+++ b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ITDistributedTableTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.distributed;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -28,7 +29,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import org.apache.ignite.internal.affinity.RendezvousAffinityFunction;
import org.apache.ignite.internal.raft.server.RaftServer;
-import org.apache.ignite.internal.raft.server.impl.RaftServerImpl;
+import org.apache.ignite.internal.raft.server.impl.JRaftServerImpl;
import org.apache.ignite.internal.schema.BinaryRow;
import org.apache.ignite.internal.schema.ByteBufferRow;
import org.apache.ignite.internal.schema.Column;
@@ -43,6 +44,8 @@ import org.apache.ignite.internal.table.distributed.command.InsertCommand;
import org.apache.ignite.internal.table.distributed.command.response.SingleRowResponse;
import org.apache.ignite.internal.table.distributed.raft.PartitionListener;
import org.apache.ignite.internal.table.distributed.storage.InternalTableImpl;
+import org.apache.ignite.internal.testframework.WorkDirectory;
+import org.apache.ignite.internal.testframework.WorkDirectoryExtension;
import org.apache.ignite.lang.IgniteLogger;
import org.apache.ignite.network.ClusterLocalConfiguration;
import org.apache.ignite.network.ClusterNode;
@@ -64,6 +67,7 @@ import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -73,6 +77,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Distributed internal table tests.
*/
+@ExtendWith(WorkDirectoryExtension.class)
public class ITDistributedTableTest {
/** The logger. */
private static final IgniteLogger LOG = IgniteLogger.forClass(ITDistributedTableTest.class);
@@ -108,6 +113,10 @@ public class ITDistributedTableTest {
/** Cluster. */
private ArrayList<ClusterService> cluster = new ArrayList<>();
+ /** */
+ @WorkDirectory
+ private Path dataPath;
+
/**
* Start all cluster nodes before each test.
*/
@@ -154,7 +163,7 @@ public class ITDistributedTableTest {
public void partitionListener() throws Exception {
String grpId = "part";
- RaftServer partSrv = new RaftServerImpl(cluster.get(0), FACTORY);
+ RaftServer partSrv = new JRaftServerImpl(cluster.get(0), dataPath.toString());
List<Peer> conf = List.of(new Peer(cluster.get(0).topologyService().localMember().address()));
@@ -215,7 +224,7 @@ public class ITDistributedTableTest {
HashMap<ClusterNode, RaftServer> raftServers = new HashMap<>(NODES);
for (int i = 0; i < NODES; i++)
- raftServers.put(cluster.get(i).topologyService().localMember(), new RaftServerImpl(cluster.get(i), FACTORY));
+ raftServers.put(cluster.get(i).topologyService().localMember(), new JRaftServerImpl(cluster.get(i), dataPath.toString()));
List<List<ClusterNode>> assignment = RendezvousAffinityFunction.assignPartitions(
cluster.stream().map(node -> node.topologyService().localMember()).collect(Collectors.toList()),
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
index b8e8a7e..380a269 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
@@ -155,7 +155,7 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
HashMap<Integer, RaftGroupService> partitionMap = new HashMap<>(partitions);
for (int p = 0; p < partitions; p++) {
- partitionMap.put(p, raftMgr.startRaftGroup(
+ partitionMap.put(p, raftMgr.prepareRaftGroup(
raftGroupName(tblId, p),
assignment.get(p),
new PartitionListener()
diff --git a/parent/pom.xml b/parent/pom.xml
index 0130ec1..9aeaaa9 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -448,6 +448,13 @@
<dependency>
<groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-raft</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
<artifactId>ignite-storage-api</artifactId>
<version>${project.version}</version>
<type>test-jar</type>