You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ratis.apache.org by sz...@apache.org on 2020/11/15 03:25:26 UTC
[incubator-ratis] branch master updated: RATIS-1130. Add E2E test
for Ratis streaming based on MiniRaftCluster (#272). Contributed by Rui
Wang
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ratis.git
The following commit(s) were added to refs/heads/master by this push:
new b092951 RATIS-1130. Add E2E test for Ratis streaming based on MiniRaftCluster (#272). Contributed by Rui Wang
b092951 is described below
commit b0929519d72abf88bb07055d346b4ba92f1d3603
Author: Rui Wang <am...@users.noreply.github.com>
AuthorDate: Sat Nov 14 19:24:52 2020 -0800
RATIS-1130. Add E2E test for Ratis streaming based on MiniRaftCluster (#272). Contributed by Rui Wang
---
.../ratis/netty/server/NettyServerStreamRpc.java | 7 +-
.../ratis/netty/MiniRaftClusterWithNetty.java | 1 +
.../apache/ratis/server/DataStreamServerRpc.java | 4 +
.../server/DisabledDataStreamServerFactory.java | 6 ++
.../java/org/apache/ratis/MiniRaftCluster.java | 43 ++++++++-
.../apache/ratis/datastream/DataStreamTests.java | 105 +++++++++++++++++++++
.../ratis/datastream/TestDataStreamNetty.java | 2 +-
.../TestDataStreamWithNettyMiniRaftCluster.java | 21 ++---
8 files changed, 171 insertions(+), 18 deletions(-)
diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/server/NettyServerStreamRpc.java b/ratis-netty/src/main/java/org/apache/ratis/netty/server/NettyServerStreamRpc.java
index 772e62c..5fc9adc 100644
--- a/ratis-netty/src/main/java/org/apache/ratis/netty/server/NettyServerStreamRpc.java
+++ b/ratis-netty/src/main/java/org/apache/ratis/netty/server/NettyServerStreamRpc.java
@@ -53,6 +53,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -148,7 +149,6 @@ public class NettyServerStreamRpc implements DataStreamServerRpc {
proxies.addPeers(newPeers);
}
-
private ChannelInboundHandler newChannelInboundHandlerAdapter(){
return new ChannelInboundHandlerAdapter(){
@Override
@@ -200,6 +200,11 @@ public class NettyServerStreamRpc implements DataStreamServerRpc {
}
@Override
+ public InetSocketAddress getInetSocketAddress() {
+ return (InetSocketAddress) channelFuture.channel().localAddress();
+ }
+
+ @Override
public void close() {
try {
channelFuture.channel().close().sync();
diff --git a/ratis-netty/src/test/java/org/apache/ratis/netty/MiniRaftClusterWithNetty.java b/ratis-netty/src/test/java/org/apache/ratis/netty/MiniRaftClusterWithNetty.java
index e7597d6..657dd11 100644
--- a/ratis-netty/src/test/java/org/apache/ratis/netty/MiniRaftClusterWithNetty.java
+++ b/ratis-netty/src/test/java/org/apache/ratis/netty/MiniRaftClusterWithNetty.java
@@ -59,6 +59,7 @@ public class MiniRaftClusterWithNetty extends MiniRaftCluster.RpcBase {
RaftPeerId id, StateMachine.Registry stateMachineRegistry , RaftGroup group,
RaftProperties properties) throws IOException {
NettyConfigKeys.Server.setPort(properties, getPort(id, group));
+ NettyConfigKeys.DataStream.setPort(properties, getDataStreamPort(id, group));
return ServerImplUtils.newRaftServer(id, group, stateMachineRegistry, properties, null);
}
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java b/ratis-server/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java
index 2653000..4e948c6 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java
@@ -20,6 +20,7 @@ package org.apache.ratis.server;
import org.apache.ratis.protocol.RaftPeer;
import java.io.Closeable;
+import java.net.InetSocketAddress;
/**
* A server interface handling incoming streams
@@ -30,4 +31,7 @@ public interface DataStreamServerRpc extends RaftPeer.Add, Closeable {
* start server
*/
void start();
+
+ /** @return the address where this RPC server is listening to. */
+ InetSocketAddress getInetSocketAddress();
}
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/DisabledDataStreamServerFactory.java b/ratis-server/src/main/java/org/apache/ratis/server/DisabledDataStreamServerFactory.java
index 67425fa..ffc5b96 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/DisabledDataStreamServerFactory.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/DisabledDataStreamServerFactory.java
@@ -21,6 +21,7 @@ import org.apache.ratis.conf.Parameters;
import org.apache.ratis.datastream.SupportedDataStreamType;
import org.apache.ratis.protocol.RaftPeer;
+import java.net.InetSocketAddress;
import java.util.Collection;
/** A stream factory that does nothing when data stream is disabled. */
@@ -34,6 +35,11 @@ public class DisabledDataStreamServerFactory implements DataStreamServerFactory
public void start() {}
@Override
+ public InetSocketAddress getInetSocketAddress() {
+ return null;
+ }
+
+ @Override
public void close() {}
@Override
diff --git a/ratis-server/src/test/java/org/apache/ratis/MiniRaftCluster.java b/ratis-server/src/test/java/org/apache/ratis/MiniRaftCluster.java
index 05e6428..dff7640 100644
--- a/ratis-server/src/test/java/org/apache/ratis/MiniRaftCluster.java
+++ b/ratis-server/src/test/java/org/apache/ratis/MiniRaftCluster.java
@@ -62,6 +62,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -192,6 +193,16 @@ public abstract class MiniRaftCluster implements Closeable {
protected int getPort(RaftPeerId id, RaftGroup g) {
final RaftPeer p = g != null? g.getPeer(id): peers.get(id);
final String address = p == null? null : p.getAddress();
+ return getPort(address);
+ }
+
+ protected int getDataStreamPort(RaftPeerId id, RaftGroup g) {
+ final RaftPeer p = g != null? g.getPeer(id): peers.get(id);
+ final String address = p == null? null : p.getDataStreamAddress();
+ return getPort(address);
+ }
+
+ private int getPort(String address) {
final InetSocketAddress inetAddress = address != null?
NetUtils.createSocketAddr(address): NetUtils.createLocalServerAddress();
return inetAddress.getPort();
@@ -213,7 +224,10 @@ public abstract class MiniRaftCluster implements Closeable {
public static RaftGroup initRaftGroup(Collection<String> ids) {
final RaftPeer[] peers = ids.stream()
.map(RaftPeerId::valueOf)
- .map(id -> RaftPeer.newBuilder().setId(id).setAddress(NetUtils.createLocalServerAddress()).build())
+ .map(id -> RaftPeer.newBuilder().setId(id)
+ .setAddress(NetUtils.createLocalServerAddress())
+ .setDataStreamAddress(NetUtils.createLocalServerAddress())
+ .build())
.toArray(RaftPeer[]::new);
return RaftGroup.valueOf(RaftGroupId.randomId(), peers);
}
@@ -269,6 +283,17 @@ public abstract class MiniRaftCluster implements Closeable {
return this;
}
+ private void initDataStreamServer() {
+ LOG.info("Setting up data stream servers");
+ for (RaftServerProxy serverProxy : servers.values()) {
+ serverProxy.getDataStreamServerRpc().addRaftPeers(getOtherRaftPeers(serverProxy.getId()));
+ }
+ }
+
+ private Collection<RaftPeer> getOtherRaftPeers(RaftPeerId id) {
+ return peers.values().stream().filter(r -> !r.getId().equals(id)).collect(Collectors.toList());
+ }
+
public RaftServerProxy putNewServer(RaftPeerId id, RaftGroup group, boolean format) {
final RaftServerProxy s = newRaftServer(id, group, format);
Preconditions.assertTrue(servers.put(id, s) == null);
@@ -291,6 +316,7 @@ public abstract class MiniRaftCluster implements Closeable {
LOG.info(".............................................................. ");
initServers();
+ initDataStreamServer();
startServers(servers.values());
this.timer.updateAndGet(t -> t != null? t
@@ -391,7 +417,11 @@ public abstract class MiniRaftCluster implements Closeable {
}
public static RaftPeer toRaftPeer(RaftServerProxy s) {
- return RaftPeer.newBuilder().setId(s.getId()).setAddress(s.getServerRpc().getInetSocketAddress()).build();
+ return RaftPeer.newBuilder()
+ .setId(s.getId())
+ .setAddress(s.getServerRpc().getInetSocketAddress())
+ .setDataStreamAddress(s.getDataStreamServerRpc().getInetSocketAddress())
+ .build();
}
public PeerChanges addNewPeers(int number, boolean startNewPeer)
@@ -668,12 +698,21 @@ public abstract class MiniRaftCluster implements Closeable {
return createClient(leaderId, group, getDefaultRetryPolicy());
}
+ public RaftClient createClient(RaftPeer primaryServer) {
+ return createClient(null, group, getDefaultRetryPolicy(), primaryServer);
+ }
+
public RaftClient createClient(RaftPeerId leaderId, RaftGroup group, RetryPolicy retryPolicy) {
+ return createClient(leaderId, group, retryPolicy, null);
+ }
+
+ public RaftClient createClient(RaftPeerId leaderId, RaftGroup group, RetryPolicy retryPolicy, RaftPeer primaryServer) {
RaftClient.Builder builder = RaftClient.newBuilder()
.setRaftGroup(group)
.setLeaderId(leaderId)
.setProperties(properties)
.setParameters(parameters)
+ .setPrimaryDataStreamServer(primaryServer)
.setRetryPolicy(retryPolicy);
return builder.build();
}
diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamTests.java b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamTests.java
new file mode 100644
index 0000000..1e6459f
--- /dev/null
+++ b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamTests.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.datastream;
+
+import static org.apache.ratis.RaftTestUtil.waitForLeader;
+
+import org.apache.ratis.BaseTest;
+import org.apache.ratis.MiniRaftCluster;
+import org.apache.ratis.RaftConfigKeys;
+import org.apache.ratis.client.DataStreamOutputRpc;
+import org.apache.ratis.client.RaftClient;
+import org.apache.ratis.datastream.DataStreamBaseTest.MultiDataStreamStateMachine;
+import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.proto.RaftProtos.StateMachineLogEntryProto;
+import org.apache.ratis.protocol.DataStreamReply;
+import org.apache.ratis.protocol.RaftGroup;
+import org.apache.ratis.protocol.RaftPeer;
+import org.apache.ratis.server.impl.RaftServerImpl;
+import org.apache.ratis.server.protocol.TermIndex;
+import org.apache.ratis.server.raftlog.RaftLog;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.concurrent.ThreadLocalRandom;
+
+public abstract class DataStreamTests <CLUSTER extends MiniRaftCluster> extends BaseTest
+ implements MiniRaftCluster.Factory.Get<CLUSTER> {
+ {
+ RaftConfigKeys.DataStream.setType(getProperties(), SupportedDataStreamType.NETTY);
+ setStateMachine(MultiDataStreamStateMachine.class);
+ }
+
+ public static final int NUM_SERVERS = 3;
+ // TODO: change bufferSize and bufferNum configurable
+ private static int bufferSize = 1_000_000;
+ private static int bufferNum = 10;
+
+ @Test
+ public void testStreamWrites() throws Exception {
+ runWithNewCluster(NUM_SERVERS, this::testStreamWrites);
+ }
+
+ void testStreamWrites(CLUSTER cluster) throws Exception {
+ final RaftServerImpl leader = waitForLeader(cluster);
+
+ final RaftGroup raftGroup = cluster.getGroup();
+ final Collection<RaftPeer> peers = raftGroup.getPeers();
+ Assert.assertEquals(NUM_SERVERS, peers.size());
+ RaftPeer raftPeer = peers.iterator().next();
+
+ try (RaftClient client = cluster.createClient(raftPeer)) {
+ // send header
+ DataStreamOutputRpc dataStreamOutputRpc = (DataStreamOutputRpc) client.getDataStreamApi().stream();
+
+ // send data
+ final int halfBufferSize = bufferSize / 2;
+ int dataSize = 0;
+ for(int i = 0; i < bufferNum; i++) {
+ final int size = halfBufferSize + ThreadLocalRandom.current().nextInt(halfBufferSize);
+
+ final ByteBuffer bf = DataStreamBaseTest.initBuffer(dataSize, size);
+ dataStreamOutputRpc.writeAsync(bf);
+ dataSize += size;
+ }
+
+ // send close
+ dataStreamOutputRpc.closeAsync().join();
+
+ // get request call id
+ long callId = dataStreamOutputRpc.getHeaderFuture().get().getStreamId();
+
+ // verify the write request is in the Raft log.
+ RaftLog log = leader.getState().getLog();
+ boolean transactionFound = false;
+ for (TermIndex termIndex : log.getEntries(0, Long.MAX_VALUE)) {
+ RaftProtos.LogEntryProto entryProto = log.get(termIndex.getIndex());
+ if (entryProto.hasStateMachineLogEntry()) {
+ StateMachineLogEntryProto stateMachineEntryProto = entryProto.getStateMachineLogEntry();
+ if (stateMachineEntryProto.getCallId() == callId) {
+ transactionFound = true;
+ break;
+ }
+ }
+ }
+ Assert.assertTrue(transactionFound);
+ }
+ }
+}
diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamNetty.java b/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamNetty.java
index 7ae9860..21800ab 100644
--- a/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamNetty.java
+++ b/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamNetty.java
@@ -66,7 +66,7 @@ public class TestDataStreamNetty extends DataStreamBaseTest {
@Override
protected RaftServer newRaftServer(RaftPeer peer, RaftProperties properties) {
final RaftProperties p = new RaftProperties(properties);
- NettyConfigKeys.DataStream.setPort(p, NetUtils.createSocketAddr(peer.getDataStreamAddress()).getPort());
+ NettyConfigKeys.DataStream.setPort(p, NetUtils.createSocketAddr(peer.getDataStreamAddress()).getPort());
return super.newRaftServer(peer, p);
}
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java b/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamWithNettyMiniRaftCluster.java
similarity index 68%
copy from ratis-server/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java
copy to ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamWithNettyMiniRaftCluster.java
index 2653000..8156820 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java
+++ b/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamWithNettyMiniRaftCluster.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,19 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.ratis.server;
-
-import org.apache.ratis.protocol.RaftPeer;
+package org.apache.ratis.datastream;
-import java.io.Closeable;
+import org.apache.ratis.RaftConfigKeys;
+import org.apache.ratis.datastream.DataStreamBaseTest.MultiDataStreamStateMachine;
+import org.apache.ratis.netty.MiniRaftClusterWithNetty;
-/**
- * A server interface handling incoming streams
- * Relays those streams to other servers after persisting
- */
-public interface DataStreamServerRpc extends RaftPeer.Add, Closeable {
- /**
- * start server
- */
- void start();
+public class TestDataStreamWithNettyMiniRaftCluster extends DataStreamTests<MiniRaftClusterWithNetty>
+ implements MiniRaftClusterWithNetty.FactoryGet {
}