You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by du...@apache.org on 2021/12/28 05:44:00 UTC
[rocketmq-streams] branch main updated: add multi rocketmq source example (#106)
This is an automated email from the ASF dual-hosted git repository.
duhengforever pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
The following commit(s) were added to refs/heads/main by this push:
new c6696df add multi rocketmq source example (#106)
c6696df is described below
commit c6696df2d009b8b652c05b31d3ed8cac3a391599
Author: lizhiboo <li...@yeah.net>
AuthorDate: Tue Dec 28 13:43:56 2021 +0800
add multi rocketmq source example (#106)
* add multi rocketmq source example
* code style
---
.../mutilconsumer/MutilStreamsClientTest.java | 2 +-
.../streams/examples/mutilconsumer/Producer.java | 8 +-
.../streams/examples/rocketmqsource/Constant.java | 4 +-
.../MultiRocketMQSourceStreamsExample.java} | 90 ++++++++++++++--------
4 files changed, 63 insertions(+), 41 deletions(-)
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java
index 69fc569..cc8b7c4 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java
@@ -44,7 +44,7 @@ public class MutilStreamsClientTest {
producerPool.submit(new Runnable() {
@Override
public void run() {
- Producer.produceInLoop("data.txt");
+ Producer.produceInLoop(RMQ_TOPIC, "data.txt");
}
});
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/Producer.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/Producer.java
index 7f88c1c..4a237ff 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/Producer.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/Producer.java
@@ -26,8 +26,7 @@ import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.streams.examples.rocketmqsource.ProducerFromFile;
-import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.NAMESRV_ADDRESS;
-import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_TOPIC;
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.*;
public class Producer {
private static final AtomicInteger count = new AtomicInteger(0);
@@ -36,7 +35,7 @@ public class Producer {
* total produce 1000 data.
* @param fileName
*/
- public static void produceInLoop(String fileName) {
+ public static void produceInLoop(String topic, String fileName) {
DefaultMQProducer producer = new DefaultMQProducer("test-group");
try {
@@ -51,11 +50,10 @@ public class Producer {
}
for (String str : result) {
- Message msg = new Message(RMQ_TOPIC, "", str.getBytes(RemotingHelper.DEFAULT_CHARSET));
+ Message msg = new Message(topic, "", str.getBytes(RemotingHelper.DEFAULT_CHARSET));
producer.send(msg);
count.getAndIncrement();
}
-
Thread.sleep(100);
}
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/Constant.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/Constant.java
index c9287f4..6bfe877 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/Constant.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/Constant.java
@@ -22,6 +22,8 @@ package org.apache.rocketmq.streams.examples.rocketmqsource;
public class Constant {
public static final String NAMESRV_ADDRESS = "127.0.0.1:9876";
public static final String RMQ_TOPIC = "NormalTestTopic";
- public static final String RMQ_CONSUMER_GROUP_NAME = "test-group-02";
+ public static final String RMQ_TOPIC_OTHER = "NormalTestTopic1";
+ public static final String RMQ_CONSUMER_GROUP_NAME = "test-group-01";
+ public static final String RMQ_CONSUMER_GROUP_NAME_OTHER = "test-group-02";
public static final String TAGS = "*";
}
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/MultiRocketMQSourceStreamsExample.java
similarity index 53%
copy from rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java
copy to rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/MultiRocketMQSourceStreamsExample.java
index 69fc569..f8537c1 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/MultiRocketMQSourceStreamsExample.java
@@ -17,24 +17,28 @@
*
*/
-package org.apache.rocketmq.streams.examples.mutilconsumer;
+package org.apache.rocketmq.streams.examples.rocketmqsource;
-import com.alibaba.fastjson.JSONObject;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import org.apache.rocketmq.streams.client.StreamBuilder;
import org.apache.rocketmq.streams.client.source.DataStreamSource;
import org.apache.rocketmq.streams.client.strategy.WindowStrategy;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.client.transform.JoinStream;
import org.apache.rocketmq.streams.client.transform.window.Time;
-import org.apache.rocketmq.streams.client.transform.window.TumblingWindow;
+import org.apache.rocketmq.streams.examples.mutilconsumer.Producer;
+
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_TOPIC;
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_TOPIC_OTHER;
import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.NAMESRV_ADDRESS;
import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_CONSUMER_GROUP_NAME;
-import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_TOPIC;
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_CONSUMER_GROUP_NAME_OTHER;
-public class MutilStreamsClientTest {
- private static ExecutorService producerPool = Executors.newFixedThreadPool(1);
+public class MultiRocketMQSourceStreamsExample {
+ private static ExecutorService producerPool = Executors.newFixedThreadPool(2);
private static ExecutorService consumerPool = Executors.newCachedThreadPool();
private static Random random = new Random();
@@ -44,17 +48,29 @@ public class MutilStreamsClientTest {
producerPool.submit(new Runnable() {
@Override
public void run() {
- Producer.produceInLoop("data.txt");
+ Producer.produceInLoop(RMQ_TOPIC, "data.txt");
+ }
+ });
+
+ //producer
+ producerPool.submit(new Runnable() {
+ @Override
+ public void run() {
+ Producer.produceInLoop(RMQ_TOPIC_OTHER, "data.txt");
}
});
//consumer
- for (int i = 0; i < 2; i++) {
+ for (int i = 0; i < 1; i++) {
consumerPool.submit(new Runnable() {
@Override
public void run() {
- runOneStreamsClient(random.nextInt(100));
+ try {
+ runOneStreamsClient(77);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
});
}
@@ -62,33 +78,39 @@ public class MutilStreamsClientTest {
}
private static void runOneStreamsClient(int index) {
- DataStreamSource source = StreamBuilder.dataStream("namespace" + index, "pipeline" + index);
- source.fromRocketmq(
+ int namespaceIndex = index;
+ int pipelineIndex = index;
+ DataStreamSource leftSource = StreamBuilder.dataStream("namespace" + namespaceIndex, "pipeline" + pipelineIndex);
+ DataStream left = leftSource.fromRocketmq(
RMQ_TOPIC,
RMQ_CONSUMER_GROUP_NAME,
- false,
- NAMESRV_ADDRESS)
- .filter((message) -> {
- try {
- JSONObject.parseObject((String) message);
- } catch (Throwable t) {
- // if can not convert to json, discard it.because all operator are base on json.
- return true;
- }
- return false;
- })
- //must convert message to json.
- .map(message -> JSONObject.parseObject((String) message))
- .window(TumblingWindow.of(Time.seconds(10)))
- .groupBy("ProjectName", "LogStore")
- .sum("OutFlow", "OutFlow")
- .sum("InFlow", "InFlow")
- .count("total")
- .waterMark(5)
- .setLocalStorageOnly(true)
+ true,
+ NAMESRV_ADDRESS);
+
+ int otherPipelineIndex = index + 1;
+ DataStreamSource rightSource = StreamBuilder.dataStream("namespace" + namespaceIndex, "pipeline" + otherPipelineIndex);
+ DataStream right = rightSource.fromRocketmq(
+ RMQ_TOPIC_OTHER,
+ RMQ_CONSUMER_GROUP_NAME_OTHER,
+ true,
+ NAMESRV_ADDRESS);
+
+ left.join(right)
+ .setJoinType(JoinStream.JoinType.LEFT_JOIN)
+ .setCondition("(InFlow,==,InFlow)")
+ .window(Time.minutes(1))
.toDataSteam()
+ .map(message -> {
+ System.out.println(message);
+ return message + "===";
+ })
.toPrint(1)
.with(WindowStrategy.highPerformance())
.start();
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
}
}