You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by du...@apache.org on 2021/12/31 03:42:25 UTC
[rocketmq-streams] branch main updated: [ISSUE #113]Multi rocketmq source example (#113)
This is an automated email from the ASF dual-hosted git repository.
duhengforever pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
The following commit(s) were added to refs/heads/main by this push:
new 35f0620 [ISSUE #113]Multi rocketmq source example (#113)
35f0620 is described below
commit 35f06200d752a5dcc3bcc04324b3c4fce28ded58
Author: lizhiboo <li...@yeah.net>
AuthorDate: Fri Dec 31 11:42:19 2021 +0800
[ISSUE #113]Multi rocketmq source example (#113)
* add multi rocketmq source example
* code style
* code style
---
.../mutilconsumer/MutilStreamsClientTest.java | 2 +-
.../streams/examples/mutilconsumer/Producer.java | 8 +-
.../streams/examples/rocketmqsource/Constant.java | 4 +-
.../MultiRocketMQSourceStreamsExample.java | 115 +++++++++++++++++++++
4 files changed, 122 insertions(+), 7 deletions(-)
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java
index 550bb34..2d0caf5 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MutilStreamsClientTest.java
@@ -43,7 +43,7 @@ public class MutilStreamsClientTest {
producerPool.submit(new Runnable() {
@Override
public void run() {
- Producer.produceInLoop("data.txt");
+ Producer.produceInLoop(RMQ_TOPIC, "data.txt");
}
});
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/Producer.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/Producer.java
index e1591db..ed01963 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/Producer.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/Producer.java
@@ -26,8 +26,7 @@ import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.streams.examples.rocketmqsource.ProducerFromFile;
-import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.NAMESRV_ADDRESS;
-import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_TOPIC;
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.*;
public class Producer {
private static final AtomicInteger count = new AtomicInteger(0);
@@ -37,7 +36,7 @@ public class Producer {
*
* @param fileName
*/
- public static void produceInLoop(String fileName) {
+ public static void produceInLoop(String topic, String fileName) {
DefaultMQProducer producer = new DefaultMQProducer("test-group");
try {
@@ -52,11 +51,10 @@ public class Producer {
}
for (String str : result) {
- Message msg = new Message(RMQ_TOPIC, "", str.getBytes(RemotingHelper.DEFAULT_CHARSET));
+ Message msg = new Message(topic, "", str.getBytes(RemotingHelper.DEFAULT_CHARSET));
producer.send(msg);
count.getAndIncrement();
}
-
Thread.sleep(100);
}
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/Constant.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/Constant.java
index c9287f4..6bfe877 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/Constant.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/Constant.java
@@ -22,6 +22,8 @@ package org.apache.rocketmq.streams.examples.rocketmqsource;
public class Constant {
public static final String NAMESRV_ADDRESS = "127.0.0.1:9876";
public static final String RMQ_TOPIC = "NormalTestTopic";
- public static final String RMQ_CONSUMER_GROUP_NAME = "test-group-02";
+ public static final String RMQ_TOPIC_OTHER = "NormalTestTopic1";
+ public static final String RMQ_CONSUMER_GROUP_NAME = "test-group-01";
+ public static final String RMQ_CONSUMER_GROUP_NAME_OTHER = "test-group-02";
public static final String TAGS = "*";
}
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/MultiRocketMQSourceStreamsExample.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/MultiRocketMQSourceStreamsExample.java
new file mode 100644
index 0000000..fc30177
--- /dev/null
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/rocketmqsource/MultiRocketMQSourceStreamsExample.java
@@ -0,0 +1,115 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.rocketmq.streams.examples.rocketmqsource;
+
+import org.apache.rocketmq.streams.client.StreamBuilder;
+import org.apache.rocketmq.streams.client.source.DataStreamSource;
+import org.apache.rocketmq.streams.client.strategy.WindowStrategy;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.client.transform.JoinStream;
+import org.apache.rocketmq.streams.client.transform.window.Time;
+import org.apache.rocketmq.streams.examples.mutilconsumer.Producer;
+
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_TOPIC;
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_TOPIC_OTHER;
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.NAMESRV_ADDRESS;
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_CONSUMER_GROUP_NAME;
+import static org.apache.rocketmq.streams.examples.rocketmqsource.Constant.RMQ_CONSUMER_GROUP_NAME_OTHER;
+
+public class MultiRocketMQSourceStreamsExample {
+ private static ExecutorService producerPool = Executors.newFixedThreadPool(2);
+ private static ExecutorService consumerPool = Executors.newCachedThreadPool();
+ private static Random random = new Random();
+
+
+ public static void main(String[] args) {
+ //producer
+ producerPool.submit(new Runnable() {
+ @Override
+ public void run() {
+ Producer.produceInLoop(RMQ_TOPIC, "data.txt");
+ }
+ });
+
+ //producer
+ producerPool.submit(new Runnable() {
+ @Override
+ public void run() {
+ Producer.produceInLoop(RMQ_TOPIC_OTHER, "data.txt");
+ }
+ });
+
+
+ //consumer
+ for (int i = 0; i < 1; i++) {
+ consumerPool.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ runOneStreamsClient(77);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+
+ }
+
+ private static void runOneStreamsClient(int index) {
+ int namespaceIndex = index;
+ int pipelineIndex = index;
+ DataStreamSource leftSource = StreamBuilder.dataStream("namespace" + namespaceIndex, "pipeline" + pipelineIndex);
+ DataStream left = leftSource.fromRocketmq(
+ RMQ_TOPIC,
+ RMQ_CONSUMER_GROUP_NAME,
+ true,
+ NAMESRV_ADDRESS);
+
+ int otherPipelineIndex = index + 1;
+ DataStreamSource rightSource = StreamBuilder.dataStream("namespace" + namespaceIndex, "pipeline" + otherPipelineIndex);
+ DataStream right = rightSource.fromRocketmq(
+ RMQ_TOPIC_OTHER,
+ RMQ_CONSUMER_GROUP_NAME_OTHER,
+ true,
+ NAMESRV_ADDRESS);
+
+ left.leftJoin(right)
+ .on("(InFlow,==,InFlow)")
+ .window(Time.minutes(1))
+ .toDataSteam()
+ .map(message -> {
+ System.out.println(message);
+ return message + "===";
+ })
+ .toPrint(1)
+ .with(WindowStrategy.highPerformance())
+ .start();
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+}