You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by do...@apache.org on 2017/06/08 07:54:22 UTC
[01/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-172]log
improvement for rocketmq client closes apache/incubator-rocketmq#90 [Forced
Update!]
Repository: incubator-rocketmq
Updated Branches:
refs/heads/release-4.1.0-incubating 5c16892e5 -> 10933cc0a (forced update)
[ROCKETMQ-172]log improvement for rocketmq client closes apache/incubator-rocketmq#90
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/c183e0d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/c183e0d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/c183e0d4
Branch: refs/heads/release-4.1.0-incubating
Commit: c183e0d4026770a68bedce02507446431cdf6265
Parents: 7bcb3b3
Author: Jaskey <li...@gmail.com>
Authored: Mon Apr 17 19:28:26 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Mon Apr 17 19:28:26 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java | 6 +++---
.../apache/rocketmq/client/impl/factory/MQClientInstance.java | 2 +-
.../apache/rocketmq/remoting/netty/NettyRemotingClient.java | 2 ++
3 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/c183e0d4/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
index 6119e24..ff25334 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
@@ -593,12 +593,12 @@ public class MQClientAPIImpl {
}
} else {
if (!responseFuture.isSendRequestOK()) {
- pullCallback.onException(new MQClientException("send request failed", responseFuture.getCause()));
+ pullCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
} else if (responseFuture.isTimeout()) {
- pullCallback.onException(new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms",
+ pullCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
responseFuture.getCause()));
} else {
- pullCallback.onException(new MQClientException("unknow reseaon", responseFuture.getCause()));
+ pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/c183e0d4/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
index 11266c4..d7e02fe 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
@@ -596,7 +596,7 @@ public class MQClientInstance {
}
}
}
- log.info("topicRouteTable.put TopicRouteData[{}]", cloneTopicRouteData);
+ log.info("topicRouteTable.put. Topic = {}, TopicRouteData[{}]", topic, cloneTopicRouteData);
this.topicRouteTable.put(topic, cloneTopicRouteData);
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/c183e0d4/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
----------------------------------------------------------------------
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
index 85f9244..26088aa 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
@@ -321,6 +321,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
if (update) {
Collections.shuffle(addrs);
+ log.info("name server address updated. NEW : {} , OLD: {}",addrs,old);
this.namesrvAddrList.set(addrs);
}
}
@@ -398,6 +399,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
String newAddr = addrList.get(index);
this.namesrvAddrChoosed.set(newAddr);
+ log.info("new name server is chosen. OLD: {} , NEW: {}. namesrvIndex = {}", addr, newAddr, namesrvIndex);
Channel channelNew = this.createChannel(newAddr);
if (channelNew != null)
return channelNew;
[12/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-179] Fix
errors of IT test cases closes apache/incubator-rocketmq#94
Posted by do...@apache.org.
[ROCKETMQ-179] Fix errors of IT test cases closes apache/incubator-rocketmq#94
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/6a9628b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/6a9628b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/6a9628b3
Branch: refs/heads/release-4.1.0-incubating
Commit: 6a9628b3c3e6835e37baf7b58ad9300364d4d384
Parents: 58f1574
Author: vsair <li...@gmail.com>
Authored: Fri Apr 21 18:21:35 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Fri Apr 21 18:21:35 2017 +0800
----------------------------------------------------------------------
pom.xml | 4 --
.../test/clientinterface/MQCollector.java | 20 +++++++---
.../rmq/concurrent/RMQNormalListner.java | 4 +-
.../org/apache/rocketmq/test/util/MQAdmin.java | 1 -
.../rocketmq/test/base/IntegrationTestBase.java | 39 ++++++++++++++------
.../tag/TagMessageWithSameGroupConsumerIT.java | 10 ++---
.../async/AsyncSendWithMessageQueueIT.java | 5 +--
.../AsyncSendWithMessageQueueSelectorIT.java | 1 -
.../rocketmq/test/delay/NormalMsgDelayIT.java | 3 +-
9 files changed, 53 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6a9628b3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index feb8b14..6fd59ac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -461,10 +461,6 @@
<argLine>@{failsafeArgLine}</argLine>
<excludes>
<exclude>**/NormalMsgDelayIT.java</exclude>
- <exclude>**/BroadCastNormalMsgNotRecvIT.java</exclude>
- <exclude>**/TagMessageWithSameGroupConsumerIT.java</exclude>
- <exclude>**/AsyncSendWithMessageQueueSelectorIT.java</exclude>
- <exclude>**/AsyncSendWithMessageQueueIT.java</exclude>
</excludes>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6a9628b3/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQCollector.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQCollector.java b/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQCollector.java
index 42d4b62..7ccf92a 100644
--- a/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQCollector.java
+++ b/test/src/main/java/org/apache/rocketmq/test/clientinterface/MQCollector.java
@@ -91,11 +91,21 @@ public abstract class MQCollector {
}
public void clearMsg() {
- msgBodys.resetData();
- originMsgs.resetData();
- errorMsgs.resetData();
- originMsgIndex.clear();
- msgRTs.resetData();
+ if (msgBodys != null) {
+ msgBodys.resetData();
+ }
+ if (originMsgs != null) {
+ originMsgs.resetData();
+ }
+ if (originMsgs != null) {
+ errorMsgs.resetData();
+ }
+ if (originMsgIndex != null) {
+ originMsgIndex.clear();
+ }
+ if (msgRTs != null) {
+ msgRTs.resetData();
+ }
}
public void lockCollectors() {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6a9628b3/test/src/main/java/org/apache/rocketmq/test/listener/rmq/concurrent/RMQNormalListner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/rocketmq/test/listener/rmq/concurrent/RMQNormalListner.java b/test/src/main/java/org/apache/rocketmq/test/listener/rmq/concurrent/RMQNormalListner.java
index 0d40881..471fb48 100644
--- a/test/src/main/java/org/apache/rocketmq/test/listener/rmq/concurrent/RMQNormalListner.java
+++ b/test/src/main/java/org/apache/rocketmq/test/listener/rmq/concurrent/RMQNormalListner.java
@@ -63,7 +63,9 @@ public class RMQNormalListner extends AbstractListener implements MessageListene
msgBodys.addData(new String(msg.getBody()));
originMsgs.addData(msg);
- originMsgIndex.put(new String(msg.getBody()), msg);
+ if (originMsgIndex != null) {
+ originMsgIndex.put(new String(msg.getBody()), msg);
+ }
}
return consumeStatus;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6a9628b3/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java b/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java
index 680780a..bd151d0 100644
--- a/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java
+++ b/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java
@@ -45,7 +45,6 @@ public class MQAdmin {
mqAdminExt.start();
mqAdminExt.createTopic(clusterName, topic, queueNum);
} catch (Exception e) {
- e.printStackTrace();
}
long startTime = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6a9628b3/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
index 5329991..9805eba 100644
--- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
+++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
@@ -46,6 +46,8 @@ public class IntegrationTestBase {
protected static final List<BrokerController> BROKER_CONTROLLERS = new ArrayList<>();
protected static final List<NamesrvController> NAMESRV_CONTROLLERS = new ArrayList<>();
protected static int topicCreateTime = 30 * 1000;
+ protected static final int COMMIT_LOG_SIZE = 1024 * 1024 * 256;
+ protected static final int INDEX_NUM = 1000;
protected static Random random = new Random();
@@ -53,18 +55,30 @@ public class IntegrationTestBase {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override public void run() {
- for (NamesrvController namesrvController : NAMESRV_CONTROLLERS) {
- if (namesrvController != null) {
- namesrvController.shutdown();
+ try {
+ for (BrokerController brokerController : BROKER_CONTROLLERS) {
+ if (brokerController != null) {
+ brokerController.shutdown();
+ }
}
- }
- for (BrokerController brokerController : BROKER_CONTROLLERS) {
- if (brokerController != null) {
- brokerController.shutdown();
+
+ // should destroy message store, otherwise could not delete the temp files.
+ for (BrokerController brokerController : BROKER_CONTROLLERS) {
+ if (brokerController != null) {
+ brokerController.getMessageStore().destroy();
+ }
}
- }
- for (File file : TMPE_FILES) {
- deleteFile(file);
+
+ for (NamesrvController namesrvController : NAMESRV_CONTROLLERS) {
+ if (namesrvController != null) {
+ namesrvController.shutdown();
+ }
+ }
+ for (File file : TMPE_FILES) {
+ deleteFile(file);
+ }
+ } catch (Exception e){
+ logger.error("Shutdown error", e);
}
}
});
@@ -75,7 +89,7 @@ public class IntegrationTestBase {
String baseDir = System.getProperty("user.home") + SEP + "unitteststore-" + UUID.randomUUID();
final File file = new File(baseDir);
if (file.exists()) {
- logger.info(String.format("[%s] has already existed, please bake up and remove it for integration tests", baseDir));
+ logger.info(String.format("[%s] has already existed, please back up and remove it for integration tests", baseDir));
System.exit(1);
}
TMPE_FILES.add(file);
@@ -116,6 +130,9 @@ public class IntegrationTestBase {
storeConfig.setStorePathRootDir(baseDir);
storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog");
storeConfig.setHaListenPort(8000 + random.nextInt(1000));
+ storeConfig.setMapedFileSizeCommitLog(COMMIT_LOG_SIZE);
+ storeConfig.setMaxIndexNum(INDEX_NUM);
+ storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);
nettyServerConfig.setListenPort(10000 + random.nextInt(1000));
BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, nettyClientConfig, storeConfig);
try {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6a9628b3/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithSameGroupConsumerIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithSameGroupConsumerIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithSameGroupConsumerIT.java
index 4cf8161..135cbec 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithSameGroupConsumerIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithSameGroupConsumerIT.java
@@ -36,6 +36,7 @@ public class TagMessageWithSameGroupConsumerIT extends BaseConf {
private static Logger logger = Logger.getLogger(TagMessageWith1ConsumerIT.class);
private RMQNormalProducer producer = null;
private String topic = null;
+ private String tag = "tag";
@Before
public void setUp() {
@@ -51,13 +52,12 @@ public class TagMessageWithSameGroupConsumerIT extends BaseConf {
@Test
public void testTwoConsumerWithSameGroup() {
- String tag = "jueyin";
int msgSize = 20;
String originMsgDCName = RandomUtils.getStringByUUID();
String msgBodyDCName = RandomUtils.getStringByUUID();
RMQNormalConsumer consumer1 = getConsumer(nsAddr, topic, tag,
new RMQNormalListner(originMsgDCName, msgBodyDCName));
- RMQNormalConsumer consumer2 = getConsumer(nsAddr, consumer1.getConsumerGroup(), tag,
+ getConsumer(nsAddr, consumer1.getConsumerGroup(), tag,
new RMQNormalListner(originMsgDCName, msgBodyDCName));
producer.send(tag, msgSize);
Assert.assertEquals("Not all are sent", msgSize, producer.getAllUndupMsgBody().size());
@@ -70,7 +70,6 @@ public class TagMessageWithSameGroupConsumerIT extends BaseConf {
@Test
public void testConsumerStartWithInterval() {
- String tag = "jueyin";
int msgSize = 100;
String originMsgDCName = RandomUtils.getStringByUUID();
String msgBodyDCName = RandomUtils.getStringByUUID();
@@ -79,7 +78,7 @@ public class TagMessageWithSameGroupConsumerIT extends BaseConf {
new RMQNormalListner(originMsgDCName, msgBodyDCName));
producer.send(tag, msgSize, 100);
TestUtils.waitForMoment(5);
- RMQNormalConsumer consumer2 = getConsumer(nsAddr, consumer1.getConsumerGroup(), tag,
+ getConsumer(nsAddr, consumer1.getConsumerGroup(), tag,
new RMQNormalListner(originMsgDCName, msgBodyDCName));
TestUtils.waitForMoment(5);
@@ -90,8 +89,7 @@ public class TagMessageWithSameGroupConsumerIT extends BaseConf {
}
@Test
- public void testConsumerStartTwoAndCrashOnsAfterWhile() {
- String tag = "jueyin";
+ public void testConsumerStartTwoAndCrashOneAfterWhile() {
int msgSize = 100;
String originMsgDCName = RandomUtils.getStringByUUID();
String msgBodyDCName = RandomUtils.getStringByUUID();
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6a9628b3/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java
index 53a992c..24a7547 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java
@@ -33,7 +33,6 @@ import static com.google.common.truth.Truth.assertThat;
public class AsyncSendWithMessageQueueIT extends BaseConf {
private static Logger logger = Logger.getLogger(TagMessageWith1ConsumerIT.class);
- private static boolean sendFail = false;
private RMQAsyncSendProducer producer = null;
private String topic = null;
@@ -57,7 +56,7 @@ public class AsyncSendWithMessageQueueIT extends BaseConf {
MessageQueue mq = new MessageQueue(topic, broker1Name, queueId);
producer.asyncSend(msgSize, mq);
- producer.waitForResponse(5 * 1000);
+ producer.waitForResponse(10 * 1000);
assertThat(producer.getSuccessMsgCount()).isEqualTo(msgSize);
consumer.getListner().waitForMessageConsume(producer.getAllMsgBody(), consumeTime);
@@ -72,7 +71,7 @@ public class AsyncSendWithMessageQueueIT extends BaseConf {
mq = new MessageQueue(topic, broker2Name, queueId);
producer.asyncSend(msgSize, mq);
- producer.waitForResponse(5 * 1000);
+ producer.waitForResponse(10 * 1000);
assertThat(producer.getSuccessMsgCount()).isEqualTo(msgSize);
consumer.getListner().waitForMessageConsume(producer.getAllMsgBody(), consumeTime);
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6a9628b3/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java
index 68c2b0e..843441d 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java
@@ -36,7 +36,6 @@ import static com.google.common.truth.Truth.assertThat;
public class AsyncSendWithMessageQueueSelectorIT extends BaseConf {
private static Logger logger = Logger.getLogger(TagMessageWith1ConsumerIT.class);
- private static boolean sendFail = false;
private RMQAsyncSendProducer producer = null;
private String topic = null;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6a9628b3/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java b/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
index 5206dcb..dc5f230 100644
--- a/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
@@ -24,7 +24,6 @@ import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.factory.MQMessageFactory;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQDelayListner;
-import org.apache.rocketmq.test.listener.rmq.order.RMQOrderListener;
import org.apache.rocketmq.test.util.VerifyUtils;
import org.junit.After;
import org.junit.Assert;
@@ -43,7 +42,7 @@ public class NormalMsgDelayIT extends DelayConf {
topic = initTopic();
logger.info(String.format("use topic: %s;", topic));
producer = getProducer(nsAddr, topic);
- consumer = getConsumer(nsAddr, topic, "*", new RMQOrderListener());
+ consumer = getConsumer(nsAddr, topic, "*", new RMQDelayListner());
}
@After
[14/50] [abbrv] incubator-rocketmq git commit: Removed unnecessary
semicolon.
Posted by do...@apache.org.
Removed unnecessary semicolon.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/8feb88d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/8feb88d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/8feb88d1
Branch: refs/heads/release-4.1.0-incubating
Commit: 8feb88d1481a86793846b38aa69c66ace860694e
Parents: 6609c86
Author: shroman <rs...@yahoo.com>
Authored: Thu May 4 20:07:10 2017 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Thu May 4 20:07:10 2017 +0900
----------------------------------------------------------------------
.../org/apache/rocketmq/filter/expression/UnaryInExpression.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/8feb88d1/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java
index 7d9083c..a6cf173 100644
--- a/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java
@@ -58,4 +58,4 @@ abstract public class UnaryInExpression extends UnaryExpression implements Boole
public void setInList(Collection inList) {
this.inList = inList;
}
-};
+}
[46/50] [abbrv] incubator-rocketmq git commit: Release zip too
Posted by do...@apache.org.
Release zip too
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/7374914b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/7374914b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/7374914b
Branch: refs/heads/release-4.1.0-incubating
Commit: 7374914bac077d5de433fe9491d2eb1ed0d38825
Parents: e068ec1
Author: dongeforever <do...@apache.org>
Authored: Tue Jun 6 17:02:15 2017 +0800
Committer: dongeforever <do...@apache.org>
Committed: Tue Jun 6 20:41:41 2017 +0800
----------------------------------------------------------------------
distribution/release.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/7374914b/distribution/release.xml
----------------------------------------------------------------------
diff --git a/distribution/release.xml b/distribution/release.xml
index c67d23e..d87ad5d 100644
--- a/distribution/release.xml
+++ b/distribution/release.xml
@@ -21,6 +21,7 @@
<formats>
<format>dir</format>
<format>tar.gz</format>
+ <format>zip</format>
</formats>
<fileSets>
<fileSet>
[36/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-67]
Consistent Hash allocate strategy closes apache/incubator-rocketmq#67
Posted by do...@apache.org.
[ROCKETMQ-67] Consistent Hash allocate strategy closes apache/incubator-rocketmq#67
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/adae1624
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/adae1624
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/adae1624
Branch: refs/heads/release-4.1.0-incubating
Commit: adae1624d05346cd3632c778a656e4055de6bff3
Parents: c796140
Author: Jaskey <li...@gmail.com>
Authored: Sat May 27 11:42:03 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 11:42:03 2017 +0800
----------------------------------------------------------------------
.../AllocateMessageQueueConsistentHash.java | 124 ++++++++++
.../AllocateMessageQueueConsitentHashTest.java | 243 +++++++++++++++++++
.../consistenthash/ConsistentHashRouter.java | 140 +++++++++++
.../common/consistenthash/HashFunction.java | 24 ++
.../rocketmq/common/consistenthash/Node.java | 28 +++
.../common/consistenthash/VirtualNode.java | 41 ++++
6 files changed, 600 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/adae1624/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java
new file mode 100644
index 0000000..77198b7
--- /dev/null
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.consumer.rebalance;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter;
+import org.apache.rocketmq.common.consistenthash.HashFunction;
+import org.apache.rocketmq.common.consistenthash.Node;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.slf4j.Logger;
+
+/**
+ * Consistent Hashing queue algorithm
+ */
+public class AllocateMessageQueueConsistentHash implements AllocateMessageQueueStrategy {
+ private final Logger log = ClientLogger.getLog();
+
+ private final int virtualNodeCnt;
+ private final HashFunction customHashFunction;
+
+ public AllocateMessageQueueConsistentHash() {
+ this(10);
+ }
+
+ public AllocateMessageQueueConsistentHash(int virtualNodeCnt) {
+ this(virtualNodeCnt,null);
+ }
+
+ public AllocateMessageQueueConsistentHash(int virtualNodeCnt, HashFunction customHashFunction) {
+ if (virtualNodeCnt < 0) {
+ throw new IllegalArgumentException("illegal virtualNodeCnt :" + virtualNodeCnt);
+ }
+ this.virtualNodeCnt = virtualNodeCnt;
+ this.customHashFunction = customHashFunction;
+ }
+
+ @Override
+ public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
+ List<String> cidAll) {
+
+ if (currentCID == null || currentCID.length() < 1) {
+ throw new IllegalArgumentException("currentCID is empty");
+ }
+ if (mqAll == null || mqAll.isEmpty()) {
+ throw new IllegalArgumentException("mqAll is null or mqAll empty");
+ }
+ if (cidAll == null || cidAll.isEmpty()) {
+ throw new IllegalArgumentException("cidAll is null or cidAll empty");
+ }
+
+ List<MessageQueue> result = new ArrayList<MessageQueue>();
+ if (!cidAll.contains(currentCID)) {
+ log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}",
+ consumerGroup,
+ currentCID,
+ cidAll);
+ return result;
+ }
+
+
+ Collection<ClientNode> cidNodes = new ArrayList<>();
+ for (String cid : cidAll) {
+ cidNodes.add(new ClientNode(cid));
+ }
+
+ final ConsistentHashRouter<ClientNode> router; //for building hash ring
+ if (customHashFunction != null) {
+ router = new ConsistentHashRouter<>(cidNodes, virtualNodeCnt, customHashFunction);
+ } else {
+ router = new ConsistentHashRouter<>(cidNodes, virtualNodeCnt);
+ }
+
+ List<MessageQueue> results = new ArrayList<>();
+ for (MessageQueue mq : mqAll) {
+ ClientNode clientNode = router.routeNode(mq.toString());
+ if (clientNode != null && currentCID.equals(clientNode.getKey())) {
+ results.add(mq);
+ }
+ }
+
+ return results;
+
+ }
+
+ @Override
+ public String getName() {
+ return "CONSISTENT_HASH";
+ }
+
+
+ private static class ClientNode implements Node {
+ private final String clientID;
+
+ public ClientNode(String clientID) {
+ this.clientID = clientID;
+ }
+
+ @Override
+ public String getKey() {
+ return clientID;
+ }
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/adae1624/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java
----------------------------------------------------------------------
diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java
new file mode 100644
index 0000000..fc7ab9f
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.consumer.rebalance;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class AllocateMessageQueueConsitentHashTest {
+
+ private String topic;
+ private static final String CID_PREFIX = "CID-";
+
+ @Before
+ public void init() {
+ topic = "topic_test";
+ }
+
+
+
+ public void printMessageQueue(List<MessageQueue> messageQueueList, String name) {
+ if (messageQueueList == null || messageQueueList.size() < 1)
+ return;
+ System.out.println(name + ".......................................start");
+ for (MessageQueue messageQueue : messageQueueList) {
+ System.out.println(messageQueue);
+ }
+ System.out.println(name + ".......................................end");
+ }
+
+ @Test
+ public void testCurrentCIDNotExists() {
+ String currentCID = String.valueOf(Integer.MAX_VALUE);
+ List<String> consumerIdList = createConsumerIdList(2);
+ List<MessageQueue> messageQueueList = createMessageQueueList(6);
+ List<MessageQueue> result = new AllocateMessageQueueConsistentHash().allocate("", currentCID, messageQueueList, consumerIdList);
+ printMessageQueue(result, "testCurrentCIDNotExists");
+ Assert.assertEquals(result.size(), 0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCurrentCIDIllegalArgument() {
+ List<String> consumerIdList = createConsumerIdList(2);
+ List<MessageQueue> messageQueueList = createMessageQueueList(6);
+ new AllocateMessageQueueConsistentHash().allocate("", "", messageQueueList, consumerIdList);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testMessageQueueIllegalArgument() {
+ String currentCID = "0";
+ List<String> consumerIdList = createConsumerIdList(2);
+ new AllocateMessageQueueConsistentHash().allocate("", currentCID, null, consumerIdList);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConsumerIdIllegalArgument() {
+ String currentCID = "0";
+ List<MessageQueue> messageQueueList = createMessageQueueList(6);
+ new AllocateMessageQueueConsistentHash().allocate("", currentCID, messageQueueList, null);
+ }
+
+ @Test
+ public void testAllocate1() {
+ testAllocate(20,10);
+ }
+
+ @Test
+ public void testAllocate2() {
+ testAllocate(10,20);
+ }
+
+
+ @Test
+ public void testRun100RandomCase(){
+ for(int i=0;i<100;i++){
+ int consumerSize = new Random().nextInt(200)+1;//1-200
+ int queueSize = new Random().nextInt(100)+1;//1-100
+ testAllocate(queueSize,consumerSize);
+ try {
+ Thread.sleep(1);
+ } catch (InterruptedException e) {}
+ }
+ }
+
+
+ public void testAllocate(int queueSize, int consumerSize) {
+ AllocateMessageQueueStrategy allocateMessageQueueConsistentHash = new AllocateMessageQueueConsistentHash(3);
+
+ List<MessageQueue> mqAll = createMessageQueueList(queueSize);
+ //System.out.println("mqAll:" + mqAll.toString());
+
+ List<String> cidAll = createConsumerIdList(consumerSize);
+ List<MessageQueue> allocatedResAll = new ArrayList<>();
+
+ Map<MessageQueue, String> allocateToAllOrigin = new TreeMap<>();
+ //test allocate all
+ {
+
+ List<String> cidBegin = new ArrayList<>(cidAll);
+
+ //System.out.println("cidAll:" + cidBegin.toString());
+ for (String cid : cidBegin) {
+ List<MessageQueue> rs = allocateMessageQueueConsistentHash.allocate("testConsumerGroup", cid, mqAll, cidBegin);
+ for (MessageQueue mq : rs) {
+ allocateToAllOrigin.put(mq, cid);
+ }
+ allocatedResAll.addAll(rs);
+ //System.out.println("rs[" + cid + "]:" + rs.toString());
+ }
+
+ Assert.assertTrue(
+ verifyAllocateAll(cidBegin,mqAll, allocatedResAll));
+ }
+
+ Map<MessageQueue, String> allocateToAllAfterRemoveOne = new TreeMap<>();
+ List<String> cidAfterRemoveOne = new ArrayList<>(cidAll);
+ //test allocate remove one cid
+ {
+ String removeCID = cidAfterRemoveOne.remove(0);
+ //System.out.println("removing one cid "+removeCID);
+ List<MessageQueue> mqShouldOnlyChanged = new ArrayList<>();
+ Iterator<Map.Entry<MessageQueue, String>> it = allocateToAllOrigin.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<MessageQueue, String> entry = it.next();
+ if (entry.getValue().equals(removeCID)) {
+ mqShouldOnlyChanged.add(entry.getKey());
+ }
+ }
+
+ //System.out.println("cidAll:" + cidAfterRemoveOne.toString());
+ List<MessageQueue> allocatedResAllAfterRemove = new ArrayList<>();
+ for (String cid : cidAfterRemoveOne) {
+ List<MessageQueue> rs = allocateMessageQueueConsistentHash.allocate("testConsumerGroup", cid, mqAll, cidAfterRemoveOne);
+ allocatedResAllAfterRemove.addAll(rs);
+ for (MessageQueue mq : rs) {
+ allocateToAllAfterRemoveOne.put(mq, cid);
+ }
+ //System.out.println("rs[" + cid + "]:" + "[" + rs.size() + "]" + rs.toString());
+ }
+
+ Assert.assertTrue("queueSize"+queueSize+"consumerSize:"+consumerSize+"\nmqAll:"+mqAll+"\nallocatedResAllAfterRemove"+allocatedResAllAfterRemove,
+ verifyAllocateAll(cidAfterRemoveOne, mqAll, allocatedResAllAfterRemove));
+ verifyAfterRemove(allocateToAllOrigin, allocateToAllAfterRemoveOne, removeCID);
+ }
+
+ List<String> cidAfterAdd = new ArrayList<>(cidAfterRemoveOne);
+ //test allocate add one more cid
+ {
+ String newCid = CID_PREFIX+"NEW";
+ //System.out.println("add one more cid "+newCid);
+ cidAfterAdd.add(newCid);
+ List<MessageQueue> mqShouldOnlyChanged = new ArrayList<>();
+ //System.out.println("cidAll:" + cidAfterAdd.toString());
+ List<MessageQueue> allocatedResAllAfterAdd = new ArrayList<>();
+ Map<MessageQueue, String> allocateToAll3 = new TreeMap<>();
+ for (String cid : cidAfterAdd) {
+ List<MessageQueue> rs = allocateMessageQueueConsistentHash.allocate("testConsumerGroup", cid, mqAll, cidAfterAdd);
+ allocatedResAllAfterAdd.addAll(rs);
+ for (MessageQueue mq : rs) {
+ allocateToAll3.put(mq, cid);
+ if (cid.equals(newCid)){
+ mqShouldOnlyChanged.add(mq);
+ }
+ }
+ //System.out.println("rs[" + cid + "]:" + "[" + rs.size() + "]" + rs.toString());
+ }
+
+ Assert.assertTrue(
+ verifyAllocateAll(cidAfterAdd,mqAll, allocatedResAllAfterAdd));
+ verifyAfterAdd(allocateToAllAfterRemoveOne, allocateToAll3, newCid);
+ }
+ }
+
+ private boolean verifyAllocateAll(List<String> cidAll,List<MessageQueue> mqAll, List<MessageQueue> allocatedResAll) {
+ if (cidAll.isEmpty()){
+ return allocatedResAll.isEmpty();
+ }
+ return mqAll.containsAll(allocatedResAll) && allocatedResAll.containsAll(mqAll);
+ }
+
+ private void verifyAfterRemove(Map<MessageQueue, String> allocateToBefore, Map<MessageQueue, String> allocateAfter, String removeCID) {
+ for (MessageQueue mq : allocateToBefore.keySet()) {
+ String allocateToOrigin = allocateToBefore.get(mq);
+ if (allocateToOrigin.equals(removeCID)) {
+
+ } else {//the rest queue should be the same
+ Assert.assertTrue(allocateAfter.get(mq).equals(allocateToOrigin));//should be the same
+ }
+ }
+ }
+
+ private void verifyAfterAdd(Map<MessageQueue, String> allocateBefore, Map<MessageQueue, String> allocateAfter, String newCID) {
+ for (MessageQueue mq : allocateAfter.keySet()) {
+ String allocateToOrigin = allocateBefore.get(mq);
+ String allocateToAfter = allocateAfter.get(mq);
+ if (allocateToAfter.equals(newCID)) {
+
+ } else {//the rest queue should be the same
+ Assert.assertTrue("it was allocated to "+allocateToOrigin+". Now, it is to "+allocateAfter.get(mq)+" mq:"+mq,allocateAfter.get(mq).equals(allocateToOrigin));//should be the same
+ }
+ }
+ }
+
+ private List<String> createConsumerIdList(int size) {
+ List<String> consumerIdList = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ consumerIdList.add(CID_PREFIX + String.valueOf(i));
+ }
+ return consumerIdList;
+ }
+
+ private List<MessageQueue> createMessageQueueList(int size) {
+ List<MessageQueue> messageQueueList = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ MessageQueue mq = new MessageQueue(topic, "brokerName", i);
+ messageQueueList.add(mq);
+ }
+ return messageQueueList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/adae1624/common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java b/common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java
new file mode 100644
index 0000000..8606c43
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.consistenthash;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/**
+ * To hash Node objects to a hash ring with a certain amount of virtual node.
+ * Method routeNode will return a Node instance which the object key should be allocated to according to consistent hash algorithm
+ *
+ * @param <T>
+ */
+public class ConsistentHashRouter<T extends Node> {
+ private final SortedMap<Long, VirtualNode<T>> ring = new TreeMap<>();
+ private final HashFunction hashFunction;
+
+ public ConsistentHashRouter(Collection<T> pNodes, int vNodeCount) {
+ this(pNodes,vNodeCount, new MD5Hash());
+ }
+
+ /**
+ *
+ * @param pNodes collections of physical nodes
+ * @param vNodeCount amounts of virtual nodes
+ * @param hashFunction hash Function to hash Node instances
+ */
+ public ConsistentHashRouter(Collection<T> pNodes, int vNodeCount, HashFunction hashFunction) {
+ if (hashFunction == null) {
+ throw new NullPointerException("Hash Function is null");
+ }
+ this.hashFunction = hashFunction;
+ if (pNodes != null) {
+ for (T pNode : pNodes) {
+ addNode(pNode, vNodeCount);
+ }
+ }
+ }
+
+ /**
+ * add physic node to the hash ring with some virtual nodes
+ * @param pNode physical node needs added to hash ring
+ * @param vNodeCount the number of virtual node of the physical node. Value should be greater than or equals to 0
+ */
+ public void addNode(T pNode, int vNodeCount) {
+ if (vNodeCount < 0) throw new IllegalArgumentException("illegal virtual node counts :" + vNodeCount);
+ int existingReplicas = getExistingReplicas(pNode);
+ for (int i = 0; i < vNodeCount; i++) {
+ VirtualNode<T> vNode = new VirtualNode<>(pNode, i + existingReplicas);
+ ring.put(hashFunction.hash(vNode.getKey()), vNode);
+ }
+ }
+
+ /**
+ * remove the physical node from the hash ring
+ * @param pNode
+ */
+ public void removeNode(T pNode) {
+ Iterator<Long> it = ring.keySet().iterator();
+ while (it.hasNext()) {
+ Long key = it.next();
+ VirtualNode<T> virtualNode = ring.get(key);
+ if (virtualNode.isVirtualNodeOf(pNode)) {
+ it.remove();
+ }
+ }
+ }
+
+ /**
+ * with a specified key, route the nearest Node instance in the current hash ring
+ * @param objectKey the object key to find a nearest Node
+ * @return
+ */
+ public T routeNode(String objectKey) {
+ if (ring.isEmpty()) {
+ return null;
+ }
+ Long hashVal = hashFunction.hash(objectKey);
+ SortedMap<Long,VirtualNode<T>> tailMap = ring.tailMap(hashVal);
+ Long nodeHashVal = !tailMap.isEmpty() ? tailMap.firstKey() : ring.firstKey();
+ return ring.get(nodeHashVal).getPhysicalNode();
+ }
+
+
+ public int getExistingReplicas(T pNode) {
+ int replicas = 0;
+ for (VirtualNode<T> vNode : ring.values()) {
+ if (vNode.isVirtualNodeOf(pNode)) {
+ replicas++;
+ }
+ }
+ return replicas;
+ }
+
+
+ //default hash function
+ private static class MD5Hash implements HashFunction {
+ MessageDigest instance;
+
+ public MD5Hash() {
+ try {
+ instance = MessageDigest.getInstance("MD5");
+ } catch (NoSuchAlgorithmException e) {
+ }
+ }
+
+ @Override
+ public long hash(String key) {
+ instance.reset();
+ instance.update(key.getBytes());
+ byte[] digest = instance.digest();
+
+ long h = 0;
+ for (int i = 0; i < 4; i++) {
+ h <<= 8;
+ h |= ((int) digest[i]) & 0xFF;
+ }
+ return h;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/adae1624/common/src/main/java/org/apache/rocketmq/common/consistenthash/HashFunction.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/consistenthash/HashFunction.java b/common/src/main/java/org/apache/rocketmq/common/consistenthash/HashFunction.java
new file mode 100644
index 0000000..58fd777
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/consistenthash/HashFunction.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.consistenthash;
+
+/**
+ * Hash String to long value
+ */
+public interface HashFunction {
+ long hash(String key);
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/adae1624/common/src/main/java/org/apache/rocketmq/common/consistenthash/Node.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/consistenthash/Node.java b/common/src/main/java/org/apache/rocketmq/common/consistenthash/Node.java
new file mode 100644
index 0000000..0ece210
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/consistenthash/Node.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.consistenthash;
+
+/**
+ * Represent a node which should be mapped to a hash ring
+ */
+public interface Node {
+ /**
+ *
+ * @return the key which will be used for hash mapping
+ */
+ String getKey();
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/adae1624/common/src/main/java/org/apache/rocketmq/common/consistenthash/VirtualNode.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/consistenthash/VirtualNode.java b/common/src/main/java/org/apache/rocketmq/common/consistenthash/VirtualNode.java
new file mode 100644
index 0000000..c8b72d9
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/consistenthash/VirtualNode.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.consistenthash;
+
+
+public class VirtualNode<T extends Node> implements Node {
+ final T physicalNode;
+ final int replicaIndex;
+
+ public VirtualNode(T physicalNode, int replicaIndex) {
+ this.replicaIndex = replicaIndex;
+ this.physicalNode = physicalNode;
+ }
+
+ @Override
+ public String getKey() {
+ return physicalNode.getKey() + "-" + replicaIndex;
+ }
+
+ public boolean isVirtualNodeOf(T pNode) {
+ return physicalNode.getKey().equals(pNode.getKey());
+ }
+
+ public T getPhysicalNode() {
+ return physicalNode;
+ }
+}
[41/50] [abbrv] incubator-rocketmq git commit: Polish github links
Posted by do...@apache.org.
Polish github links
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/0fe94717
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/0fe94717
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/0fe94717
Branch: refs/heads/release-4.1.0-incubating
Commit: 0fe947173a85d8931b1068805713e89dbba4125a
Parents: de4c948
Author: dongeforever <zh...@yeah.net>
Authored: Sat May 27 14:39:30 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 14:39:30 2017 +0800
----------------------------------------------------------------------
pom.xml | 3 ---
.../src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java | 1 -
2 files changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/0fe94717/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a77f21d..75dbf5b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,6 @@
<id>stevenschew</id>
<name>Wei Zhou</name>
<email>stevenschew@@apache.org</email>
- <url>https://github.com/stevenschew</url>
<roles>
<role>committer</role>
</roles>
@@ -111,7 +110,6 @@
<id>lollipop</id>
<name>Jixiang Jin</name>
<email>lollipop@apache.org</email>
- <url>https://github.com/lollipopjin</url>
<roles>
<role>committer</role>
</roles>
@@ -121,7 +119,6 @@
<id>lizhanhui</id>
<name>Zhanhui Li</name>
<email>lizhanhui@apache.org</email>
- <url>https://github.com/lizhanhui</url>
<roles>
<role>committer</role>
</roles>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/0fe94717/store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java b/store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java
index 2b36fe7..5484dce 100644
--- a/store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java
+++ b/store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java
@@ -105,7 +105,6 @@ public class StoreCheckpoint {
public long getMinTimestamp() {
long min = Math.min(this.physicMsgTimestamp, this.logicsMsgTimestamp);
- // fixed https://github.org/apache/rocketmqissues/467
min -= 1000 * 3;
if (min < 0)
min = 0;
[32/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-98]Fix risk
of unable to release putMessage Lock forever closes
apache/incubator-rocketmq#61
Posted by do...@apache.org.
[ROCKETMQ-98]Fix risk of unable to release putMessage Lock forever closes apache/incubator-rocketmq#61
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/b1fcf1b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/b1fcf1b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/b1fcf1b8
Branch: refs/heads/release-4.1.0-incubating
Commit: b1fcf1b83b659bd03bcebf651d9e88c294a89e07
Parents: 0adad6f
Author: Jaskey <li...@gmail.com>
Authored: Sat May 27 11:21:09 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 11:21:09 2017 +0800
----------------------------------------------------------------------
.../apache/rocketmq/common/BrokerConfig.java | 3 ++
.../org/apache/rocketmq/store/CommitLog.java | 40 ++++--------------
.../apache/rocketmq/store/PutMessageLock.java | 25 ++++++++++++
.../rocketmq/store/PutMessageReentrantLock.java | 37 +++++++++++++++++
.../rocketmq/store/PutMessageSpinLock.java | 43 ++++++++++++++++++++
.../store/config/MessageStoreConfig.java | 4 ++
6 files changed, 119 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/b1fcf1b8/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
index f0a73bd..5bce013 100644
--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
+++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
@@ -47,6 +47,9 @@ public class BrokerConfig {
private boolean autoCreateSubscriptionGroup = true;
private String messageStorePlugIn = "";
+ /**
+ * thread numbers for send message thread pool, since spin lock will be used by default since 4.0.x, the default value is 1.
+ */
private int sendMessageThreadPoolNums = 1; //16 + Runtime.getRuntime().availableProcessors() * 4;
private int pullMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2;
private int adminBrokerThreadPoolNums = 16;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/b1fcf1b8/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
index 7841feb..7b29263 100644
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
@@ -23,8 +23,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantLock;
import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
@@ -63,12 +61,7 @@ public class CommitLog {
private volatile long confirmOffset = -1L;
private volatile long beginTimeInLock = 0;
-
- //true: Can lock, false : in lock.
- private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true);
-
- private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync
-
+ private final PutMessageLock putMessageLock;
public CommitLog(final DefaultMessageStore defaultMessageStore) {
this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService());
@@ -88,6 +81,8 @@ public class CommitLog {
return new MessageExtBatchEncoder(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
}
};
+ this.putMessageLock = defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();
+
}
public boolean load() {
@@ -577,7 +572,7 @@ public class CommitLog {
MappedFile unlockMappedFile = null;
MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
- lockForPutMessage(); //spin...
+ putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
try {
long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
this.beginTimeInLock = beginLockTimestamp;
@@ -626,7 +621,7 @@ public class CommitLog {
eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
beginTimeInLock = 0;
} finally {
- releasePutMessageLock();
+ putMessageLock.unlock();
}
if (eclipseTimeInLock > 500) {
@@ -861,7 +856,7 @@ public class CommitLog {
}
public boolean appendData(long startOffset, byte[] data) {
- lockForPutMessage(); //spin...
+ putMessageLock.lock();
try {
MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(startOffset);
if (null == mappedFile) {
@@ -871,7 +866,7 @@ public class CommitLog {
return mappedFile.appendMessage(data);
} finally {
- releasePutMessageLock();
+ putMessageLock.unlock();
}
}
@@ -906,28 +901,7 @@ public class CommitLog {
return diff;
}
- /**
- * Spin util acquired the lock.
- */
- private void lockForPutMessage() {
- if (this.defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage()) {
- putMessageNormalLock.lock();
- } else {
- boolean flag;
- do {
- flag = this.putMessageSpinLock.compareAndSet(true, false);
- }
- while (!flag);
- }
- }
- private void releasePutMessageLock() {
- if (this.defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage()) {
- putMessageNormalLock.unlock();
- } else {
- this.putMessageSpinLock.compareAndSet(false, true);
- }
- }
public static class GroupCommitRequest {
private final long nextOffset;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/b1fcf1b8/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java
new file mode 100644
index 0000000..a03e41a
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store;
+
+/**
+ * Used when trying to put message
+ */
+public interface PutMessageLock {
+ void lock();
+ void unlock();
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/b1fcf1b8/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java
new file mode 100644
index 0000000..9198f1c
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store;
+
+
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Exclusive lock implementation to put message
+ */
+public class PutMessageReentrantLock implements PutMessageLock {
+ private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync
+
+ @Override
+ public void lock() {
+ putMessageNormalLock.lock();
+ }
+
+ @Override
+ public void unlock() {
+ putMessageNormalLock.unlock();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/b1fcf1b8/store/src/main/java/org/apache/rocketmq/store/PutMessageSpinLock.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/PutMessageSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/PutMessageSpinLock.java
new file mode 100644
index 0000000..baa809d
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/PutMessageSpinLock.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store;
+
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Spin lock Implementation to put message, suggest using this witb low race conditions
+ *
+ */
+public class PutMessageSpinLock implements PutMessageLock {
+ //true: Can lock, false : in lock.
+ private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true);
+
+ @Override
+ public void lock() {
+ boolean flag;
+ do {
+ flag = this.putMessageSpinLock.compareAndSet(true, false);
+ }
+ while (!flag);
+ }
+
+ @Override
+ public void unlock() {
+ this.putMessageSpinLock.compareAndSet(false, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/b1fcf1b8/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
index 29f800c..19ed211 100644
--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
+++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
@@ -52,6 +52,10 @@ public class MessageStoreConfig {
@ImportantField
private int commitIntervalCommitLog = 200;
+ /**
+ * introduced since 4.0.x. Determine whether to use mutex reentrantLock when putting message.<br/>
+ * By default it is set to false indicating using spin lock when putting message.
+ */
private boolean useReentrantLockWhenPutMessage = false;
// Whether schedule flush,default is real-time
[04/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-168] Polish
the BUILDING guide.
Posted by do...@apache.org.
[ROCKETMQ-168] Polish the BUILDING guide.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/42f78c28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/42f78c28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/42f78c28
Branch: refs/heads/release-4.1.0-incubating
Commit: 42f78c281cbeb5072b04eaf03b1a8059b8d281a7
Parents: deb0820
Author: yukon <yu...@apache.org>
Authored: Thu Apr 20 14:11:37 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Thu Apr 20 14:11:37 2017 +0800
----------------------------------------------------------------------
BUILDING | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/42f78c28/BUILDING
----------------------------------------------------------------------
diff --git a/BUILDING b/BUILDING
index 8a30495..c6e23f5 100644
--- a/BUILDING
+++ b/BUILDING
@@ -34,4 +34,4 @@ Build Instructions for Apache RocketMQ
Execute the following command in order to build the tar.gz packages and install JAR into local repository:
- $ mvn -Prelease-all -DskipTests clean package install -U
\ No newline at end of file
+ $ mvn -Prelease-all -DskipTests clean install -U
\ No newline at end of file
[08/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-121]Support
message filtering based on SQL92 closes apache/incubator-rocketmq#82
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/Expression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/Expression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/Expression.java
new file mode 100644
index 0000000..3e6d9b3
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/Expression.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+/**
+ * Interface of expression.
+ * <p>
+ * This class was taken from ActiveMQ org.apache.activemq.filter.Expression,
+ * but the parameter is changed to an interface.
+ * </p>
+ *
+ * @see org.apache.rocketmq.filter.expression.EvaluationContext
+ */
+public interface Expression {
+
+ /**
+ * Calculate express result with context.
+ *
+ * @param context context of evaluation
+ * @return the value of this expression
+ */
+ Object evaluate(EvaluationContext context) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/LogicExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/LogicExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/LogicExpression.java
new file mode 100644
index 0000000..1062bb8
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/LogicExpression.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+/**
+ * A filter performing a comparison of two objects
+ * <p>
+ * This class was taken from ActiveMQ org.apache.activemq.filter.LogicExpression,
+ * </p>
+ */
+public abstract class LogicExpression extends BinaryExpression implements BooleanExpression {
+
+ /**
+ * @param left
+ * @param right
+ */
+ public LogicExpression(BooleanExpression left, BooleanExpression right) {
+ super(left, right);
+ }
+
+ public static BooleanExpression createOR(BooleanExpression lvalue, BooleanExpression rvalue) {
+ return new LogicExpression(lvalue, rvalue) {
+
+ public Object evaluate(EvaluationContext context) throws Exception {
+
+ Boolean lv = (Boolean) left.evaluate(context);
+ if (lv != null && lv.booleanValue()) {
+ return Boolean.TRUE;
+ }
+ Boolean rv = (Boolean) right.evaluate(context);
+ if (rv != null && rv.booleanValue()) {
+ return Boolean.TRUE;
+ }
+ if (lv == null || rv == null) {
+ return null;
+ }
+ return Boolean.FALSE;
+ }
+
+ public String getExpressionSymbol() {
+ return "||";
+ }
+ };
+ }
+
+ public static BooleanExpression createAND(BooleanExpression lvalue, BooleanExpression rvalue) {
+ return new LogicExpression(lvalue, rvalue) {
+
+ public Object evaluate(EvaluationContext context) throws Exception {
+
+ Boolean lv = (Boolean) left.evaluate(context);
+
+ if (lv != null && !lv.booleanValue()) {
+ return Boolean.FALSE;
+ }
+ Boolean rv = (Boolean) right.evaluate(context);
+ if (rv != null && !rv.booleanValue()) {
+ return Boolean.FALSE;
+ }
+ if (lv == null || rv == null) {
+ return null;
+ }
+ return Boolean.TRUE;
+ }
+
+ public String getExpressionSymbol() {
+ return "&&";
+ }
+ };
+ }
+
+ public abstract Object evaluate(EvaluationContext context) throws Exception;
+
+ public boolean matches(EvaluationContext context) throws Exception {
+ Object object = evaluate(context);
+ return object != null && object == Boolean.TRUE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/MQFilterException.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/MQFilterException.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/MQFilterException.java
new file mode 100644
index 0000000..676a17b
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/MQFilterException.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+/**
+ * Exception.
+ */
+public class MQFilterException extends Exception {
+ private static final long serialVersionUID = 1L;
+ private final int responseCode;
+ private final String errorMessage;
+
+ public MQFilterException(String errorMessage, Throwable cause) {
+ super(cause);
+ this.responseCode = -1;
+ this.errorMessage = errorMessage;
+ }
+
+ public MQFilterException(int responseCode, String errorMessage) {
+ this.responseCode = responseCode;
+ this.errorMessage = errorMessage;
+ }
+
+ public int getResponseCode() {
+ return responseCode;
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/NowExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/NowExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/NowExpression.java
new file mode 100644
index 0000000..d76caca
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/NowExpression.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+/**
+ * Current time expression.Just for test.
+ */
+public class NowExpression extends ConstantExpression {
+ public NowExpression() {
+ super("now");
+ }
+
+ @Override
+ public Object evaluate(EvaluationContext context) throws Exception {
+ return new Long(System.currentTimeMillis());
+ }
+
+ public Object getValue() {
+ return new Long(System.currentTimeMillis());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/PropertyExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/PropertyExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/PropertyExpression.java
new file mode 100644
index 0000000..b9657b0
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/PropertyExpression.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+/**
+ * Represents a property expression
+ * <p>
+ * This class was taken from ActiveMQ org.apache.activemq.filter.PropertyExpression,
+ * but more simple and no transfer between expression and message property.
+ * </p>
+ */
+public class PropertyExpression implements Expression {
+ private final String name;
+
+ public PropertyExpression(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public Object evaluate(EvaluationContext context) throws Exception {
+ return context.get(name);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @see Object#toString()
+ */
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ /**
+ * @see Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ /**
+ * @see Object#equals(Object)
+ */
+ @Override
+ public boolean equals(Object o) {
+
+ if (o == null || !this.getClass().equals(o.getClass())) {
+ return false;
+ }
+ return name.equals(((PropertyExpression) o).name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryExpression.java
new file mode 100644
index 0000000..0519f4d
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryExpression.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+import org.apache.rocketmq.filter.constant.UnaryType;
+
+import java.math.BigDecimal;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * An expression which performs an operation on two expression values
+ * <p>
+ * This class was taken from ActiveMQ org.apache.activemq.filter.UnaryExpression,
+ * but:
+ * 1. remove XPath and XQuery expression;
+ * 2. Add constant UnaryType to distinguish different unary expression;
+ * 3. Extract UnaryInExpression to an independent class.
+ * </p>
+ */
+public abstract class UnaryExpression implements Expression {
+
+ private static final BigDecimal BD_LONG_MIN_VALUE = BigDecimal.valueOf(Long.MIN_VALUE);
+ protected Expression right;
+
+ public UnaryType unaryType;
+
+ public UnaryExpression(Expression left) {
+ this.right = left;
+ }
+
+ public UnaryExpression(Expression left, UnaryType unaryType) {
+ this.setUnaryType(unaryType);
+ this.right = left;
+ }
+
+ public static Expression createNegate(Expression left) {
+ return new UnaryExpression(left, UnaryType.NEGATE) {
+ public Object evaluate(EvaluationContext context) throws Exception {
+ Object rvalue = right.evaluate(context);
+ if (rvalue == null) {
+ return null;
+ }
+ if (rvalue instanceof Number) {
+ return negate((Number) rvalue);
+ }
+ return null;
+ }
+
+ public String getExpressionSymbol() {
+ return "-";
+ }
+ };
+ }
+
+ public static BooleanExpression createInExpression(PropertyExpression right, List<Object> elements,
+ final boolean not) {
+
+ // Use a HashSet if there are many elements.
+ Collection<Object> t;
+ if (elements.size() == 0) {
+ t = null;
+ } else if (elements.size() < 5) {
+ t = elements;
+ } else {
+ t = new HashSet<Object>(elements);
+ }
+ final Collection inList = t;
+
+ return new UnaryInExpression(right, UnaryType.IN, inList, not) {
+ public Object evaluate(EvaluationContext context) throws Exception {
+
+ Object rvalue = right.evaluate(context);
+ if (rvalue == null) {
+ return null;
+ }
+ if (rvalue.getClass() != String.class) {
+ return null;
+ }
+
+ if ((inList != null && inList.contains(rvalue)) ^ not) {
+ return Boolean.TRUE;
+ } else {
+ return Boolean.FALSE;
+ }
+
+ }
+
+ public String toString() {
+ StringBuffer answer = new StringBuffer();
+ answer.append(right);
+ answer.append(" ");
+ answer.append(getExpressionSymbol());
+ answer.append(" ( ");
+
+ int count = 0;
+ for (Iterator i = inList.iterator(); i.hasNext(); ) {
+ Object o = (Object) i.next();
+ if (count != 0) {
+ answer.append(", ");
+ }
+ answer.append(o);
+ count++;
+ }
+
+ answer.append(" )");
+ return answer.toString();
+ }
+
+ public String getExpressionSymbol() {
+ if (not) {
+ return "NOT IN";
+ } else {
+ return "IN";
+ }
+ }
+ };
+ }
+
+ abstract static class BooleanUnaryExpression extends UnaryExpression implements BooleanExpression {
+ public BooleanUnaryExpression(Expression left, UnaryType unaryType) {
+ super(left, unaryType);
+ }
+
+ public boolean matches(EvaluationContext context) throws Exception {
+ Object object = evaluate(context);
+ return object != null && object == Boolean.TRUE;
+ }
+ }
+
+ public static BooleanExpression createNOT(BooleanExpression left) {
+ return new BooleanUnaryExpression(left, UnaryType.NOT) {
+ public Object evaluate(EvaluationContext context) throws Exception {
+ Boolean lvalue = (Boolean) right.evaluate(context);
+ if (lvalue == null) {
+ return null;
+ }
+ return lvalue.booleanValue() ? Boolean.FALSE : Boolean.TRUE;
+ }
+
+ public String getExpressionSymbol() {
+ return "NOT";
+ }
+ };
+ }
+
+ public static BooleanExpression createBooleanCast(Expression left) {
+ return new BooleanUnaryExpression(left, UnaryType.BOOLEANCAST) {
+ public Object evaluate(EvaluationContext context) throws Exception {
+ Object rvalue = right.evaluate(context);
+ if (rvalue == null) {
+ return null;
+ }
+ if (!rvalue.getClass().equals(Boolean.class)) {
+ return Boolean.FALSE;
+ }
+ return ((Boolean) rvalue).booleanValue() ? Boolean.TRUE : Boolean.FALSE;
+ }
+
+ public String toString() {
+ return right.toString();
+ }
+
+ public String getExpressionSymbol() {
+ return "";
+ }
+ };
+ }
+
+ private static Number negate(Number left) {
+ Class clazz = left.getClass();
+ if (clazz == Integer.class) {
+ return new Integer(-left.intValue());
+ } else if (clazz == Long.class) {
+ return new Long(-left.longValue());
+ } else if (clazz == Float.class) {
+ return new Float(-left.floatValue());
+ } else if (clazz == Double.class) {
+ return new Double(-left.doubleValue());
+ } else if (clazz == BigDecimal.class) {
+ // We ussually get a big deciamal when we have Long.MIN_VALUE
+ // constant in the
+ // Selector. Long.MIN_VALUE is too big to store in a Long as a
+ // positive so we store it
+ // as a Big decimal. But it gets Negated right away.. to here we try
+ // to covert it back
+ // to a Long.
+ BigDecimal bd = (BigDecimal) left;
+ bd = bd.negate();
+
+ if (BD_LONG_MIN_VALUE.compareTo(bd) == 0) {
+ return Long.valueOf(Long.MIN_VALUE);
+ }
+ return bd;
+ } else {
+ throw new RuntimeException("Don't know how to negate: " + left);
+ }
+ }
+
+ public Expression getRight() {
+ return right;
+ }
+
+ public void setRight(Expression expression) {
+ right = expression;
+ }
+
+ public UnaryType getUnaryType() {
+ return unaryType;
+ }
+
+ public void setUnaryType(UnaryType unaryType) {
+ this.unaryType = unaryType;
+ }
+
+ /**
+ * @see Object#toString()
+ */
+ public String toString() {
+ return "(" + getExpressionSymbol() + " " + right.toString() + ")";
+ }
+
+ /**
+ * @see Object#hashCode()
+ */
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ /**
+ * @see Object#equals(Object)
+ */
+ public boolean equals(Object o) {
+
+ if (o == null || !this.getClass().equals(o.getClass())) {
+ return false;
+ }
+ return toString().equals(o.toString());
+
+ }
+
+ /**
+ * Returns the symbol that represents this binary expression. For example,
+ * addition is represented by "+"
+ *
+ * @return
+ */
+ public abstract String getExpressionSymbol();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java
new file mode 100644
index 0000000..7d9083c
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+import org.apache.rocketmq.filter.constant.UnaryType;
+
+import java.util.Collection;
+
+/**
+ * In expression.
+ */
+abstract public class UnaryInExpression extends UnaryExpression implements BooleanExpression {
+
+ private boolean not;
+
+ private Collection inList;
+
+ public UnaryInExpression(Expression left, UnaryType unaryType,
+ Collection inList, boolean not) {
+ super(left, unaryType);
+ this.setInList(inList);
+ this.setNot(not);
+
+ }
+
+ public boolean matches(EvaluationContext context) throws Exception {
+ Object object = evaluate(context);
+ return object != null && object == Boolean.TRUE;
+ }
+
+ public boolean isNot() {
+ return not;
+ }
+
+ public void setNot(boolean not) {
+ this.not = not;
+ }
+
+ public Collection getInList() {
+ return inList;
+ }
+
+ public void setInList(Collection inList) {
+ this.inList = inList;
+ }
+};
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/parser/ParseException.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/ParseException.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/ParseException.java
new file mode 100644
index 0000000..2ccccaf
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/ParseException.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 5.0 */
+/* JavaCCOptions:KEEP_LINE_COL=null */
+package org.apache.rocketmq.filter.parser;
+
+/**
+ * This exception is thrown when parse errors are encountered.
+ * You can explicitly create objects of this exception type by
+ * calling the method generateParseException in the generated
+ * parser.
+ * <p/>
+ * You can modify this class to customize your error reporting
+ * mechanisms so long as you retain the public fields.
+ */
+public class ParseException extends Exception {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * This constructor is used by the method "generateParseException"
+ * in the generated parser. Calling this constructor generates
+ * a new object of this type with the fields "currentToken",
+ * "expectedTokenSequences", and "TOKEN_IMAGE" set.
+ */
+ public ParseException(Token currentTokenVal,
+ int[][] expectedTokenSequencesVal,
+ String[] tokenImageVal
+ ) {
+ super(initialise(currentTokenVal, expectedTokenSequencesVal, tokenImageVal));
+ currentToken = currentTokenVal;
+ expectedTokenSequences = expectedTokenSequencesVal;
+ tokenImage = tokenImageVal;
+ }
+
+ /**
+ * The following constructors are for use by you for whatever
+ * purpose you can think of. Constructing the exception in this
+ * manner makes the exception behave in the normal way - i.e., as
+ * documented in the class "Throwable". The fields "errorToken",
+ * "expectedTokenSequences", and "TOKEN_IMAGE" do not contain
+ * relevant information. The JavaCC generated code does not use
+ * these constructors.
+ */
+
+ public ParseException() {
+ super();
+ }
+
+ /**
+ * Constructor with message.
+ */
+ public ParseException(String message) {
+ super(message);
+ }
+
+ /**
+ * This is the last token that has been consumed successfully. If
+ * this object has been created due to a parse error, the token
+ * followng this token will (therefore) be the first error token.
+ */
+ public Token currentToken;
+
+ /**
+ * Each entry in this array is an array of integers. Each array
+ * of integers represents a sequence of tokens (by their ordinal
+ * values) that is expected at this point of the parse.
+ */
+ public int[][] expectedTokenSequences;
+
+ /**
+ * This is a reference to the "TOKEN_IMAGE" array of the generated
+ * parser within which the parse error occurred. This array is
+ * defined in the generated ...Constants interface.
+ */
+ public String[] tokenImage;
+
+ /**
+ * It uses "currentToken" and "expectedTokenSequences" to generate a parse
+ * error message and returns it. If this object has been created
+ * due to a parse error, and you do not catch it (it gets thrown
+ * from the parser) the correct error message
+ * gets displayed.
+ */
+ private static String initialise(Token currentToken,
+ int[][] expectedTokenSequences,
+ String[] tokenImage) {
+ String eol = System.getProperty("line.separator", "\n");
+ StringBuffer expected = new StringBuffer();
+ int maxSize = 0;
+ for (int i = 0; i < expectedTokenSequences.length; i++) {
+ if (maxSize < expectedTokenSequences[i].length) {
+ maxSize = expectedTokenSequences[i].length;
+ }
+ for (int j = 0; j < expectedTokenSequences[i].length; j++) {
+ expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');
+ }
+ if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
+ expected.append("...");
+ }
+ expected.append(eol).append(" ");
+ }
+ String retval = "Encountered \"";
+ Token tok = currentToken.next;
+ for (int i = 0; i < maxSize; i++) {
+ if (i != 0)
+ retval += " ";
+ if (tok.kind == 0) {
+ retval += tokenImage[0];
+ break;
+ }
+ retval += " " + tokenImage[tok.kind];
+ retval += " \"";
+ retval += add_escapes(tok.image);
+ retval += " \"";
+ tok = tok.next;
+ }
+ retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
+ retval += "." + eol;
+ if (expectedTokenSequences.length == 1) {
+ retval += "Was expecting:" + eol + " ";
+ } else {
+ retval += "Was expecting one of:" + eol + " ";
+ }
+ retval += expected.toString();
+ return retval;
+ }
+
+ /**
+ * The end of line string for this machine.
+ */
+ protected String eol = System.getProperty("line.separator", "\n");
+
+ /**
+ * Used to convert raw characters to their escaped version
+ * when these raw version cannot be used as part of an ASCII
+ * string literal.
+ */
+ static String add_escapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for (int i = 0; i < str.length(); i++) {
+ switch (str.charAt(i)) {
+ case 0:
+ continue;
+ case '\b':
+ retval.append("\\b");
+ continue;
+ case '\t':
+ retval.append("\\t");
+ continue;
+ case '\n':
+ retval.append("\\n");
+ continue;
+ case '\f':
+ retval.append("\\f");
+ continue;
+ case '\r':
+ retval.append("\\r");
+ continue;
+ case '\"':
+ retval.append("\\\"");
+ continue;
+ case '\'':
+ retval.append("\\\'");
+ continue;
+ case '\\':
+ retval.append("\\\\");
+ continue;
+ default:
+ if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ } else {
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+
+}
+/* JavaCC - OriginalChecksum=4c829b0daa2c9af00ddafe2441eb9097 (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.java
new file mode 100644
index 0000000..74e5501
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.java
@@ -0,0 +1,1354 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. SelectorParser.java */
+package org.apache.rocketmq.filter.parser;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.rocketmq.filter.expression.BooleanExpression;
+import org.apache.rocketmq.filter.expression.ComparisonExpression;
+import org.apache.rocketmq.filter.expression.ConstantExpression;
+import org.apache.rocketmq.filter.expression.Expression;
+import org.apache.rocketmq.filter.expression.LogicExpression;
+import org.apache.rocketmq.filter.expression.MQFilterException;
+import org.apache.rocketmq.filter.expression.PropertyExpression;
+import org.apache.rocketmq.filter.expression.UnaryExpression;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+
+/**
+ * JMS Selector Parser generated by JavaCC
+ * <p/>
+ * Do not edit this .java file directly - it is autogenerated from SelectorParser.jj
+ */
+public class SelectorParser implements SelectorParserConstants {
+
+ private static final Cache<String, Object> PARSE_CACHE = CacheBuilder.newBuilder().maximumSize(100).build();
+ // private static final String CONVERT_STRING_EXPRESSIONS_PREFIX = "convert_string_expressions:";
+
+ public static BooleanExpression parse(String sql) throws MQFilterException {
+ // sql = "("+sql+")";
+ Object result = PARSE_CACHE.getIfPresent(sql);
+ if (result instanceof MQFilterException) {
+ throw (MQFilterException) result;
+ } else if (result instanceof BooleanExpression) {
+ return (BooleanExpression) result;
+ } else {
+
+ // boolean convertStringExpressions = false;
+ // if( sql.startsWith(CONVERT_STRING_EXPRESSIONS_PREFIX)) {
+ // convertStringExpressions = true;
+ // sql = sql.substring(CONVERT_STRING_EXPRESSIONS_PREFIX.length());
+ // }
+ //
+ // if( convertStringExpressions ) {
+ // ComparisonExpression.CONVERT_STRING_EXPRESSIONS.set(true);
+ // }
+ ComparisonExpression.CONVERT_STRING_EXPRESSIONS.set(true);
+ try {
+
+ BooleanExpression e = new SelectorParser(sql).parse();
+ PARSE_CACHE.put(sql, e);
+ return e;
+ } catch (MQFilterException t) {
+ PARSE_CACHE.put(sql, t);
+ throw t;
+ } finally {
+ ComparisonExpression.CONVERT_STRING_EXPRESSIONS.remove();
+ // if( convertStringExpressions ) {
+ // ComparisonExpression.CONVERT_STRING_EXPRESSIONS.remove();
+ // }
+ }
+ }
+ }
+
+ public static void clearCache() {
+ PARSE_CACHE.cleanUp();
+ }
+
+ private String sql;
+
+ protected SelectorParser(String sql) {
+ this(new StringReader(sql));
+ this.sql = sql;
+ }
+
+ protected BooleanExpression parse() throws MQFilterException {
+ try {
+ return this.JmsSelector();
+ } catch (Throwable e) {
+ throw new MQFilterException("Invalid MessageSelector. ", e);
+ }
+ }
+
+ private BooleanExpression asBooleanExpression(Expression value) throws ParseException {
+ if (value instanceof BooleanExpression) {
+ return (BooleanExpression) value;
+ }
+ if (value instanceof PropertyExpression) {
+ return UnaryExpression.createBooleanCast(value);
+ }
+ throw new ParseException("Expression will not result in a boolean value: " + value);
+ }
+
+ // ----------------------------------------------------------------------------
+ // Grammer
+ // ----------------------------------------------------------------------------
+ final public BooleanExpression JmsSelector() throws ParseException {
+ Expression left = null;
+ left = orExpression();
+ {
+ if (true)
+ return asBooleanExpression(left);
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Expression orExpression() throws ParseException {
+ Expression left;
+ Expression right;
+ left = andExpression();
+ label_1:
+ while (true) {
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case OR:
+ break;
+ default:
+ jjLa1[0] = jjGen;
+ break label_1;
+ }
+ jj_consume_token(OR);
+ right = andExpression();
+ left = LogicExpression.createOR(asBooleanExpression(left), asBooleanExpression(right));
+ }
+ {
+ if (true)
+ return left;
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Expression andExpression() throws ParseException {
+ Expression left;
+ Expression right;
+ left = equalityExpression();
+ label_2:
+ while (true) {
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case AND:
+ break;
+ default:
+ jjLa1[1] = jjGen;
+ break label_2;
+ }
+ jj_consume_token(AND);
+ right = equalityExpression();
+ left = LogicExpression.createAND(asBooleanExpression(left), asBooleanExpression(right));
+ }
+ {
+ if (true)
+ return left;
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Expression equalityExpression() throws ParseException {
+ Expression left;
+ Expression right;
+ left = comparisonExpression();
+ label_3:
+ while (true) {
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case IS:
+ case 22:
+ case 23:
+ break;
+ default:
+ jjLa1[2] = jjGen;
+ break label_3;
+ }
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case 22:
+ jj_consume_token(22);
+ right = comparisonExpression();
+ left = ComparisonExpression.createEqual(left, right);
+ break;
+ case 23:
+ jj_consume_token(23);
+ right = comparisonExpression();
+ left = ComparisonExpression.createNotEqual(left, right);
+ break;
+ default:
+ jjLa1[3] = jjGen;
+ if (jj_2_1(2)) {
+ jj_consume_token(IS);
+ jj_consume_token(NULL);
+ left = ComparisonExpression.createIsNull(left);
+ } else {
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case IS:
+ jj_consume_token(IS);
+ jj_consume_token(NOT);
+ jj_consume_token(NULL);
+ left = ComparisonExpression.createIsNotNull(left);
+ break;
+ default:
+ jjLa1[4] = jjGen;
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ }
+ }
+ }
+ {
+ if (true)
+ return left;
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Expression comparisonExpression() throws ParseException {
+ Expression left;
+ Expression right;
+ Expression low;
+ Expression high;
+ String t, u;
+ boolean not;
+ ArrayList list;
+ left = unaryExpr();
+ label_4:
+ while (true) {
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case NOT:
+ case BETWEEN:
+ case IN:
+ case 24:
+ case 25:
+ case 26:
+ case 27:
+ break;
+ default:
+ jjLa1[5] = jjGen;
+ break label_4;
+ }
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case 24:
+ jj_consume_token(24);
+ right = unaryExpr();
+ left = ComparisonExpression.createGreaterThan(left, right);
+ break;
+ case 25:
+ jj_consume_token(25);
+ right = unaryExpr();
+ left = ComparisonExpression.createGreaterThanEqual(left, right);
+ break;
+ case 26:
+ jj_consume_token(26);
+ right = unaryExpr();
+ left = ComparisonExpression.createLessThan(left, right);
+ break;
+ case 27:
+ jj_consume_token(27);
+ right = unaryExpr();
+ left = ComparisonExpression.createLessThanEqual(left, right);
+ break;
+ case BETWEEN:
+ jj_consume_token(BETWEEN);
+ low = unaryExpr();
+ jj_consume_token(AND);
+ high = unaryExpr();
+ left = ComparisonExpression.createBetween(left, low, high);
+ break;
+ default:
+ jjLa1[8] = jjGen;
+ if (jj_2_2(2)) {
+ jj_consume_token(NOT);
+ jj_consume_token(BETWEEN);
+ low = unaryExpr();
+ jj_consume_token(AND);
+ high = unaryExpr();
+ left = ComparisonExpression.createNotBetween(left, low, high);
+ } else {
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case IN:
+ jj_consume_token(IN);
+ jj_consume_token(28);
+ t = stringLitteral();
+ list = new ArrayList();
+ list.add(t);
+ label_5:
+ while (true) {
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case 29:
+ break;
+ default:
+ jjLa1[6] = jjGen;
+ break label_5;
+ }
+ jj_consume_token(29);
+ t = stringLitteral();
+ list.add(t);
+ }
+ jj_consume_token(30);
+ left = ComparisonExpression.createInFilter(left, list);
+ break;
+ default:
+ jjLa1[9] = jjGen;
+ if (jj_2_3(2)) {
+ jj_consume_token(NOT);
+ jj_consume_token(IN);
+ jj_consume_token(28);
+ t = stringLitteral();
+ list = new ArrayList();
+ list.add(t);
+ label_6:
+ while (true) {
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case 29:
+ break;
+ default:
+ jjLa1[7] = jjGen;
+ break label_6;
+ }
+ jj_consume_token(29);
+ t = stringLitteral();
+ list.add(t);
+ }
+ jj_consume_token(30);
+ left = ComparisonExpression.createNotInFilter(left, list);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ }
+ }
+ }
+ }
+ {
+ if (true)
+ return left;
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Expression unaryExpr() throws ParseException {
+ String s = null;
+ Expression left = null;
+ if (jj_2_4(2147483647)) {
+ jj_consume_token(31);
+ left = unaryExpr();
+ } else {
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case 32:
+ jj_consume_token(32);
+ left = unaryExpr();
+ left = UnaryExpression.createNegate(left);
+ break;
+ case NOT:
+ jj_consume_token(NOT);
+ left = unaryExpr();
+ left = UnaryExpression.createNOT(asBooleanExpression(left));
+ break;
+ case TRUE:
+ case FALSE:
+ case NULL:
+ case DECIMAL_LITERAL:
+ case FLOATING_POINT_LITERAL:
+ case STRING_LITERAL:
+ case ID:
+ case 28:
+ left = primaryExpr();
+ break;
+ default:
+ jjLa1[10] = jjGen;
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ }
+ {
+ if (true)
+ return left;
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Expression primaryExpr() throws ParseException {
+ Expression left = null;
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case TRUE:
+ case FALSE:
+ case NULL:
+ case DECIMAL_LITERAL:
+ case FLOATING_POINT_LITERAL:
+ case STRING_LITERAL:
+ left = literal();
+ break;
+ case ID:
+ left = variable();
+ break;
+ case 28:
+ jj_consume_token(28);
+ left = orExpression();
+ jj_consume_token(30);
+ break;
+ default:
+ jjLa1[11] = jjGen;
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ {
+ if (true)
+ return left;
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public ConstantExpression literal() throws ParseException {
+ Token t;
+ String s;
+ ConstantExpression left = null;
+ switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {
+ case STRING_LITERAL:
+ s = stringLitteral();
+ left = new ConstantExpression(s);
+ break;
+ case DECIMAL_LITERAL:
+ t = jj_consume_token(DECIMAL_LITERAL);
+ left = ConstantExpression.createFromDecimal(t.image);
+ break;
+ case FLOATING_POINT_LITERAL:
+ t = jj_consume_token(FLOATING_POINT_LITERAL);
+ left = ConstantExpression.createFloat(t.image);
+ break;
+ case TRUE:
+ jj_consume_token(TRUE);
+ left = ConstantExpression.TRUE;
+ break;
+ case FALSE:
+ jj_consume_token(FALSE);
+ left = ConstantExpression.FALSE;
+ break;
+ case NULL:
+ jj_consume_token(NULL);
+ left = ConstantExpression.NULL;
+ break;
+ default:
+ jjLa1[12] = jjGen;
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ {
+ if (true)
+ return left;
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public String stringLitteral() throws ParseException {
+ Token t;
+ StringBuffer rc = new StringBuffer();
+ boolean first = true;
+ t = jj_consume_token(STRING_LITERAL);
+ // Decode the sting value.
+ String image = t.image;
+ for (int i = 1; i < image.length() - 1; i++) {
+ char c = image.charAt(i);
+ if (c == '\'')
+ i++;
+ rc.append(c);
+ }
+ {
+ if (true)
+ return rc.toString();
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public PropertyExpression variable() throws ParseException {
+ Token t;
+ PropertyExpression left = null;
+ t = jj_consume_token(ID);
+ left = new PropertyExpression(t.image);
+ {
+ if (true)
+ return left;
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ private boolean jj_2_1(int xla) {
+ jjLa = xla;
+ jjLastpos = jjScanpos = token;
+ try {
+ return !jj_3_1();
+ } catch (LookaheadSuccess ls) {
+ return true;
+ } finally {
+ jj_save(0, xla);
+ }
+ }
+
+ private boolean jj_2_2(int xla) {
+ jjLa = xla;
+ jjLastpos = jjScanpos = token;
+ try {
+ return !jj_3_2();
+ } catch (LookaheadSuccess ls) {
+ return true;
+ } finally {
+ jj_save(1, xla);
+ }
+ }
+
+ private boolean jj_2_3(int xla) {
+ jjLa = xla;
+ jjLastpos = jjScanpos = token;
+ try {
+ return !jj_3_3();
+ } catch (LookaheadSuccess ls) {
+ return true;
+ } finally {
+ jj_save(2, xla);
+ }
+ }
+
+ private boolean jj_2_4(int xla) {
+ jjLa = xla;
+ jjLastpos = jjScanpos = token;
+ try {
+ return !jj_3_4();
+ } catch (LookaheadSuccess ls) {
+ return true;
+ } finally {
+ jj_save(3, xla);
+ }
+ }
+
+ private boolean jj_3R_7() {
+ Token xsp;
+ xsp = jjScanpos;
+ if (jj_3R_8()) {
+ jjScanpos = xsp;
+ if (jj_3R_9()) {
+ jjScanpos = xsp;
+ if (jj_3R_10()) {
+ jjScanpos = xsp;
+ if (jj_3R_11())
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3R_43() {
+ if (jj_scan_token(29))
+ return true;
+ if (jj_3R_27())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_24() {
+ if (jj_scan_token(NULL))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_35() {
+ if (jj_scan_token(IS))
+ return true;
+ if (jj_scan_token(NOT))
+ return true;
+ if (jj_scan_token(NULL))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3_1() {
+ if (jj_scan_token(IS))
+ return true;
+ if (jj_scan_token(NULL))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_23() {
+ if (jj_scan_token(FALSE))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_34() {
+ if (jj_scan_token(23))
+ return true;
+ if (jj_3R_30())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_22() {
+ if (jj_scan_token(TRUE))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3_3() {
+ if (jj_scan_token(NOT))
+ return true;
+ if (jj_scan_token(IN))
+ return true;
+ if (jj_scan_token(28))
+ return true;
+ if (jj_3R_27())
+ return true;
+ Token xsp;
+ while (true) {
+ xsp = jjScanpos;
+ if (jj_3R_43()) {
+ jjScanpos = xsp;
+ break;
+ }
+ }
+ if (jj_scan_token(30))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_31() {
+ Token xsp;
+ xsp = jjScanpos;
+ if (jj_3R_33()) {
+ jjScanpos = xsp;
+ if (jj_3R_34()) {
+ jjScanpos = xsp;
+ if (jj_3_1()) {
+ jjScanpos = xsp;
+ if (jj_3R_35())
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3R_33() {
+ if (jj_scan_token(22))
+ return true;
+ if (jj_3R_30())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_42() {
+ if (jj_scan_token(29))
+ return true;
+ if (jj_3R_27())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_21() {
+ if (jj_scan_token(FLOATING_POINT_LITERAL))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_20() {
+ if (jj_scan_token(DECIMAL_LITERAL))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_28() {
+ if (jj_3R_30())
+ return true;
+ Token xsp;
+ while (true) {
+ xsp = jjScanpos;
+ if (jj_3R_31()) {
+ jjScanpos = xsp;
+ break;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3R_41() {
+ if (jj_scan_token(IN))
+ return true;
+ if (jj_scan_token(28))
+ return true;
+ if (jj_3R_27())
+ return true;
+ Token xsp;
+ while (true) {
+ xsp = jjScanpos;
+ if (jj_3R_42()) {
+ jjScanpos = xsp;
+ break;
+ }
+ }
+ if (jj_scan_token(30))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_19() {
+ if (jj_3R_27())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_29() {
+ if (jj_scan_token(AND))
+ return true;
+ if (jj_3R_28())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_16() {
+ Token xsp;
+ xsp = jjScanpos;
+ if (jj_3R_19()) {
+ jjScanpos = xsp;
+ if (jj_3R_20()) {
+ jjScanpos = xsp;
+ if (jj_3R_21()) {
+ jjScanpos = xsp;
+ if (jj_3R_22()) {
+ jjScanpos = xsp;
+ if (jj_3R_23()) {
+ jjScanpos = xsp;
+ if (jj_3R_24())
+ return true;
+ }
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3_2() {
+ if (jj_scan_token(NOT))
+ return true;
+ if (jj_scan_token(BETWEEN))
+ return true;
+ if (jj_3R_7())
+ return true;
+ if (jj_scan_token(AND))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_40() {
+ if (jj_scan_token(BETWEEN))
+ return true;
+ if (jj_3R_7())
+ return true;
+ if (jj_scan_token(AND))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_25() {
+ if (jj_3R_28())
+ return true;
+ Token xsp;
+ while (true) {
+ xsp = jjScanpos;
+ if (jj_3R_29()) {
+ jjScanpos = xsp;
+ break;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3R_39() {
+ if (jj_scan_token(27))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_15() {
+ if (jj_scan_token(28))
+ return true;
+ if (jj_3R_18())
+ return true;
+ if (jj_scan_token(30))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_14() {
+ if (jj_3R_17())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_38() {
+ if (jj_scan_token(26))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_13() {
+ if (jj_3R_16())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_26() {
+ if (jj_scan_token(OR))
+ return true;
+ if (jj_3R_25())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_17() {
+ if (jj_scan_token(ID))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_37() {
+ if (jj_scan_token(25))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_12() {
+ Token xsp;
+ xsp = jjScanpos;
+ if (jj_3R_13()) {
+ jjScanpos = xsp;
+ if (jj_3R_14()) {
+ jjScanpos = xsp;
+ if (jj_3R_15())
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3R_32() {
+ Token xsp;
+ xsp = jjScanpos;
+ if (jj_3R_36()) {
+ jjScanpos = xsp;
+ if (jj_3R_37()) {
+ jjScanpos = xsp;
+ if (jj_3R_38()) {
+ jjScanpos = xsp;
+ if (jj_3R_39()) {
+ jjScanpos = xsp;
+ if (jj_3R_40()) {
+ jjScanpos = xsp;
+ if (jj_3_2()) {
+ jjScanpos = xsp;
+ if (jj_3R_41()) {
+ jjScanpos = xsp;
+ if (jj_3_3())
+ return true;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3R_36() {
+ if (jj_scan_token(24))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_11() {
+ if (jj_3R_12())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_18() {
+ if (jj_3R_25())
+ return true;
+ Token xsp;
+ while (true) {
+ xsp = jjScanpos;
+ if (jj_3R_26()) {
+ jjScanpos = xsp;
+ break;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3_4() {
+ if (jj_scan_token(31))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_10() {
+ if (jj_scan_token(NOT))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_9() {
+ if (jj_scan_token(32))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_27() {
+ if (jj_scan_token(STRING_LITERAL))
+ return true;
+ return false;
+ }
+
+ private boolean jj_3R_30() {
+ if (jj_3R_7())
+ return true;
+ Token xsp;
+ while (true) {
+ xsp = jjScanpos;
+ if (jj_3R_32()) {
+ jjScanpos = xsp;
+ break;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3R_8() {
+ if (jj_scan_token(31))
+ return true;
+ if (jj_3R_7())
+ return true;
+ return false;
+ }
+
+ /**
+ * Generated Token Manager.
+ */
+ public SelectorParserTokenManager tokenSource;
+ SimpleCharStream jjInputStream;
+ /**
+ * Current token.
+ */
+ public Token token;
+ /**
+ * Next token.
+ */
+ public Token jjNt;
+ private int jjNtk;
+ private Token jjScanpos, jjLastpos;
+ private int jjLa;
+ private int jjGen;
+ final private int[] jjLa1 = new int[13];
+ static private int[] jjLa10;
+ static private int[] jjLa11;
+
+ static {
+ jj_la1_init_0();
+ jj_la1_init_1();
+ }
+
+ private static void jj_la1_init_0() {
+ jjLa10 = new int[]{0x400, 0x200, 0xc10000, 0xc00000, 0x10000, 0xf001900, 0x20000000, 0x20000000, 0xf000800,
+ 0x1000, 0x1036e100, 0x1036e000, 0x16e000};
+ }
+
+ private static void jj_la1_init_1() {
+ jjLa11 = new int[]{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0};
+ }
+
+ final private JJCalls[] jj2Rtns = new JJCalls[4];
+ private boolean jjRescan = false;
+ private int jjGc = 0;
+
+ /**
+ * Constructor with InputStream.
+ */
+ public SelectorParser(java.io.InputStream stream) {
+ this(stream, null);
+ }
+
+ /**
+ * Constructor with InputStream and supplied encoding
+ */
+ public SelectorParser(java.io.InputStream stream, String encoding) {
+ try {
+ jjInputStream = new SimpleCharStream(stream, encoding, 1, 1);
+ } catch (java.io.UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ tokenSource = new SelectorParserTokenManager(jjInputStream);
+ token = new Token();
+ jjNtk = -1;
+ jjGen = 0;
+ for (int i = 0; i < 13; i++)
+ jjLa1[i] = -1;
+ for (int i = 0; i < jj2Rtns.length; i++)
+ jj2Rtns[i] = new JJCalls();
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.InputStream stream) {
+ ReInit(stream, null);
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.InputStream stream, String encoding) {
+ try {
+ jjInputStream.ReInit(stream, encoding, 1, 1);
+ } catch (java.io.UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ tokenSource.ReInit(jjInputStream);
+ token = new Token();
+ jjNtk = -1;
+ jjGen = 0;
+ for (int i = 0; i < 13; i++)
+ jjLa1[i] = -1;
+ for (int i = 0; i < jj2Rtns.length; i++)
+ jj2Rtns[i] = new JJCalls();
+ }
+
+ /**
+ * Constructor.
+ */
+ public SelectorParser(java.io.Reader stream) {
+ jjInputStream = new SimpleCharStream(stream, 1, 1);
+ tokenSource = new SelectorParserTokenManager(jjInputStream);
+ token = new Token();
+ jjNtk = -1;
+ jjGen = 0;
+ for (int i = 0; i < 13; i++)
+ jjLa1[i] = -1;
+ for (int i = 0; i < jj2Rtns.length; i++)
+ jj2Rtns[i] = new JJCalls();
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.Reader stream) {
+ jjInputStream.ReInit(stream, 1, 1);
+ tokenSource.ReInit(jjInputStream);
+ token = new Token();
+ jjNtk = -1;
+ jjGen = 0;
+ for (int i = 0; i < 13; i++)
+ jjLa1[i] = -1;
+ for (int i = 0; i < jj2Rtns.length; i++)
+ jj2Rtns[i] = new JJCalls();
+ }
+
+ /**
+ * Constructor with generated Token Manager.
+ */
+ public SelectorParser(SelectorParserTokenManager tm) {
+ tokenSource = tm;
+ token = new Token();
+ jjNtk = -1;
+ jjGen = 0;
+ for (int i = 0; i < 13; i++)
+ jjLa1[i] = -1;
+ for (int i = 0; i < jj2Rtns.length; i++)
+ jj2Rtns[i] = new JJCalls();
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(SelectorParserTokenManager tm) {
+ tokenSource = tm;
+ token = new Token();
+ jjNtk = -1;
+ jjGen = 0;
+ for (int i = 0; i < 13; i++)
+ jjLa1[i] = -1;
+ for (int i = 0; i < jj2Rtns.length; i++)
+ jj2Rtns[i] = new JJCalls();
+ }
+
+ private Token jj_consume_token(int kind) throws ParseException {
+ Token oldToken;
+ if ((oldToken = token).next != null)
+ token = token.next;
+ else
+ token = token.next = tokenSource.getNextToken();
+ jjNtk = -1;
+ if (token.kind == kind) {
+ jjGen++;
+ if (++jjGc > 100) {
+ jjGc = 0;
+ for (int i = 0; i < jj2Rtns.length; i++) {
+ JJCalls c = jj2Rtns[i];
+ while (c != null) {
+ if (c.gen < jjGen)
+ c.first = null;
+ c = c.next;
+ }
+ }
+ }
+ return token;
+ }
+ token = oldToken;
+ jjKind = kind;
+ throw generateParseException();
+ }
+
+ static private final class LookaheadSuccess extends java.lang.Error {
+ }
+
+ final private LookaheadSuccess jjLs = new LookaheadSuccess();
+
+ private boolean jj_scan_token(int kind) {
+ if (jjScanpos == jjLastpos) {
+ jjLa--;
+ if (jjScanpos.next == null) {
+ jjLastpos = jjScanpos = jjScanpos.next = tokenSource.getNextToken();
+ } else {
+ jjLastpos = jjScanpos = jjScanpos.next;
+ }
+ } else {
+ jjScanpos = jjScanpos.next;
+ }
+ if (jjRescan) {
+ int i = 0;
+ Token tok = token;
+ while (tok != null && tok != jjScanpos) {
+ i++;
+ tok = tok.next;
+ }
+ if (tok != null)
+ jj_add_error_token(kind, i);
+ }
+ if (jjScanpos.kind != kind)
+ return true;
+ if (jjLa == 0 && jjScanpos == jjLastpos)
+ throw jjLs;
+ return false;
+ }
+
+ /**
+ * Get the next Token.
+ */
+ final public Token getNextToken() {
+ if (token.next != null)
+ token = token.next;
+ else
+ token = token.next = tokenSource.getNextToken();
+ jjNtk = -1;
+ jjGen++;
+ return token;
+ }
+
+ /**
+ * Get the specific Token.
+ */
+ final public Token getToken(int index) {
+ Token t = token;
+ for (int i = 0; i < index; i++) {
+ if (t.next != null)
+ t = t.next;
+ else
+ t = t.next = tokenSource.getNextToken();
+ }
+ return t;
+ }
+
+ private int jj_ntk() {
+ if ((jjNt = token.next) == null)
+ return jjNtk = (token.next = tokenSource.getNextToken()).kind;
+ else
+ return jjNtk = jjNt.kind;
+ }
+
+ private java.util.List<int[]> jjExpentries = new java.util.ArrayList<int[]>();
+ private int[] jjExpentry;
+ private int jjKind = -1;
+ private int[] jjLasttokens = new int[100];
+ private int jjEndpos;
+
+ private void jj_add_error_token(int kind, int pos) {
+ if (pos >= 100)
+ return;
+ if (pos == jjEndpos + 1) {
+ jjLasttokens[jjEndpos++] = kind;
+ } else if (jjEndpos != 0) {
+ jjExpentry = new int[jjEndpos];
+ for (int i = 0; i < jjEndpos; i++) {
+ jjExpentry[i] = jjLasttokens[i];
+ }
+ jj_entries_loop:
+ for (java.util.Iterator<?> it = jjExpentries.iterator(); it.hasNext(); ) {
+ int[] oldentry = (int[]) (it.next());
+ if (oldentry.length == jjExpentry.length) {
+ for (int i = 0; i < jjExpentry.length; i++) {
+ if (oldentry[i] != jjExpentry[i]) {
+ continue jj_entries_loop;
+ }
+ }
+ jjExpentries.add(jjExpentry);
+ break jj_entries_loop;
+ }
+ }
+ if (pos != 0)
+ jjLasttokens[(jjEndpos = pos) - 1] = kind;
+ }
+ }
+
+ /**
+ * Generate ParseException.
+ */
+ public ParseException generateParseException() {
+ jjExpentries.clear();
+ boolean[] la1tokens = new boolean[33];
+ if (jjKind >= 0) {
+ la1tokens[jjKind] = true;
+ jjKind = -1;
+ }
+ for (int i = 0; i < 13; i++) {
+ if (jjLa1[i] == jjGen) {
+ for (int j = 0; j < 32; j++) {
+ if ((jjLa10[i] & (1 << j)) != 0) {
+ la1tokens[j] = true;
+ }
+ if ((jjLa11[i] & (1 << j)) != 0) {
+ la1tokens[32 + j] = true;
+ }
+ }
+ }
+ }
+ for (int i = 0; i < 33; i++) {
+ if (la1tokens[i]) {
+ jjExpentry = new int[1];
+ jjExpentry[0] = i;
+ jjExpentries.add(jjExpentry);
+ }
+ }
+ jjEndpos = 0;
+ jj_rescan_token();
+ jj_add_error_token(0, 0);
+ int[][] exptokseq = new int[jjExpentries.size()][];
+ for (int i = 0; i < jjExpentries.size(); i++) {
+ exptokseq[i] = jjExpentries.get(i);
+ }
+ return new ParseException(token, exptokseq, TOKEN_IMAGE);
+ }
+
+ /**
+ * Enable tracing.
+ */
+ final public void enable_tracing() {
+ }
+
+ /**
+ * Disable tracing.
+ */
+ final public void disable_tracing() {
+ }
+
+ private void jj_rescan_token() {
+ jjRescan = true;
+ for (int i = 0; i < 4; i++) {
+ try {
+ JJCalls p = jj2Rtns[i];
+ do {
+ if (p.gen > jjGen) {
+ jjLa = p.arg;
+ jjLastpos = jjScanpos = p.first;
+ switch (i) {
+ case 0:
+ jj_3_1();
+ break;
+ case 1:
+ jj_3_2();
+ break;
+ case 2:
+ jj_3_3();
+ break;
+ case 3:
+ jj_3_4();
+ break;
+ }
+ }
+ p = p.next;
+ } while (p != null);
+ } catch (LookaheadSuccess ls) {
+ }
+ }
+ jjRescan = false;
+ }
+
+ private void jj_save(int index, int xla) {
+ JJCalls p = jj2Rtns[index];
+ while (p.gen > jjGen) {
+ if (p.next == null) {
+ p = p.next = new JJCalls();
+ break;
+ }
+ p = p.next;
+ }
+ p.gen = jjGen + xla - jjLa;
+ p.first = token;
+ p.arg = xla;
+ }
+
+ static final class JJCalls {
+ int gen;
+ Token first;
+ int arg;
+ JJCalls next;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.jj
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.jj b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.jj
new file mode 100644
index 0000000..5d1a4a7
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.jj
@@ -0,0 +1,524 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This file was taken from ActiveMQ activemq-client/src/main/grammar/SelectorParser.jj.
+ *
+ * There are some modifications:
+ * 1. Convert string expressions default;
+ * 2. HEX_LITERAL and OCTAL_LITERAL were removed;
+ * 3. LIKE, ESCAPE, XPATH and XQUERY were removed;
+ * 4. Computation expressions were removed;
+ */
+
+// ----------------------------------------------------------------------------
+// OPTIONS
+// ----------------------------------------------------------------------------
+options {
+ STATIC = false;
+ UNICODE_INPUT = true;
+
+ //ERROR_REPORTING = false;
+}
+
+// ----------------------------------------------------------------------------
+// PARSER
+// ----------------------------------------------------------------------------
+
+PARSER_BEGIN(SelectorParser)
+
+package org.apache.rocketmq.filter.parser;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.rocketmq.filter.expression.BooleanExpression;
+import org.apache.rocketmq.filter.expression.ComparisonExpression;
+import org.apache.rocketmq.filter.expression.ConstantExpression;
+import org.apache.rocketmq.filter.expression.Expression;
+import org.apache.rocketmq.filter.expression.LogicExpression;
+import org.apache.rocketmq.filter.expression.MQFilterException;
+import org.apache.rocketmq.filter.expression.PropertyExpression;
+import org.apache.rocketmq.filter.expression.UnaryExpression;
+import org.apache.rocketmq.filter.util.LRUCache;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+
+/**
+ * JMS Selector Parser generated by JavaCC
+ *
+ * Do not edit this .java file directly - it is autogenerated from SelectorParser.jj
+ */
+public class SelectorParser {
+
+ private static final Cache<String, Object> PARSE_CACHE = CacheBuilder.newBuilder().maximumSize(100).build();
+// private static final String CONVERT_STRING_EXPRESSIONS_PREFIX = "convert_string_expressions:";
+
+ public static BooleanExpression parse(String sql) throws MQFilterException {
+// sql = "("+sql+")";
+ Object result = PARSE_CACHE.getIfPresent(sql);
+ if (result instanceof MQFilterException) {
+ throw (MQFilterException) result;
+ } else if (result instanceof BooleanExpression) {
+ return (BooleanExpression) result;
+ } else {
+
+// boolean convertStringExpressions = false;
+// if( sql.startsWith(CONVERT_STRING_EXPRESSIONS_PREFIX)) {
+// convertStringExpressions = true;
+// sql = sql.substring(CONVERT_STRING_EXPRESSIONS_PREFIX.length());
+// }
+//
+// if( convertStringExpressions ) {
+// ComparisonExpression.CONVERT_STRING_EXPRESSIONS.set(true);
+// }
+ ComparisonExpression.CONVERT_STRING_EXPRESSIONS.set(true);
+ try {
+
+ BooleanExpression e = new SelectorParser(sql).parse();
+ PARSE_CACHE.put(sql, e);
+ return e;
+ } catch (MQFilterException t) {
+ PARSE_CACHE.put(sql, t);
+ throw t;
+ } finally {
+ ComparisonExpression.CONVERT_STRING_EXPRESSIONS.remove();
+// if( convertStringExpressions ) {
+// ComparisonExpression.CONVERT_STRING_EXPRESSIONS.remove();
+// }
+ }
+ }
+ }
+
+ public static void clearCache() {
+ PARSE_CACHE.cleanUp();
+ }
+
+ private String sql;
+
+ protected SelectorParser(String sql) {
+ this(new StringReader(sql));
+ this.sql = sql;
+ }
+
+ protected BooleanExpression parse() throws MQFilterException {
+ try {
+ return this.JmsSelector();
+ }
+ catch (Throwable e) {
+ throw new MQFilterException("Invalid MessageSelector. ", e);
+ }
+ }
+
+ private BooleanExpression asBooleanExpression(Expression value) throws ParseException {
+ if (value instanceof BooleanExpression) {
+ return (BooleanExpression) value;
+ }
+ if (value instanceof PropertyExpression) {
+ return UnaryExpression.createBooleanCast( value );
+ }
+ throw new ParseException("Expression will not result in a boolean value: " + value);
+ }
+
+}
+
+PARSER_END(SelectorParser)
+
+// ----------------------------------------------------------------------------
+// Tokens
+// ----------------------------------------------------------------------------
+
+/* White Space */
+SPECIAL_TOKEN :
+{
+ " " | "\t" | "\n" | "\r" | "\f"
+}
+
+/* Comments */
+SKIP:
+{
+ <LINE_COMMENT: "--" (~["\n","\r"])* ("\n"|"\r"|"\r\n") >
+}
+
+SKIP:
+{
+ <BLOCK_COMMENT: "/*" (~["*"])* "*" ("*" | (~["*","/"] (~["*"])* "*"))* "/">
+}
+
+/* Reserved Words */
+TOKEN [IGNORE_CASE] :
+{
+ < NOT : "NOT">
+ | < AND : "AND">
+ | < OR : "OR">
+ | < BETWEEN : "BETWEEN">
+ | < IN : "IN">
+ | < TRUE : "TRUE" >
+ | < FALSE : "FALSE" >
+ | < NULL : "NULL" >
+ | < IS : "IS" >
+}
+
+/* Literals */
+TOKEN [IGNORE_CASE] :
+
+{
+
+ < DECIMAL_LITERAL: "0" | ["1"-"9"] (["0"-"9"])* (["l","L"])? >
+ | < FLOATING_POINT_LITERAL:
+ (["0"-"9"])+ "." (["0"-"9"])* (<EXPONENT>)? // matches: 5.5 or 5. or 5.5E10 or 5.E10
+ | "." (["0"-"9"])+ (<EXPONENT>)? // matches: .5 or .5E10
+ | (["0"-"9"])+ <EXPONENT> // matches: 5E10
+ >
+ | < #EXPONENT: "E" (["+","-"])? (["0"-"9"])+ >
+ | < STRING_LITERAL: "'" ( ("''") | ~["'"] )* "'" >
+}
+
+TOKEN [IGNORE_CASE] :
+{
+ < ID : ["a"-"z", "_", "$"] (["a"-"z","0"-"9","_", "$"])* >
+}
+
+// ----------------------------------------------------------------------------
+// Grammer
+// ----------------------------------------------------------------------------
+BooleanExpression JmsSelector() :
+{
+ Expression left=null;
+}
+{
+ (
+ left = orExpression()
+ )
+ {
+ return asBooleanExpression(left);
+ }
+
+}
+
+Expression orExpression() :
+{
+ Expression left;
+ Expression right;
+}
+{
+ (
+ left = andExpression()
+ (
+ <OR> right = andExpression()
+ {
+ left = LogicExpression.createOR(asBooleanExpression(left), asBooleanExpression(right));
+ }
+ )*
+ )
+ {
+ return left;
+ }
+
+}
+
+
+Expression andExpression() :
+{
+ Expression left;
+ Expression right;
+}
+{
+ (
+ left = equalityExpression()
+ (
+ <AND> right = equalityExpression()
+ {
+ left = LogicExpression.createAND(asBooleanExpression(left), asBooleanExpression(right));
+ }
+ )*
+ )
+ {
+ return left;
+ }
+}
+
+Expression equalityExpression() :
+{
+ Expression left;
+ Expression right;
+}
+{
+ (
+ left = comparisonExpression()
+ (
+
+ "=" right = comparisonExpression()
+ {
+ left = ComparisonExpression.createEqual(left, right);
+ }
+ |
+ "<>" right = comparisonExpression()
+ {
+ left = ComparisonExpression.createNotEqual(left, right);
+ }
+ |
+ LOOKAHEAD(2)
+ <IS> <NULL>
+ {
+ left = ComparisonExpression.createIsNull(left);
+ }
+ |
+ <IS> <NOT> <NULL>
+ {
+ left = ComparisonExpression.createIsNotNull(left);
+ }
+ )*
+ )
+ {
+ return left;
+ }
+}
+
+Expression comparisonExpression() :
+{
+ Expression left;
+ Expression right;
+ Expression low;
+ Expression high;
+ String t, u;
+ boolean not;
+ ArrayList list;
+}
+{
+ (
+ left = unaryExpr()
+ (
+
+ ">" right = unaryExpr()
+ {
+ left = ComparisonExpression.createGreaterThan(left, right);
+ }
+ |
+ ">=" right = unaryExpr()
+ {
+ left = ComparisonExpression.createGreaterThanEqual(left, right);
+ }
+ |
+ "<" right = unaryExpr()
+ {
+ left = ComparisonExpression.createLessThan(left, right);
+ }
+ |
+ "<=" right = unaryExpr()
+ {
+ left = ComparisonExpression.createLessThanEqual(left, right);
+ }
+ |
+ <BETWEEN> low = unaryExpr() <AND> high = unaryExpr()
+ {
+ left = ComparisonExpression.createBetween(left, low, high);
+ }
+ |
+ LOOKAHEAD(2)
+ <NOT> <BETWEEN> low = unaryExpr() <AND> high = unaryExpr()
+ {
+ left = ComparisonExpression.createNotBetween(left, low, high);
+ }
+ |
+ <IN>
+ "("
+ t = stringLitteral()
+ {
+ list = new ArrayList();
+ list.add( t );
+ }
+ (
+ ","
+ t = stringLitteral()
+ {
+ list.add( t );
+ }
+
+ )*
+ ")"
+ {
+ left = ComparisonExpression.createInFilter(left, list);
+ }
+ |
+ LOOKAHEAD(2)
+ <NOT> <IN>
+ "("
+ t = stringLitteral()
+ {
+ list = new ArrayList();
+ list.add( t );
+ }
+ (
+ ","
+ t = stringLitteral()
+ {
+ list.add( t );
+ }
+
+ )*
+ ")"
+ {
+ left = ComparisonExpression.createNotInFilter(left, list);
+ }
+
+ )*
+ )
+ {
+ return left;
+ }
+}
+
+Expression unaryExpr() :
+{
+ String s=null;
+ Expression left=null;
+}
+{
+ (
+ LOOKAHEAD( "+" unaryExpr() )
+ "+" left=unaryExpr()
+ |
+ "-" left=unaryExpr()
+ {
+ left = UnaryExpression.createNegate(left);
+ }
+ |
+ <NOT> left=unaryExpr()
+ {
+ left = UnaryExpression.createNOT( asBooleanExpression(left) );
+ }
+ |
+ left = primaryExpr()
+ )
+ {
+ return left;
+ }
+
+}
+
+Expression primaryExpr() :
+{
+ Expression left=null;
+}
+{
+ (
+ left = literal()
+ |
+ left = variable()
+ |
+ "(" left = orExpression() ")"
+ )
+ {
+ return left;
+ }
+}
+
+
+
+ConstantExpression literal() :
+{
+ Token t;
+ String s;
+ ConstantExpression left=null;
+}
+{
+ (
+ (
+ s = stringLitteral()
+ {
+ left = new ConstantExpression(s);
+ }
+ )
+ |
+ (
+ t = <DECIMAL_LITERAL>
+ {
+ left = ConstantExpression.createFromDecimal(t.image);
+ }
+ )
+ |
+ (
+ t = <FLOATING_POINT_LITERAL>
+ {
+ left = ConstantExpression.createFloat(t.image);
+ }
+ )
+ |
+ (
+ <TRUE>
+ {
+ left = ConstantExpression.TRUE;
+ }
+ )
+ |
+ (
+ <FALSE>
+ {
+ left = ConstantExpression.FALSE;
+ }
+ )
+ |
+ (
+ <NULL>
+ {
+ left = ConstantExpression.NULL;
+ }
+ )
+ )
+ {
+ return left;
+ }
+}
+
+String stringLitteral() :
+{
+ Token t;
+ StringBuffer rc = new StringBuffer();
+ boolean first=true;
+}
+{
+ t = <STRING_LITERAL>
+ {
+ // Decode the sting value.
+ String image = t.image;
+ for( int i=1; i < image.length()-1; i++ ) {
+ char c = image.charAt(i);
+ if( c == '\'' )
+ i++;
+ rc.append(c);
+ }
+ return rc.toString();
+ }
+}
+
+PropertyExpression variable() :
+{
+ Token t;
+ PropertyExpression left=null;
+}
+{
+ (
+ t = <ID>
+ {
+ left = new PropertyExpression(t.image);
+ }
+ )
+ {
+ return left;
+ }
+}
[02/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-107] Fix
possible concurrency problem on ServiceState when consumer start/shutdown,
closes apache/incubator-rocketmq#68
Posted by do...@apache.org.
[ROCKETMQ-107] Fix possible concurrency problem on ServiceState when consumer start/shutdown, closes apache/incubator-rocketmq#68
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/f508f131
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/f508f131
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/f508f131
Branch: refs/heads/release-4.1.0-incubating
Commit: f508f131f7dee2bcb86e66a6beb7bbdedbe31bc6
Parents: c183e0d
Author: Jaskey <li...@gmail.com>
Authored: Wed Apr 19 11:58:48 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Wed Apr 19 11:58:48 2017 +0800
----------------------------------------------------------------------
.../consumer/DefaultMQPullConsumerImpl.java | 21 +++++++++++++-------
.../consumer/DefaultMQPushConsumerImpl.java | 15 +++++++-------
2 files changed, 22 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/f508f131/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
index b26d062..7d43b37 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
@@ -70,7 +70,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
private final RPCHook rpcHook;
private final ArrayList<ConsumeMessageHook> consumeMessageHookList = new ArrayList<ConsumeMessageHook>();
private final ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<FilterMessageHook>();
- private ServiceState serviceState = ServiceState.CREATE_JUST;
+ private volatile ServiceState serviceState = ServiceState.CREATE_JUST;
private MQClientInstance mQClientFactory;
private PullAPIWrapper pullAPIWrapper;
private OffsetStore offsetStore;
@@ -161,7 +161,8 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
return this.pullSyncImpl(mq, subExpression, offset, maxNums, false, timeout);
}
- private PullResult pullSyncImpl(MessageQueue mq, String subExpression, long offset, int maxNums, boolean block, long timeout)
+ private PullResult pullSyncImpl(MessageQueue mq, String subExpression, long offset, int maxNums, boolean block,
+ long timeout)
throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
this.makeSureStateOK();
@@ -365,7 +366,8 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
pull(mq, subExpression, offset, maxNums, pullCallback, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis());
}
- public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback, long timeout)
+ public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback,
+ long timeout)
throws MQClientException, RemotingException, InterruptedException {
this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, false, timeout);
}
@@ -449,7 +451,8 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
return defaultMQPullConsumer;
}
- public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback)
+ public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums,
+ PullCallback pullCallback)
throws MQClientException, RemotingException, InterruptedException {
this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, true,
this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());
@@ -510,7 +513,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
}
}
- public void shutdown() {
+ public synchronized void shutdown() {
switch (this.serviceState) {
case CREATE_JUST:
break;
@@ -528,7 +531,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
}
}
- public void start() throws MQClientException {
+ public synchronized void start() throws MQClientException {
switch (this.serviceState) {
case CREATE_JUST:
this.serviceState = ServiceState.START_FAILED;
@@ -593,6 +596,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
default:
break;
}
+
}
private void checkConfig() throws MQClientException {
@@ -662,7 +666,8 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
this.offsetStore.updateOffset(mq, offset, false);
}
- public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+ public MessageExt viewMessage(String msgId)
+ throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
this.makeSureStateOK();
return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId);
}
@@ -692,6 +697,8 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
return serviceState;
}
+ //Don't use this deprecated setter, which will be removed soon.
+ @Deprecated
public void setServiceState(ServiceState serviceState) {
this.serviceState = serviceState;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/f508f131/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
index 4f33732..67f3ebe 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
@@ -97,7 +97,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
private final long consumerStartTimestamp = System.currentTimeMillis();
private final ArrayList<ConsumeMessageHook> consumeMessageHookList = new ArrayList<ConsumeMessageHook>();
private final RPCHook rpcHook;
- private ServiceState serviceState = ServiceState.CREATE_JUST;
+ private volatile ServiceState serviceState = ServiceState.CREATE_JUST;
private MQClientInstance mQClientFactory;
private PullAPIWrapper pullAPIWrapper;
private volatile boolean pause = false;
@@ -515,7 +515,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
}
}
- public void shutdown() {
+ public synchronized void shutdown() {
switch (this.serviceState) {
case CREATE_JUST:
break;
@@ -535,7 +535,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
}
}
- public void start() throws MQClientException {
+ public synchronized void start() throws MQClientException {
switch (this.serviceState) {
case CREATE_JUST:
log.info("the consumer [{}] start beginning. messageModel={}, isUnitMode={}", this.defaultMQPushConsumer.getConsumerGroup(),
@@ -615,9 +615,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
}
this.updateTopicSubscribeInfoWhenSubscriptionChanged();
-
this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();
-
this.mQClientFactory.rebalanceImmediately();
}
@@ -855,7 +853,8 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
this.consumeMessageService.updateCorePoolSize(corePoolSize);
}
- public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+ public MessageExt viewMessage(String msgId)
+ throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId);
}
@@ -1014,7 +1013,9 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
return serviceState;
}
- public void setServiceState(ServiceState serviceState) {
+ //Don't use this deprecated setter, which will be removed soon.
+ @Deprecated
+ public synchronized void setServiceState(ServiceState serviceState) {
this.serviceState = serviceState;
}
[33/50] [abbrv] incubator-rocketmq git commit:
[ROCKETMQ-188]RemotingExecption is not consistent between invoke async and
invoke oneway closes apache/incubator-rocketmq#98
Posted by do...@apache.org.
[ROCKETMQ-188]RemotingExecption is not consistent between invoke async and invoke oneway closes apache/incubator-rocketmq#98
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/8c8610f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/8c8610f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/8c8610f9
Branch: refs/heads/release-4.1.0-incubating
Commit: 8c8610f9121d19bf7108903e41276d5f6afaa81a
Parents: b1fcf1b
Author: Jaskey <li...@gmail.com>
Authored: Sat May 27 11:22:46 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 11:22:46 2017 +0800
----------------------------------------------------------------------
.../remoting/netty/NettyRemotingAbstract.java | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/8c8610f9/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
----------------------------------------------------------------------
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
index cddab3d..15586cb 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
@@ -417,14 +417,18 @@ public abstract class NettyRemotingAbstract {
throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e);
}
} else {
- String info =
- String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", //
- timeoutMillis, //
- this.semaphoreAsync.getQueueLength(), //
- this.semaphoreAsync.availablePermits()//
- );
- PLOG.warn(info);
- throw new RemotingTooMuchRequestException(info);
+ if (timeoutMillis <= 0) {
+ throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast");
+ } else {
+ String info =
+ String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", //
+ timeoutMillis, //
+ this.semaphoreAsync.getQueueLength(), //
+ this.semaphoreAsync.availablePermits()//
+ );
+ PLOG.warn(info);
+ throw new RemotingTimeoutException(info);
+ }
}
}
[31/50] [abbrv] incubator-rocketmq git commit: Add test case for
LocalFileOffsetStore closes apache/incubator-rocketmq#59
Posted by do...@apache.org.
Add test case for LocalFileOffsetStore closes apache/incubator-rocketmq#59
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/0adad6f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/0adad6f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/0adad6f0
Branch: refs/heads/release-4.1.0-incubating
Commit: 0adad6f0025483647e760c1145f7736462c0ec79
Parents: e5d01b4
Author: djKooks <in...@gmail.com>
Authored: Sat May 27 11:06:37 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 11:06:37 2017 +0800
----------------------------------------------------------------------
.../consumer/store/LocalFileOffsetStoreTest.java | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/0adad6f0/client/src/test/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStoreTest.java
----------------------------------------------------------------------
diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStoreTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStoreTest.java
index 22e212b..a705b30 100644
--- a/client/src/test/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStoreTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStoreTest.java
@@ -19,6 +19,8 @@ package org.apache.rocketmq.client.consumer.store;
import java.io.File;
import java.util.Collections;
import java.util.HashSet;
+import java.util.Map;
+
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.common.message.MessageQueue;
@@ -72,4 +74,15 @@ public class LocalFileOffsetStoreTest {
offsetStore.persistAll(new HashSet<MessageQueue>(Collections.singletonList(messageQueue)));
assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1024);
}
+
+ @Test
+ public void testCloneOffset() throws Exception {
+ OffsetStore offsetStore = new LocalFileOffsetStore(mQClientFactory, group);
+ MessageQueue messageQueue = new MessageQueue(topic, brokerName, 3);
+ offsetStore.updateOffset(messageQueue, 1024, false);
+ Map<MessageQueue, Long> cloneOffsetTable = offsetStore.cloneOffsetTable(topic);
+
+ assertThat(cloneOffsetTable.size()).isEqualTo(1);
+ assertThat(cloneOffsetTable.get(messageQueue)).isEqualTo(1024);
+ }
}
\ No newline at end of file
[35/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-206] Catch
the IOException when call the file2String method.
Posted by do...@apache.org.
[ROCKETMQ-206] Catch the IOException when call the file2String method.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/aced0de7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/aced0de7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/aced0de7
Branch: refs/heads/release-4.1.0-incubating
Commit: aced0de7d8f98a01d9d109dd592a6cb31fd174d9
Parents: ceeef8e
Author: yukon <yu...@apache.org>
Authored: Sat May 27 11:34:44 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Sat May 27 11:34:44 2017 +0800
----------------------------------------------------------------------
.../client/consumer/store/LocalFileOffsetStore.java | 14 ++++++++++++--
.../apache/rocketmq/example/benchmark/Consumer.java | 3 ++-
.../org/apache/rocketmq/example/filter/Consumer.java | 3 ++-
.../rocketmq/namesrv/kvconfig/KVConfigManager.java | 7 ++++++-
4 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/aced0de7/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java
index 2cde5f8..6c81516 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java
@@ -180,7 +180,12 @@ public class LocalFileOffsetStore implements OffsetStore {
}
private OffsetSerializeWrapper readLocalOffset() throws MQClientException {
- String content = MixAll.file2String(this.storePath);
+ String content = null;
+ try {
+ content = MixAll.file2String(this.storePath);
+ } catch (IOException e) {
+ log.warn("Load local offset store file exception", e);
+ }
if (null == content || content.length() == 0) {
return this.readLocalOffsetBak();
} else {
@@ -198,7 +203,12 @@ public class LocalFileOffsetStore implements OffsetStore {
}
private OffsetSerializeWrapper readLocalOffsetBak() throws MQClientException {
- String content = MixAll.file2String(this.storePath + ".bak");
+ String content = null;
+ try {
+ content = MixAll.file2String(this.storePath + ".bak");
+ } catch (IOException e) {
+ log.warn("Load local offset store bak file exception", e);
+ }
if (content != null && content.length() > 0) {
OffsetSerializeWrapper offsetSerializeWrapper = null;
try {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/aced0de7/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
index 3e1b79b..d431d3e 100644
--- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.example.benchmark;
+import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Timer;
@@ -39,7 +40,7 @@ import org.apache.rocketmq.srvutil.ServerUtil;
public class Consumer {
- public static void main(String[] args) throws MQClientException {
+ public static void main(String[] args) throws MQClientException, IOException {
Options options = ServerUtil.buildCommandlineOptions(new Options());
CommandLine commandLine = ServerUtil.parseCmdLine("benchmarkConsumer", args, buildCommandlineOptions(options), new PosixParser());
if (null == commandLine) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/aced0de7/example/src/main/java/org/apache/rocketmq/example/filter/Consumer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/filter/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/filter/Consumer.java
index d63435b..0be8e1d 100644
--- a/example/src/main/java/org/apache/rocketmq/example/filter/Consumer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/filter/Consumer.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.example.filter;
import java.io.File;
+import java.io.IOException;
import java.util.List;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
@@ -28,7 +29,7 @@ import org.apache.rocketmq.common.message.MessageExt;
public class Consumer {
- public static void main(String[] args) throws InterruptedException, MQClientException {
+ public static void main(String[] args) throws InterruptedException, MQClientException, IOException {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("ConsumerGroupNamecc4");
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/aced0de7/namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java
----------------------------------------------------------------------
diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java
index 69afcad..be13bd6 100644
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java
@@ -43,7 +43,12 @@ public class KVConfigManager {
}
public void load() {
- String content = MixAll.file2String(this.namesrvController.getNamesrvConfig().getKvConfigPath());
+ String content = null;
+ try {
+ content = MixAll.file2String(this.namesrvController.getNamesrvConfig().getKvConfigPath());
+ } catch (IOException e) {
+ log.warn("Load KV config table exception", e);
+ }
if (content != null) {
KVConfigSerializeWrapper kvConfigSerializeWrapper =
KVConfigSerializeWrapper.fromJson(content, KVConfigSerializeWrapper.class);
[10/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-121]Support
message filtering based on SQL92 closes apache/incubator-rocketmq#82
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java
index 6349ffc..67807a8 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java
@@ -22,15 +22,19 @@ import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.body.CheckClientRequestBody;
import org.apache.rocketmq.common.protocol.header.UnregisterClientRequestHeader;
import org.apache.rocketmq.common.protocol.header.UnregisterClientResponseHeader;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
import org.apache.rocketmq.common.protocol.heartbeat.ProducerData;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.common.sysflag.TopicSysFlag;
+import org.apache.rocketmq.filter.FilterFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
@@ -54,6 +58,8 @@ public class ClientManageProcessor implements NettyRequestProcessor {
return this.heartBeat(ctx, request);
case RequestCode.UNREGISTER_CLIENT:
return this.unregisterClient(ctx, request);
+ case RequestCode.CHECK_CLIENT_CONFIG:
+ return this.checkClientConfig(ctx, request);
default:
break;
}
@@ -157,4 +163,42 @@ public class ClientManageProcessor implements NettyRequestProcessor {
response.setRemark(null);
return response;
}
+
+ public RemotingCommand checkClientConfig(ChannelHandlerContext ctx, RemotingCommand request)
+ throws RemotingCommandException {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ CheckClientRequestBody requestBody = CheckClientRequestBody.decode(request.getBody(),
+ CheckClientRequestBody.class);
+
+ if (requestBody != null && requestBody.getSubscriptionData() != null) {
+ SubscriptionData subscriptionData = requestBody.getSubscriptionData();
+
+ if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
+
+ if (!this.brokerController.getBrokerConfig().isEnablePropertyFilter()) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("The broker does not support consumer to filter message by " + subscriptionData.getExpressionType());
+ return response;
+ }
+
+ try {
+ FilterFactory.INSTANCE.get(subscriptionData.getExpressionType()).compile(subscriptionData.getSubString());
+ } catch (Exception e) {
+ log.warn("Client {}@{} filter message, but failed to compile expression! sub={}, error={}",
+ requestBody.getClientId(), requestBody.getGroup(), requestBody.getSubscriptionData(), e.getMessage());
+ response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);
+ response.setRemark(e.getMessage());
+ return response;
+ }
+ }
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
index 89967d8..10945da 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
@@ -25,6 +25,10 @@ import java.nio.ByteBuffer;
import java.util.List;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.broker.filter.ConsumerFilterData;
+import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
+import org.apache.rocketmq.broker.filter.ExpressionForRetryMessageFilter;
+import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
import org.apache.rocketmq.broker.longpolling.PullRequest;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
@@ -34,6 +38,7 @@ import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.TopicFilterType;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.common.filter.FilterAPI;
import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.common.message.MessageDecoder;
@@ -54,6 +59,7 @@ import org.apache.rocketmq.remoting.netty.RequestTask;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.store.MessageFilter;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
@@ -142,13 +148,22 @@ public class PullMessageProcessor implements NettyRequestProcessor {
}
SubscriptionData subscriptionData = null;
+ ConsumerFilterData consumerFilterData = null;
if (hasSubscriptionFlag) {
try {
- subscriptionData = FilterAPI.buildSubscriptionData(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
- requestHeader.getSubscription());
+ subscriptionData = FilterAPI.build(
+ requestHeader.getTopic(), requestHeader.getSubscription(), requestHeader.getExpressionType()
+ );
+ if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
+ consumerFilterData = ConsumerFilterManager.build(
+ requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getSubscription(),
+ requestHeader.getExpressionType(), requestHeader.getSubVersion()
+ );
+ assert consumerFilterData != null;
+ }
} catch (Exception e) {
LOG.warn("Parse the consumer's subscription[{}] failed, group: {}", requestHeader.getSubscription(), //
- requestHeader.getConsumerGroup());
+ requestHeader.getConsumerGroup());
response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);
response.setRemark("parse the consumer's subscription failed");
return response;
@@ -180,16 +195,48 @@ public class PullMessageProcessor implements NettyRequestProcessor {
if (subscriptionData.getSubVersion() < requestHeader.getSubVersion()) {
LOG.warn("The broker's subscription is not latest, group: {} {}", requestHeader.getConsumerGroup(),
- subscriptionData.getSubString());
+ subscriptionData.getSubString());
response.setCode(ResponseCode.SUBSCRIPTION_NOT_LATEST);
response.setRemark("the consumer's subscription not latest");
return response;
}
+ if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
+ consumerFilterData = this.brokerController.getConsumerFilterManager().get(requestHeader.getTopic(),
+ requestHeader.getConsumerGroup());
+ if (consumerFilterData == null) {
+ response.setCode(ResponseCode.FILTER_DATA_NOT_EXIST);
+ response.setRemark("The broker's consumer filter data is not exist!Your expression may be wrong!");
+ return response;
+ }
+ if (consumerFilterData.getClientVersion() < requestHeader.getSubVersion()) {
+ LOG.warn("The broker's consumer filter data is not latest, group: {}, topic: {}, serverV: {}, clientV: {}",
+ requestHeader.getConsumerGroup(), requestHeader.getTopic(), consumerFilterData.getClientVersion(), requestHeader.getSubVersion());
+ response.setCode(ResponseCode.FILTER_DATA_NOT_LATEST);
+ response.setRemark("the consumer's consumer filter data not latest");
+ return response;
+ }
+ }
+ }
+
+ if (!ExpressionType.isTagType(subscriptionData.getExpressionType())
+ && !this.brokerController.getBrokerConfig().isEnablePropertyFilter()) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("The broker does not support consumer to filter message by " + subscriptionData.getExpressionType());
+ return response;
+ }
+
+ MessageFilter messageFilter;
+ if (this.brokerController.getBrokerConfig().isFilterSupportRetry()) {
+ messageFilter = new ExpressionForRetryMessageFilter(subscriptionData, consumerFilterData,
+ this.brokerController.getConsumerFilterManager());
+ } else {
+ messageFilter = new ExpressionMessageFilter(subscriptionData, consumerFilterData,
+ this.brokerController.getConsumerFilterManager());
}
final GetMessageResult getMessageResult =
this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
- requestHeader.getQueueId(), requestHeader.getQueueOffset(), requestHeader.getMaxMsgNums(), subscriptionData);
+ requestHeader.getQueueId(), requestHeader.getQueueOffset(), requestHeader.getMaxMsgNums(), messageFilter);
if (getMessageResult != null) {
response.setRemark(getMessageResult.getStatus().name());
responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset());
@@ -368,7 +415,7 @@ public class PullMessageProcessor implements NettyRequestProcessor {
long offset = requestHeader.getQueueOffset();
int queueId = requestHeader.getQueueId();
PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills,
- this.brokerController.getMessageStore().now(), offset, subscriptionData);
+ this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter);
this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest);
response = null;
break;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/test/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMapTest.java
----------------------------------------------------------------------
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMapTest.java b/broker/src/test/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMapTest.java
new file mode 100644
index 0000000..87f6256
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMapTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.filter;
+
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.filter.ExpressionType;
+import org.apache.rocketmq.filter.util.BitsArray;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.junit.Test;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class CommitLogDispatcherCalcBitMapTest {
+
+ @Test
+ public void testDispatch_filterDataIllegal() {
+ BrokerConfig brokerConfig = new BrokerConfig();
+ brokerConfig.setEnableCalcFilterBitMap(true);
+
+ ConsumerFilterManager filterManager = new ConsumerFilterManager();
+
+ filterManager.register("topic0", "CID_0", "a is not null and a >= 5",
+ ExpressionType.SQL92, System.currentTimeMillis());
+
+ filterManager.register("topic0", "CID_1", "a is not null and a >= 15",
+ ExpressionType.SQL92, System.currentTimeMillis());
+
+ ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
+ nullExpression.setExpression(null);
+ nullExpression.setCompiledExpression(null);
+ ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
+ nullBloomData.setBloomFilterData(null);
+
+
+ CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
+ filterManager);
+
+ for (int i = 0; i < 1; i++) {
+ Map<String, String> properties = new HashMap<String, String>(4);
+ properties.put("a", String.valueOf(i * 10 + 5));
+
+ String topic = "topic" + i;
+
+ DispatchRequest dispatchRequest = new DispatchRequest(
+ topic,
+ 0,
+ i * 100 + 123,
+ 100,
+ (long) ("tags" + i).hashCode(),
+ System.currentTimeMillis(),
+ i,
+ null,
+ UUID.randomUUID().toString(),
+ 0,
+ 0,
+ properties
+ );
+
+ calcBitMap.dispatch(dispatchRequest);
+
+ assertThat(dispatchRequest.getBitMap()).isNotNull();
+
+ BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),
+ filterManager.getBloomFilter().getM());
+
+ for (int j = 0; j < bitsArray.bitLength(); j++) {
+ assertThat(bitsArray.getBit(j)).isFalse();
+ }
+ }
+ }
+
+ @Test
+ public void testDispatch_blankFilterData() {
+ BrokerConfig brokerConfig = new BrokerConfig();
+ brokerConfig.setEnableCalcFilterBitMap(true);
+
+ ConsumerFilterManager filterManager = new ConsumerFilterManager();
+
+ CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
+ filterManager);
+
+ for (int i = 0; i < 10; i++) {
+ Map<String, String> properties = new HashMap<String, String>(4);
+ properties.put("a", String.valueOf(i * 10 + 5));
+
+ String topic = "topic" + i;
+
+ DispatchRequest dispatchRequest = new DispatchRequest(
+ topic,
+ 0,
+ i * 100 + 123,
+ 100,
+ (long) ("tags" + i).hashCode(),
+ System.currentTimeMillis(),
+ i,
+ null,
+ UUID.randomUUID().toString(),
+ 0,
+ 0,
+ properties
+ );
+
+ calcBitMap.dispatch(dispatchRequest);
+
+ assertThat(dispatchRequest.getBitMap()).isNull();
+ }
+ }
+
+ @Test
+ public void testDispatch() {
+ BrokerConfig brokerConfig = new BrokerConfig();
+ brokerConfig.setEnableCalcFilterBitMap(true);
+
+ ConsumerFilterManager filterManager = ConsumerFilterManagerTest.gen(10, 10);
+
+ CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
+ filterManager);
+
+ for (int i = 0; i < 10; i++) {
+ Map<String, String> properties = new HashMap<String, String>(4);
+ properties.put("a", String.valueOf(i * 10 + 5));
+
+ String topic = "topic" + i;
+
+ DispatchRequest dispatchRequest = new DispatchRequest(
+ topic,
+ 0,
+ i * 100 + 123,
+ 100,
+ (long) ("tags" + i).hashCode(),
+ System.currentTimeMillis(),
+ i,
+ null,
+ UUID.randomUUID().toString(),
+ 0,
+ 0,
+ properties
+ );
+
+ calcBitMap.dispatch(dispatchRequest);
+
+ assertThat(dispatchRequest.getBitMap()).isNotNull();
+
+ BitsArray bits = BitsArray.create(dispatchRequest.getBitMap());
+
+ Collection<ConsumerFilterData> filterDatas = filterManager.get(topic);
+
+ for (ConsumerFilterData filterData : filterDatas) {
+
+ if (filterManager.getBloomFilter().isHit(filterData.getBloomFilterData(), bits)) {
+ try {
+ assertThat((Boolean) filterData.getCompiledExpression().evaluate(
+ new MessageEvaluationContext(properties)
+ )).isTrue();
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+ } else {
+ try {
+ assertThat((Boolean) filterData.getCompiledExpression().evaluate(
+ new MessageEvaluationContext(properties)
+ )).isFalse();
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/test/java/org/apache/rocketmq/broker/filter/ConsumerFilterManagerTest.java
----------------------------------------------------------------------
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/filter/ConsumerFilterManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/filter/ConsumerFilterManagerTest.java
new file mode 100644
index 0000000..c8412a8
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/filter/ConsumerFilterManagerTest.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.filter;
+
+import org.apache.rocketmq.common.filter.ExpressionType;
+import org.apache.rocketmq.common.filter.FilterAPI;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ConsumerFilterManagerTest {
+
+ public static ConsumerFilterManager gen(int topicCount, int consumerCount) {
+ ConsumerFilterManager filterManager = new ConsumerFilterManager();
+
+ for (int i = 0; i < topicCount; i++) {
+ String topic = "topic" + i;
+
+ for (int j = 0; j < consumerCount; j++) {
+
+ String consumer = "CID_" + j;
+
+ filterManager.register(topic, consumer, expr(j), ExpressionType.SQL92, System.currentTimeMillis());
+ }
+ }
+
+ return filterManager;
+ }
+
+ public static String expr(int i) {
+ return "a is not null and a > " + ((i - 1) * 10) + " and a < " + ((i + 1) * 10);
+ }
+
+ @Test
+ public void testRegister_newExpressionCompileErrorAndRemoveOld() {
+ ConsumerFilterManager filterManager = gen(10, 10);
+
+ assertThat(filterManager.get("topic9", "CID_9")).isNotNull();
+
+ String newExpr = "a between 10,20";
+
+ assertThat(filterManager.register("topic9", "CID_9", newExpr, ExpressionType.SQL92, System.currentTimeMillis() + 1))
+ .isFalse();
+ assertThat(filterManager.get("topic9", "CID_9")).isNull();
+
+ newExpr = "a between 10 AND 20";
+
+ assertThat(filterManager.register("topic9", "CID_9", newExpr, ExpressionType.SQL92, System.currentTimeMillis() + 1))
+ .isTrue();
+
+ ConsumerFilterData filterData = filterManager.get("topic9", "CID_9");
+
+ assertThat(filterData).isNotNull();
+ assertThat(newExpr).isEqualTo(filterData.getExpression());
+ }
+
+ @Test
+ public void testRegister_change() {
+ ConsumerFilterManager filterManager = gen(10, 10);
+
+ ConsumerFilterData filterData = filterManager.get("topic9", "CID_9");
+
+ System.out.println(filterData.getCompiledExpression());
+
+ String newExpr = "a > 0 and a < 10";
+
+ filterManager.register("topic9", "CID_9", newExpr, ExpressionType.SQL92, System.currentTimeMillis() + 1);
+
+ filterData = filterManager.get("topic9", "CID_9");
+
+ assertThat(newExpr).isEqualTo(filterData.getExpression());
+
+ System.out.println(filterData.toString());
+ }
+
+ @Test
+ public void testRegister() {
+ ConsumerFilterManager filterManager = gen(10, 10);
+
+ ConsumerFilterData filterData = filterManager.get("topic9", "CID_9");
+
+ assertThat(filterData).isNotNull();
+ assertThat(filterData.isDead()).isFalse();
+
+ // new version
+ assertThat(filterManager.register(
+ "topic9", "CID_9", "a is not null", ExpressionType.SQL92, System.currentTimeMillis() + 1000
+ )).isTrue();
+
+ ConsumerFilterData newFilter = filterManager.get("topic9", "CID_9");
+
+ assertThat(newFilter).isNotEqualTo(filterData);
+
+ // same version
+ assertThat(filterManager.register(
+ "topic9", "CID_9", "a is null", ExpressionType.SQL92, newFilter.getClientVersion()
+ )).isFalse();
+
+ ConsumerFilterData filterData1 = filterManager.get("topic9", "CID_9");
+
+ assertThat(newFilter).isEqualTo(filterData1);
+ }
+
+ @Test
+ public void testRegister_reAlive() {
+ ConsumerFilterManager filterManager = gen(10, 10);
+
+ ConsumerFilterData filterData = filterManager.get("topic9", "CID_9");
+
+ assertThat(filterData).isNotNull();
+ assertThat(filterData.isDead()).isFalse();
+
+ //make dead
+ filterManager.unRegister("CID_9");
+
+ //reAlive
+ filterManager.register(
+ filterData.getTopic(),
+ filterData.getConsumerGroup(),
+ filterData.getExpression(),
+ filterData.getExpressionType(),
+ System.currentTimeMillis()
+ );
+
+ ConsumerFilterData newFilterData = filterManager.get("topic9", "CID_9");
+
+ assertThat(newFilterData).isNotNull();
+ assertThat(newFilterData.isDead()).isFalse();
+ }
+
+ @Test
+ public void testRegister_bySubscriptionData() {
+ ConsumerFilterManager filterManager = new ConsumerFilterManager();
+ List<SubscriptionData> subscriptionDatas = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ try {
+ subscriptionDatas.add(
+ FilterAPI.build(
+ "topic" + i,
+ "a is not null and a > " + i,
+ ExpressionType.SQL92
+ )
+ );
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+ }
+
+ filterManager.register("CID_0", subscriptionDatas);
+
+ Collection<ConsumerFilterData> filterDatas = filterManager.getByGroup("CID_0");
+
+ assertThat(filterDatas).isNotNull();
+ assertThat(filterDatas.size()).isEqualTo(10);
+
+ Iterator<ConsumerFilterData> iterator = filterDatas.iterator();
+ while (iterator.hasNext()) {
+ ConsumerFilterData filterData = iterator.next();
+
+ assertThat(filterData).isNotNull();
+ assertThat(filterManager.getBloomFilter().isValid(filterData.getBloomFilterData())).isTrue();
+ }
+ }
+
+ @Test
+ public void testRegister_tag() {
+ ConsumerFilterManager filterManager = new ConsumerFilterManager();
+
+ assertThat(filterManager.register("topic0", "CID_0", "*", null, System.currentTimeMillis())).isFalse();
+
+ Collection<ConsumerFilterData> filterDatas = filterManager.getByGroup("CID_0");
+
+ assertThat(filterDatas).isNullOrEmpty();
+ }
+
+ @Test
+ public void testUnregister() {
+ ConsumerFilterManager filterManager = gen(10, 10);
+
+ ConsumerFilterData filterData = filterManager.get("topic9", "CID_9");
+
+ assertThat(filterData).isNotNull();
+ assertThat(filterData.isDead()).isFalse();
+
+ filterManager.unRegister("CID_9");
+
+ assertThat(filterData.isDead()).isTrue();
+ }
+
+ @Test
+ public void testPersist() {
+ ConsumerFilterManager filterManager = gen(10, 10);
+
+ try {
+ filterManager.persist();
+
+ ConsumerFilterData filterData = filterManager.get("topic9", "CID_9");
+
+ assertThat(filterData).isNotNull();
+ assertThat(filterData.isDead()).isFalse();
+
+ ConsumerFilterManager loadFilter = new ConsumerFilterManager();
+
+ assertThat(loadFilter.load()).isTrue();
+
+ filterData = loadFilter.get("topic9", "CID_9");
+
+ assertThat(filterData).isNotNull();
+ assertThat(filterData.isDead()).isTrue();
+ assertThat(filterData.getCompiledExpression()).isNotNull();
+ } finally {
+ deleteDirectory("./unit_test");
+ }
+ }
+
+ @Test
+ public void testPersist_clean() {
+ ConsumerFilterManager filterManager = gen(10, 10);
+
+ String topic = "topic9";
+ for (int i = 0; i < 10; i++) {
+ String cid = "CID_" + i;
+
+ ConsumerFilterData filterData = filterManager.get(topic, cid);
+
+ assertThat(filterData).isNotNull();
+ assertThat(filterData.isDead()).isFalse();
+
+ //make dead more than 24h
+ filterData.setBornTime(System.currentTimeMillis() - 26 * 60 * 60 * 1000);
+ filterData.setDeadTime(System.currentTimeMillis() - 25 * 60 * 60 * 1000);
+ }
+
+ try {
+ filterManager.persist();
+
+ ConsumerFilterManager loadFilter = new ConsumerFilterManager();
+
+ assertThat(loadFilter.load()).isTrue();
+
+ ConsumerFilterData filterData = loadFilter.get(topic, "CID_9");
+
+ assertThat(filterData).isNull();
+
+ Collection<ConsumerFilterData> topicData = loadFilter.get(topic);
+
+ assertThat(topicData).isNullOrEmpty();
+ } finally {
+ deleteDirectory("./unit_test");
+ }
+ }
+
+ protected void deleteDirectory(String rootPath) {
+ File file = new File(rootPath);
+ deleteFile(file);
+ }
+
+ protected void deleteFile(File file) {
+ File[] subFiles = file.listFiles();
+ if (subFiles != null) {
+ for (File sub : subFiles) {
+ deleteFile(sub);
+ }
+ }
+
+ file.delete();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/test/java/org/apache/rocketmq/broker/filter/MessageStoreWithFilterTest.java
----------------------------------------------------------------------
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/filter/MessageStoreWithFilterTest.java b/broker/src/test/java/org/apache/rocketmq/broker/filter/MessageStoreWithFilterTest.java
new file mode 100644
index 0000000..53e563e
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/filter/MessageStoreWithFilterTest.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.filter;
+
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.filter.ExpressionType;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.store.CommitLogDispatcher;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.MessageArrivingListener;
+import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class MessageStoreWithFilterTest {
+
+ private static final String msg = "Once, there was a chance for me!";
+ private static final byte[] msgBody = msg.getBytes();
+
+ private static final String topic = "topic";
+ private static final int queueId = 0;
+ private static final String storePath = "." + File.separator + "unit_test_store";
+ private static final int commitLogFileSize = 1024 * 1024 * 256;
+ private static final int cqFileSize = 300000 * 20;
+ private static final int cqExtFileSize = 300000 * 128;
+
+ private static SocketAddress BornHost;
+
+ private static SocketAddress StoreHost;
+
+ static {
+ try {
+ StoreHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ }
+ try {
+ BornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0);
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public MessageExtBrokerInner buildMessage() {
+ MessageExtBrokerInner msg = new MessageExtBrokerInner();
+ msg.setTopic(topic);
+ msg.setTags("TAG1");
+ msg.setKeys("Hello");
+ msg.setBody(msgBody);
+ msg.setKeys(String.valueOf(System.currentTimeMillis()));
+ msg.setQueueId(queueId);
+ msg.setSysFlag(0);
+ msg.setBornTimestamp(System.currentTimeMillis());
+ msg.setStoreHost(StoreHost);
+ msg.setBornHost(BornHost);
+ for (int i = 1; i < 3; i++) {
+ msg.putUserProperty(String.valueOf(i), "imagoodperson" + i);
+ }
+ msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
+
+ return msg;
+ }
+
+ public MessageStoreConfig buildStoreConfig(int commitLogFileSize, int cqFileSize,
+ boolean enableCqExt, int cqExtFileSize) {
+ MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+ messageStoreConfig.setMapedFileSizeCommitLog(commitLogFileSize);
+ messageStoreConfig.setMapedFileSizeConsumeQueue(cqFileSize);
+ messageStoreConfig.setMappedFileSizeConsumeQueueExt(cqExtFileSize);
+ messageStoreConfig.setMessageIndexEnable(false);
+ messageStoreConfig.setEnableConsumeQueueExt(enableCqExt);
+
+ messageStoreConfig.setStorePathRootDir(storePath);
+ messageStoreConfig.setStorePathCommitLog(storePath + File.separator + "commitlog");
+
+ return messageStoreConfig;
+ }
+
+ protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
+ MessageStoreConfig messageStoreConfig = buildStoreConfig(
+ commitLogFileSize, cqFileSize, true, cqExtFileSize
+ );
+
+ BrokerConfig brokerConfig = new BrokerConfig();
+ brokerConfig.setEnableCalcFilterBitMap(true);
+ brokerConfig.setMaxErrorRateOfBloomFilter(20);
+ brokerConfig.setExpectConsumerNumUseFilter(64);
+
+ DefaultMessageStore master = new DefaultMessageStore(
+ messageStoreConfig,
+ new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
+ new MessageArrivingListener() {
+ @Override
+ public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
+ long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
+// System.out.println(String.format("Msg coming: %s, %d, %d, %d",
+// topic, queueId, logicOffset, tagsCode));
+ }
+ }
+ , brokerConfig);
+
+ master.getDispatcherList().addFirst(new CommitLogDispatcher() {
+ @Override
+ public void dispatch(DispatchRequest request) {
+ try {
+// System.out.println(String.format("offset:%d, bitMap:%s", request.getCommitLogOffset(),
+// BitsArray.create(request.getBitMap()).toString()));
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));
+
+ assertThat(master.load()).isTrue();
+
+ master.start();
+
+ return master;
+ }
+
+ protected List<MessageExtBrokerInner> putMsg(DefaultMessageStore master, int topicCount, int msgCountPerTopic) throws Exception {
+ List<MessageExtBrokerInner> msgs = new ArrayList<MessageExtBrokerInner>();
+ for (int i = 0; i < topicCount; i++) {
+ String realTopic = topic + i;
+ for (int j = 0; j < msgCountPerTopic; j++) {
+ MessageExtBrokerInner msg = buildMessage();
+ msg.setTopic(realTopic);
+ msg.putUserProperty("a", String.valueOf(j * 10 + 5));
+ msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
+
+ PutMessageResult result = master.putMessage(msg);
+
+ msg.setMsgId(result.getAppendMessageResult().getMsgId());
+
+ msgs.add(msg);
+ }
+ }
+
+ return msgs;
+ }
+
+ protected void deleteDirectory(String rootPath) {
+ File file = new File(rootPath);
+ deleteFile(file);
+ }
+
+ protected void deleteFile(File file) {
+ File[] subFiles = file.listFiles();
+ if (subFiles != null) {
+ for (File sub : subFiles) {
+ deleteFile(sub);
+ }
+ }
+
+ file.delete();
+ }
+
+ protected List<MessageExtBrokerInner> filtered(List<MessageExtBrokerInner> msgs, ConsumerFilterData filterData) {
+ List<MessageExtBrokerInner> filteredMsgs = new ArrayList<MessageExtBrokerInner>();
+
+ for (MessageExtBrokerInner messageExtBrokerInner : msgs) {
+
+ if (!messageExtBrokerInner.getTopic().equals(filterData.getTopic())) {
+ continue;
+ }
+
+ try {
+ Object evlRet = filterData.getCompiledExpression().evaluate(new MessageEvaluationContext(messageExtBrokerInner.getProperties()));
+
+ if (evlRet == null || !(evlRet instanceof Boolean) || (Boolean) evlRet) {
+ filteredMsgs.add(messageExtBrokerInner);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+ }
+
+ return filteredMsgs;
+ }
+
+ @Test
+ public void testGetMessage_withFilterBitMapAndConsumerChanged() {
+ int topicCount = 10, msgPerTopic = 10;
+ ConsumerFilterManager filterManager = ConsumerFilterManagerTest.gen(topicCount, msgPerTopic);
+
+ DefaultMessageStore master = null;
+ try {
+ master = gen(filterManager);
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+
+ try {
+ List<MessageExtBrokerInner> msgs = null;
+ try {
+ msgs = putMsg(master, topicCount, msgPerTopic);
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+
+ // sleep to wait for consume queue has been constructed.
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+
+ // reset consumer;
+ String topic = "topic" + 0;
+ String resetGroup = "CID_" + 2;
+ String normalGroup = "CID_" + 3;
+
+ {
+ // reset CID_2@topic0 to get all messages.
+ SubscriptionData resetSubData = new SubscriptionData();
+ resetSubData.setExpressionType(ExpressionType.SQL92);
+ resetSubData.setTopic(topic);
+ resetSubData.setClassFilterMode(false);
+ resetSubData.setSubString("a is not null OR a is null");
+
+ ConsumerFilterData resetFilterData = ConsumerFilterManager.build(topic,
+ resetGroup, resetSubData.getSubString(), resetSubData.getExpressionType(),
+ System.currentTimeMillis());
+
+ GetMessageResult resetGetResult = master.getMessage(resetGroup, topic, queueId, 0, 1000,
+ new ExpressionMessageFilter(resetSubData, resetFilterData, filterManager));
+
+ try {
+ assertThat(resetGetResult).isNotNull();
+
+ List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, resetFilterData);
+
+ assertThat(resetGetResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());
+ } finally {
+ resetGetResult.release();
+ }
+ }
+
+ {
+ ConsumerFilterData normalFilterData = filterManager.get(topic, normalGroup);
+ assertThat(normalFilterData).isNotNull();
+ assertThat(normalFilterData.getBornTime()).isLessThan(System.currentTimeMillis());
+
+ SubscriptionData normalSubData = new SubscriptionData();
+ normalSubData.setExpressionType(normalFilterData.getExpressionType());
+ normalSubData.setTopic(topic);
+ normalSubData.setClassFilterMode(false);
+ normalSubData.setSubString(normalFilterData.getExpression());
+
+ List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, normalFilterData);
+
+ GetMessageResult normalGetResult = master.getMessage(normalGroup, topic, queueId, 0, 1000,
+ new ExpressionMessageFilter(normalSubData, normalFilterData, filterManager));
+
+ try {
+ assertThat(normalGetResult).isNotNull();
+ assertThat(normalGetResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());
+ } finally {
+ normalGetResult.release();
+ }
+ }
+ } finally {
+ master.shutdown();
+ master.destroy();
+ deleteDirectory(storePath);
+ }
+ }
+
+ @Test
+ public void testGetMessage_withFilterBitMap() {
+ int topicCount = 10, msgPerTopic = 500;
+ ConsumerFilterManager filterManager = ConsumerFilterManagerTest.gen(topicCount, msgPerTopic);
+
+ DefaultMessageStore master = null;
+ try {
+ master = gen(filterManager);
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+
+ try {
+ List<MessageExtBrokerInner> msgs = null;
+ try {
+ msgs = putMsg(master, topicCount, msgPerTopic);
+ // sleep to wait for consume queue has been constructed.
+ Thread.sleep(1000);
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+
+ for (int i = 0; i < topicCount; i++) {
+ String realTopic = topic + i;
+
+ for (int j = 0; j < msgPerTopic; j++) {
+ String group = "CID_" + j;
+
+ ConsumerFilterData filterData = filterManager.get(realTopic, group);
+ assertThat(filterData).isNotNull();
+
+ List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, filterData);
+
+ SubscriptionData subscriptionData = new SubscriptionData();
+ subscriptionData.setExpressionType(filterData.getExpressionType());
+ subscriptionData.setTopic(filterData.getTopic());
+ subscriptionData.setClassFilterMode(false);
+ subscriptionData.setSubString(filterData.getExpression());
+
+ GetMessageResult getMessageResult = master.getMessage(group, realTopic, queueId, 0, 10000,
+ new ExpressionMessageFilter(subscriptionData, filterData, filterManager));
+ String assertMsg = group + "-" + realTopic;
+ try {
+ assertThat(getMessageResult).isNotNull();
+ assertThat(GetMessageStatus.FOUND).isEqualTo(getMessageResult.getStatus());
+ assertThat(getMessageResult.getMessageBufferList()).isNotNull().isNotEmpty();
+ assertThat(getMessageResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());
+
+ for (ByteBuffer buffer : getMessageResult.getMessageBufferList()) {
+ MessageExt messageExt = MessageDecoder.decode(buffer.slice(), false);
+ assertThat(messageExt).isNotNull();
+
+ Object evlRet = null;
+ try {
+ evlRet = filterData.getCompiledExpression().evaluate(new MessageEvaluationContext(messageExt.getProperties()));
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(true).isFalse();
+ }
+
+ assertThat(evlRet).isNotNull().isEqualTo(Boolean.TRUE);
+
+ // check
+ boolean find = false;
+ for (MessageExtBrokerInner messageExtBrokerInner : filteredMsgs) {
+ if (messageExtBrokerInner.getMsgId().equals(messageExt.getMsgId())) {
+ find = true;
+ }
+ }
+ assertThat(find).isTrue();
+ }
+ } finally {
+ getMessageResult.release();
+ }
+ }
+ }
+ } finally {
+ master.shutdown();
+ master.destroy();
+ deleteDirectory(storePath);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java
----------------------------------------------------------------------
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java
index d3d9812..941d4a7 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Set;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
import org.apache.rocketmq.common.BrokerConfig;
@@ -126,7 +127,7 @@ public class PullMessageProcessorTest {
@Test
public void testProcessRequest_Found() throws RemotingCommandException {
GetMessageResult getMessageResult = createGetMessageResult();
- when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(SubscriptionData.class))).thenReturn(getMessageResult);
+ when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult);
final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);
RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);
@@ -137,7 +138,7 @@ public class PullMessageProcessorTest {
@Test
public void testProcessRequest_FoundWithHook() throws RemotingCommandException {
GetMessageResult getMessageResult = createGetMessageResult();
- when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(SubscriptionData.class))).thenReturn(getMessageResult);
+ when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult);
List<ConsumeMessageHook> consumeMessageHookList = new ArrayList<>();
final ConsumeMessageContext[] messageContext = new ConsumeMessageContext[1];
ConsumeMessageHook consumeMessageHook = new ConsumeMessageHook() {
@@ -168,7 +169,7 @@ public class PullMessageProcessorTest {
public void testProcessRequest_MsgWasRemoving() throws RemotingCommandException {
GetMessageResult getMessageResult = createGetMessageResult();
getMessageResult.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING);
- when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(SubscriptionData.class))).thenReturn(getMessageResult);
+ when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult);
final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);
RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);
@@ -180,7 +181,7 @@ public class PullMessageProcessorTest {
public void testProcessRequest_NoMsgInQueue() throws RemotingCommandException {
GetMessageResult getMessageResult = createGetMessageResult();
getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);
- when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(SubscriptionData.class))).thenReturn(getMessageResult);
+ when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult);
final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);
RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
index 3903fe2..9c9b59e 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
@@ -519,6 +519,21 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
}
/**
+ * Subscribe a topic by message selector.
+ *
+ * @see org.apache.rocketmq.client.consumer.MessageSelector#bySql
+ * @see org.apache.rocketmq.client.consumer.MessageSelector#byTag
+ *
+ * @param topic topic to consume.
+ * @param messageSelector {@link org.apache.rocketmq.client.consumer.MessageSelector}
+ * @throws MQClientException
+ */
+ @Override
+ public void subscribe(final String topic, final MessageSelector messageSelector) throws MQClientException {
+ this.defaultMQPushConsumerImpl.subscribe(topic, messageSelector);
+ }
+
+ /**
* Un-subscribe the specified topic from subscription.
* @param topic message topic
*/
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPushConsumer.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPushConsumer.java
index 9255281..9c6c1f1 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPushConsumer.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPushConsumer.java
@@ -70,6 +70,27 @@ public interface MQPushConsumer extends MQConsumer {
void subscribe(final String topic, final String fullClassName, final String filterClassSource) throws MQClientException;
/**
+ * Subscribe some topic with selector.
+ * <p>
+ * This interface also has the ability of {@link #subscribe(String, String)},
+ * and, support other message selection, such as {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92}.
+ * </p>
+ * <p/>
+ * <p>
+ * Choose Tag: {@link MessageSelector#byTag(java.lang.String)}
+ * </p>
+ * <p/>
+ * <p>
+ * Choose SQL92: {@link MessageSelector#bySql(java.lang.String)}
+ * </p>
+ *
+ * @param topic
+ * @param selector message selector({@link MessageSelector}), can be null.
+ * @throws MQClientException
+ */
+ void subscribe(final String topic, final MessageSelector selector) throws MQClientException;
+
+ /**
* Unsubscribe consumption some topic
*
* @param topic message topic
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/client/src/main/java/org/apache/rocketmq/client/consumer/MessageSelector.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MessageSelector.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MessageSelector.java
new file mode 100644
index 0000000..35a5181
--- /dev/null
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MessageSelector.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.client.consumer;
+
+import org.apache.rocketmq.common.filter.ExpressionType;
+
+/**
+ *
+ * Message selector: select message at server.
+ * <p>
+ * Now, support:
+ * <li>Tag: {@link org.apache.rocketmq.common.filter.ExpressionType#TAG}
+ * </li>
+ * <li>SQL92: {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92}
+ * </li>
+ * </p>
+ */
+public class MessageSelector {
+
+ /**
+ * @see org.apache.rocketmq.common.filter.ExpressionType
+ */
+ private String type;
+
+ /**
+ * expression content.
+ */
+ private String expression;
+
+ private MessageSelector(String type, String expression) {
+ this.type = type;
+ this.expression = expression;
+ }
+
+ /**
+ * Use SLQ92 to select message.
+ *
+ * @param sql if null or empty, will be treated as select all message.
+ * @return
+ */
+ public static MessageSelector bySql(String sql) {
+ return new MessageSelector(ExpressionType.SQL92, sql);
+ }
+
+ /**
+ * Use tag to select message.
+ *
+ * @param tag if null or empty or "*", will be treated as select all message.
+ * @return
+ */
+ public static MessageSelector byTag(String tag) {
+ return new MessageSelector(ExpressionType.TAG, tag);
+ }
+
+ public String getExpressionType() {
+ return type;
+ }
+
+ public String getExpression() {
+ return expression;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/client/src/main/java/org/apache/rocketmq/client/impl/FindBrokerResult.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/FindBrokerResult.java b/client/src/main/java/org/apache/rocketmq/client/impl/FindBrokerResult.java
index 295060e..4367a4c 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/FindBrokerResult.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/FindBrokerResult.java
@@ -19,10 +19,18 @@ package org.apache.rocketmq.client.impl;
public class FindBrokerResult {
private final String brokerAddr;
private final boolean slave;
+ private final int brokerVersion;
public FindBrokerResult(String brokerAddr, boolean slave) {
this.brokerAddr = brokerAddr;
this.slave = slave;
+ this.brokerVersion = 0;
+ }
+
+ public FindBrokerResult(String brokerAddr, boolean slave, int brokerVersion) {
+ this.brokerAddr = brokerAddr;
+ this.slave = slave;
+ this.brokerVersion = brokerVersion;
}
public String getBrokerAddr() {
@@ -32,4 +40,8 @@ public class FindBrokerResult {
public boolean isSlave() {
return slave;
}
+
+ public int getBrokerVersion() {
+ return brokerVersion;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
index ff25334..4244bdd 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
@@ -59,6 +59,7 @@ import org.apache.rocketmq.common.namesrv.TopAddressing;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.body.BrokerStatsData;
+import org.apache.rocketmq.common.protocol.body.CheckClientRequestBody;
import org.apache.rocketmq.common.protocol.body.ClusterInfo;
import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult;
import org.apache.rocketmq.common.protocol.body.ConsumeStatsList;
@@ -70,6 +71,7 @@ import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody;
import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueryConsumeTimeSpanBody;
import org.apache.rocketmq.common.protocol.body.QueryCorrectionOffsetBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
@@ -103,6 +105,7 @@ import org.apache.rocketmq.common.protocol.header.GetTopicStatsInfoRequestHeader
import org.apache.rocketmq.common.protocol.header.GetTopicsByClusterRequestHeader;
import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader;
+import org.apache.rocketmq.common.protocol.header.QueryConsumeQueueRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryConsumeTimeSpanRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader;
@@ -129,6 +132,7 @@ import org.apache.rocketmq.common.protocol.header.namesrv.PutKVConfigRequestHead
import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader;
import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.remoting.InvokeCallback;
@@ -168,7 +172,7 @@ public class MQClientAPIImpl {
public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, final ClientRemotingProcessor clientRemotingProcessor,
RPCHook rpcHook, final ClientConfig clientConfig) {
this.clientConfig = clientConfig;
- topAddressing = new TopAddressing(MixAll.WS_ADDR, clientConfig.getUnitName());
+ topAddressing = new TopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName());
this.remotingClient = new NettyRemotingClient(nettyClientConfig, null);
this.clientRemotingProcessor = clientRemotingProcessor;
@@ -843,7 +847,7 @@ public class MQClientAPIImpl {
this.remotingClient.invokeOneway(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis);
}
- public void sendHearbeat(//
+ public int sendHearbeat(//
final String addr, //
final HeartbeatData heartbeatData, //
final long timeoutMillis//
@@ -855,7 +859,7 @@ public class MQClientAPIImpl {
assert response != null;
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
- return;
+ return response.getVersion();
}
default:
break;
@@ -2024,4 +2028,51 @@ public class MQClientAPIImpl {
return configMap;
}
+ public QueryConsumeQueueResponseBody queryConsumeQueue(final String brokerAddr, final String topic, final int queueId,
+ final long index, final int count, final String consumerGroup,
+ final long timeoutMillis) throws InterruptedException,
+ RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
+
+ QueryConsumeQueueRequestHeader requestHeader = new QueryConsumeQueueRequestHeader();
+ requestHeader.setTopic(topic);
+ requestHeader.setQueueId(queueId);
+ requestHeader.setIndex(index);
+ requestHeader.setCount(count);
+ requestHeader.setConsumerGroup(consumerGroup);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUME_QUEUE, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);
+
+ assert response != null;
+
+ if (ResponseCode.SUCCESS == response.getCode()) {
+ return QueryConsumeQueueResponseBody.decode(response.getBody(), QueryConsumeQueueResponseBody.class);
+ }
+
+ throw new MQClientException(response.getCode(), response.getRemark());
+ }
+
+ public void checkClientInBroker(final String brokerAddr, final String consumerGroup,
+ final String clientId, final SubscriptionData subscriptionData,
+ final long timeoutMillis)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException,
+ RemotingConnectException, MQClientException {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_CLIENT_CONFIG, null);
+
+ CheckClientRequestBody requestBody = new CheckClientRequestBody();
+ requestBody.setClientId(clientId);
+ requestBody.setGroup(consumerGroup);
+ requestBody.setSubscriptionData(subscriptionData);
+
+ request.setBody(requestBody.encode());
+
+ RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);
+
+ assert response != null;
+
+ if (ResponseCode.SUCCESS != response.getCode()) {
+ throw new MQClientException(response.getCode(), response.getRemark());
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
index 67f3ebe..2cafe29 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
@@ -30,6 +30,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.rocketmq.client.QueryResult;
import org.apache.rocketmq.client.Validators;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.MessageSelector;
import org.apache.rocketmq.client.consumer.PullCallback;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.client.consumer.listener.MessageListener;
@@ -405,15 +406,16 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
this.pullAPIWrapper.pullKernelImpl(//
pullRequest.getMessageQueue(), // 1
subExpression, // 2
- subscriptionData.getSubVersion(), // 3
- pullRequest.getNextOffset(), // 4
- this.defaultMQPushConsumer.getPullBatchSize(), // 5
- sysFlag, // 6
- commitOffsetValue, // 7
- BROKER_SUSPEND_MAX_TIME_MILLIS, // 8
- CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND, // 9
- CommunicationMode.ASYNC, // 10
- pullCallback// 11
+ subscriptionData.getExpressionType(), // 3
+ subscriptionData.getSubVersion(), // 4
+ pullRequest.getNextOffset(), // 5
+ this.defaultMQPushConsumer.getPullBatchSize(), // 6
+ sysFlag, // 7
+ commitOffsetValue, // 8
+ BROKER_SUSPEND_MAX_TIME_MILLIS, // 9
+ CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND, // 10
+ CommunicationMode.ASYNC, // 11
+ pullCallback // 12
);
} catch (Exception e) {
log.error("pullKernelImpl exception", e);
@@ -615,6 +617,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
}
this.updateTopicSubscribeInfoWhenSubscriptionChanged();
+ this.mQClientFactory.checkClientInBroker();
this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();
this.mQClientFactory.rebalanceImmediately();
}
@@ -836,6 +839,25 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
}
}
+ public void subscribe(final String topic, final MessageSelector messageSelector) throws MQClientException {
+ try {
+ if (messageSelector == null) {
+ subscribe(topic, SubscriptionData.SUB_ALL);
+ return;
+ }
+
+ SubscriptionData subscriptionData = FilterAPI.build(topic,
+ messageSelector.getExpression(), messageSelector.getExpressionType());
+
+ this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);
+ if (this.mQClientFactory != null) {
+ this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();
+ }
+ } catch (Exception e) {
+ throw new MQClientException("subscription exception", e);
+ }
+ }
+
public void suspend() {
this.pause = true;
log.info("suspend this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup());
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java
index 96e21e1..304a44a 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java
@@ -33,7 +33,9 @@ import org.apache.rocketmq.client.impl.CommunicationMode;
import org.apache.rocketmq.client.impl.FindBrokerResult;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.common.MQVersion;
import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
@@ -135,6 +137,7 @@ public class PullAPIWrapper {
public PullResult pullKernelImpl(
final MessageQueue mq,
final String subExpression,
+ final String expressionType,
final long subVersion,
final long offset,
final int maxNums,
@@ -156,6 +159,14 @@ public class PullAPIWrapper {
}
if (findBrokerResult != null) {
+ {
+ // check version
+ if (!ExpressionType.isTagType(expressionType)
+ && findBrokerResult.getBrokerVersion() < MQVersion.Version.V4_1_0_SNAPSHOT.ordinal()) {
+ throw new MQClientException("The broker[" + mq.getBrokerName() + ", "
+ + findBrokerResult.getBrokerVersion() + "] does not upgrade to support for filter message by " + expressionType, null);
+ }
+ }
int sysFlagInner = sysFlag;
if (findBrokerResult.isSlave()) {
@@ -173,6 +184,7 @@ public class PullAPIWrapper {
requestHeader.setSuspendTimeoutMillis(brokerSuspendMaxTimeMillis);
requestHeader.setSubscription(subExpression);
requestHeader.setSubVersion(subVersion);
+ requestHeader.setExpressionType(expressionType);
String brokerAddr = findBrokerResult.getBrokerAddr();
if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) {
@@ -192,6 +204,34 @@ public class PullAPIWrapper {
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
+ public PullResult pullKernelImpl(
+ final MessageQueue mq,
+ final String subExpression,
+ final long subVersion,
+ final long offset,
+ final int maxNums,
+ final int sysFlag,
+ final long commitOffset,
+ final long brokerSuspendMaxTimeMillis,
+ final long timeoutMillis,
+ final CommunicationMode communicationMode,
+ final PullCallback pullCallback
+ ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
+ return pullKernelImpl(
+ mq,
+ subExpression,
+ ExpressionType.TAG,
+ subVersion, offset,
+ maxNums,
+ sysFlag,
+ commitOffset,
+ brokerSuspendMaxTimeMillis,
+ timeoutMillis,
+ communicationMode,
+ pullCallback
+ );
+ }
+
public long recalculatePullFromWhichNode(final MessageQueue mq) {
if (this.isConnectBrokerByUser()) {
return this.defaultBrokerId;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
index d7e02fe..a8c65b2 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
@@ -61,6 +61,7 @@ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.ServiceState;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult;
@@ -98,6 +99,8 @@ public class MQClientInstance {
private final Lock lockHeartbeat = new ReentrantLock();
private final ConcurrentHashMap<String/* Broker Name */, HashMap<Long/* brokerId */, String/* address */>> brokerAddrTable =
new ConcurrentHashMap<String, HashMap<Long, String>>();
+ private final ConcurrentHashMap<String/* Broker Name */, HashMap<String/* address */, Integer>> brokerVersionTable =
+ new ConcurrentHashMap<String, HashMap<String, Integer>>();
private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
@@ -404,6 +407,44 @@ public class MQClientInstance {
}
}
+ public void checkClientInBroker() throws MQClientException {
+ Iterator<Entry<String, MQConsumerInner>> it = this.consumerTable.entrySet().iterator();
+
+ while (it.hasNext()) {
+ Entry<String, MQConsumerInner> entry = it.next();
+ Set<SubscriptionData> subscriptionInner = entry.getValue().subscriptions();
+ if (subscriptionInner == null || subscriptionInner.isEmpty()) {
+ return;
+ }
+
+ for (SubscriptionData subscriptionData : subscriptionInner) {
+ if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {
+ continue;
+ }
+ // may need to check one broker every cluster...
+ // assume that the configs of every broker in cluster are the the same.
+ String addr = findBrokerAddrByTopic(subscriptionData.getTopic());
+
+ if (addr != null) {
+ try {
+ this.getMQClientAPIImpl().checkClientInBroker(
+ addr, entry.getKey(), this.clientId, subscriptionData, 3 * 1000
+ );
+ } catch (Exception e) {
+ if (e instanceof MQClientException) {
+ throw (MQClientException) e;
+ } else {
+ throw new MQClientException("Check client in broker error, maybe because you use "
+ + subscriptionData.getExpressionType() + " to filter message, but server has not been upgraded to support!"
+ + "This error would not affect the launch of consumer, but may has impact on message receiving if you " +
+ "have use the new features which are not supported by server, please check the log!", e);
+ }
+ }
+ }
+ }
+ }
+ }
+
public void sendHeartbeatToAllBrokerWithLock() {
if (this.lockHeartbeat.tryLock()) {
try {
@@ -493,7 +534,11 @@ public class MQClientInstance {
}
try {
- this.mQClientAPIImpl.sendHearbeat(addr, heartbeatData, 3000);
+ int version = this.mQClientAPIImpl.sendHearbeat(addr, heartbeatData, 3000);
+ if (!this.brokerVersionTable.containsKey(brokerName)) {
+ this.brokerVersionTable.put(brokerName, new HashMap<String, Integer>(4));
+ }
+ this.brokerVersionTable.get(brokerName).put(addr, version);
if (times % 20 == 0) {
log.info("send heart beat to broker[{} {} {}] success", brokerName, id, addr);
log.info(heartbeatData.toString());
@@ -943,7 +988,7 @@ public class MQClientInstance {
}
if (found) {
- return new FindBrokerResult(brokerAddr, slave);
+ return new FindBrokerResult(brokerAddr, slave, findBrokerVersion(brokerName, brokerAddr));
}
return null;
@@ -982,12 +1027,21 @@ public class MQClientInstance {
}
if (found) {
- return new FindBrokerResult(brokerAddr, slave);
+ return new FindBrokerResult(brokerAddr, slave, findBrokerVersion(brokerName, brokerAddr));
}
return null;
}
+ public int findBrokerVersion(String brokerName, String brokerAddr) {
+ if (this.brokerVersionTable.containsKey(brokerName)) {
+ if (this.brokerVersionTable.get(brokerName).containsKey(brokerAddr)) {
+ return this.brokerVersionTable.get(brokerName).get(brokerAddr);
+ }
+ }
+ return 0;
+ }
+
public List<String> findConsumerIdList(final String topic, final String group) {
String brokerAddr = this.findBrokerAddrByTopic(topic);
if (null == brokerAddr) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
index f79f726..f0a73bd 100644
--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
+++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
@@ -99,6 +99,25 @@ public class BrokerConfig {
private boolean traceOn = true;
+ // Switch of filter bit map calculation.
+ // If switch on:
+ // 1. Calculate filter bit map when construct queue.
+ // 2. Filter bit map will be saved to consume queue extend file if allowed.
+ private boolean enableCalcFilterBitMap = false;
+
+ // Expect num of consumers will use filter.
+ private int expectConsumerNumUseFilter = 32;
+
+ // Error rate of bloom filter, 1~100.
+ private int maxErrorRateOfBloomFilter = 20;
+
+ //how long to clean filter data after dead.Default: 24h
+ private long filterDataCleanTimeSpan = 24 * 3600 * 1000;
+
+ // whether do filter when retry.
+ private boolean filterSupportRetry = false;
+ private boolean enablePropertyFilter = false;
+
public static String localHostName() {
try {
return InetAddress.getLocalHost().getHostName();
@@ -484,4 +503,52 @@ public class BrokerConfig {
public void setCommercialBaseCount(int commercialBaseCount) {
this.commercialBaseCount = commercialBaseCount;
}
+
+ public boolean isEnableCalcFilterBitMap() {
+ return enableCalcFilterBitMap;
+ }
+
+ public void setEnableCalcFilterBitMap(boolean enableCalcFilterBitMap) {
+ this.enableCalcFilterBitMap = enableCalcFilterBitMap;
+ }
+
+ public int getExpectConsumerNumUseFilter() {
+ return expectConsumerNumUseFilter;
+ }
+
+ public void setExpectConsumerNumUseFilter(int expectConsumerNumUseFilter) {
+ this.expectConsumerNumUseFilter = expectConsumerNumUseFilter;
+ }
+
+ public int getMaxErrorRateOfBloomFilter() {
+ return maxErrorRateOfBloomFilter;
+ }
+
+ public void setMaxErrorRateOfBloomFilter(int maxErrorRateOfBloomFilter) {
+ this.maxErrorRateOfBloomFilter = maxErrorRateOfBloomFilter;
+ }
+
+ public long getFilterDataCleanTimeSpan() {
+ return filterDataCleanTimeSpan;
+ }
+
+ public void setFilterDataCleanTimeSpan(long filterDataCleanTimeSpan) {
+ this.filterDataCleanTimeSpan = filterDataCleanTimeSpan;
+ }
+
+ public boolean isFilterSupportRetry() {
+ return filterSupportRetry;
+ }
+
+ public void setFilterSupportRetry(boolean filterSupportRetry) {
+ this.filterSupportRetry = filterSupportRetry;
+ }
+
+ public boolean isEnablePropertyFilter() {
+ return enablePropertyFilter;
+ }
+
+ public void setEnablePropertyFilter(boolean enablePropertyFilter) {
+ this.enablePropertyFilter = enablePropertyFilter;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
index 4a54a60..e75efd9 100644
--- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java
+++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
@@ -55,8 +55,8 @@ public class MixAll {
public static final String DEFAULT_NAMESRV_ADDR_LOOKUP = "jmenv.tbsite.net";
public static final String WS_DOMAIN_NAME = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP);
public static final String WS_DOMAIN_SUBGROUP = System.getProperty("rocketmq.namesrv.domain.subgroup", "nsaddr");
- // http://jmenv.tbsite.net:8080/rocketmq/nsaddr
- public static final String WS_ADDR = "http://" + WS_DOMAIN_NAME + ":8080/rocketmq/" + WS_DOMAIN_SUBGROUP;
+// // http://jmenv.tbsite.net:8080/rocketmq/nsaddr
+// public static final String WS_ADDR = "http://" + WS_DOMAIN_NAME + ":8080/rocketmq/" + WS_DOMAIN_SUBGROUP;
public static final String DEFAULT_TOPIC = "TBW102";
public static final String BENCHMARK_TOPIC = "BenchmarkTest";
public static final String DEFAULT_PRODUCER_GROUP = "DEFAULT_PRODUCER";
@@ -89,6 +89,16 @@ public class MixAll {
public static final String DEFAULT_TRACE_REGION_ID = "DefaultRegion";
public static final String CONSUME_CONTEXT_TYPE = "ConsumeContextType";
+ public static String getWSAddr() {
+ String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP);
+ String wsDomainSubgroup = System.getProperty("rocketmq.namesrv.domain.subgroup", "nsaddr");
+ String wsAddr = "http://" + wsDomainName + ":8080/rocketmq/" + wsDomainSubgroup;
+ if (wsDomainName.indexOf(":") > 0) {
+ wsAddr = "http://" + wsDomainName + "/rocketmq/" + wsDomainSubgroup;
+ }
+ return wsAddr;
+ }
+
public static String getRetryTopic(final String consumerGroup) {
return RETRY_GROUP_TOPIC_PREFIX + consumerGroup;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
index e706e28..385c121 100644
--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
+++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
@@ -34,4 +34,5 @@ public class LoggerName {
public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication";
public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection";
public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark";
+ public static final String FILTER_LOGGER_NAME = "RocketmqFilter";
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/filter/ExpressionType.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/filter/ExpressionType.java b/common/src/main/java/org/apache/rocketmq/common/filter/ExpressionType.java
new file mode 100644
index 0000000..3b7940a
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/filter/ExpressionType.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.filter;
+
+public class ExpressionType {
+
+ /**
+ * <ul>
+ * Keywords:
+ * <li>{@code AND, OR, NOT, BETWEEN, IN, TRUE, FALSE, IS, NULL}</li>
+ * </ul>
+ * <p/>
+ * <ul>
+ * Data type:
+ * <li>Boolean, like: TRUE, FALSE</li>
+ * <li>String, like: 'abc'</li>
+ * <li>Decimal, like: 123</li>
+ * <li>Float number, like: 3.1415</li>
+ * </ul>
+ * <p/>
+ * <ul>
+ * Grammar:
+ * <li>{@code AND, OR}</li>
+ * <li>{@code >, >=, <, <=, =}</li>
+ * <li>{@code BETWEEN A AND B}, equals to {@code >=A AND <=B}</li>
+ * <li>{@code NOT BETWEEN A AND B}, equals to {@code >B OR <A}</li>
+ * <li>{@code IN ('a', 'b')}, equals to {@code ='a' OR ='b'}, this operation only support String type.</li>
+ * <li>{@code IS NULL}, {@code IS NOT NULL}, check parameter whether is null, or not.</li>
+ * <li>{@code =TRUE}, {@code =FALSE}, check parameter whether is true, or false.</li>
+ * </ul>
+ * <p/>
+ * <p>
+ * Example:
+ * (a > 10 AND a < 100) OR (b IS NOT NULL AND b=TRUE)
+ * </p>
+ */
+ public static final String SQL92 = "SQL92";
+
+ /**
+ * Only support or operation such as
+ * "tag1 || tag2 || tag3", <br>
+ * If null or * expression,meaning subscribe all.
+ */
+ public static final String TAG = "TAG";
+
+ public static boolean isTagType(String type) {
+ if (type == null || TAG.equals(type)) {
+ return true;
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/filter/FilterAPI.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/filter/FilterAPI.java b/common/src/main/java/org/apache/rocketmq/common/filter/FilterAPI.java
index e9bf3fa..fc8525c 100644
--- a/common/src/main/java/org/apache/rocketmq/common/filter/FilterAPI.java
+++ b/common/src/main/java/org/apache/rocketmq/common/filter/FilterAPI.java
@@ -63,4 +63,22 @@ public class FilterAPI {
return subscriptionData;
}
+
+ public static SubscriptionData build(final String topic, final String subString,
+ final String type) throws Exception {
+ if (ExpressionType.TAG.equals(type) || type == null) {
+ return buildSubscriptionData(null, topic, subString);
+ }
+
+ if (subString == null || subString.length() < 1) {
+ throw new IllegalArgumentException("Expression can't be null! " + type);
+ }
+
+ SubscriptionData subscriptionData = new SubscriptionData();
+ subscriptionData.setTopic(topic);
+ subscriptionData.setSubString(subString);
+ subscriptionData.setExpressionType(type);
+
+ return subscriptionData;
+ }
}
[39/50] [abbrv] incubator-rocketmq git commit: Remove diamond
operator for client module with JDK 1.6
Posted by do...@apache.org.
Remove diamond operator for client module with JDK 1.6
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/8d781757
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/8d781757
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/8d781757
Branch: refs/heads/release-4.1.0-incubating
Commit: 8d781757dfb56ebd95b0494e8e2c87422c22d767
Parents: e57f9ac
Author: dongeforever <zh...@yeah.net>
Authored: Sat May 27 14:09:08 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 14:09:08 2017 +0800
----------------------------------------------------------------------
.../AllocateMessageQueueConsistentHash.java | 8 +++---
.../AllocateMessageQueueConsitentHashTest.java | 27 ++++++++++----------
.../consistenthash/ConsistentHashRouter.java | 4 +--
.../org/apache/rocketmq/store/CommitLog.java | 4 +--
4 files changed, 22 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/8d781757/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java
index 77198b7..09d940a 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java
@@ -76,19 +76,19 @@ public class AllocateMessageQueueConsistentHash implements AllocateMessageQueue
}
- Collection<ClientNode> cidNodes = new ArrayList<>();
+ Collection<ClientNode> cidNodes = new ArrayList<ClientNode>();
for (String cid : cidAll) {
cidNodes.add(new ClientNode(cid));
}
final ConsistentHashRouter<ClientNode> router; //for building hash ring
if (customHashFunction != null) {
- router = new ConsistentHashRouter<>(cidNodes, virtualNodeCnt, customHashFunction);
+ router = new ConsistentHashRouter<ClientNode>(cidNodes, virtualNodeCnt, customHashFunction);
} else {
- router = new ConsistentHashRouter<>(cidNodes, virtualNodeCnt);
+ router = new ConsistentHashRouter<ClientNode>(cidNodes, virtualNodeCnt);
}
- List<MessageQueue> results = new ArrayList<>();
+ List<MessageQueue> results = new ArrayList<MessageQueue>();
for (MessageQueue mq : mqAll) {
ClientNode clientNode = router.routeNode(mq.toString());
if (clientNode != null && currentCID.equals(clientNode.getKey())) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/8d781757/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java
----------------------------------------------------------------------
diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java
index fc7ab9f..e9e5db7 100644
--- a/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Random;
import java.util.TreeMap;
import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
+import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.junit.Assert;
import org.junit.Before;
@@ -113,13 +114,13 @@ public class AllocateMessageQueueConsitentHashTest {
//System.out.println("mqAll:" + mqAll.toString());
List<String> cidAll = createConsumerIdList(consumerSize);
- List<MessageQueue> allocatedResAll = new ArrayList<>();
+ List<MessageQueue> allocatedResAll = new ArrayList<MessageQueue>();
- Map<MessageQueue, String> allocateToAllOrigin = new TreeMap<>();
+ Map<MessageQueue, String> allocateToAllOrigin = new TreeMap<MessageQueue, String>();
//test allocate all
{
- List<String> cidBegin = new ArrayList<>(cidAll);
+ List<String> cidBegin = new ArrayList<String>(cidAll);
//System.out.println("cidAll:" + cidBegin.toString());
for (String cid : cidBegin) {
@@ -135,13 +136,13 @@ public class AllocateMessageQueueConsitentHashTest {
verifyAllocateAll(cidBegin,mqAll, allocatedResAll));
}
- Map<MessageQueue, String> allocateToAllAfterRemoveOne = new TreeMap<>();
- List<String> cidAfterRemoveOne = new ArrayList<>(cidAll);
+ Map<MessageQueue, String> allocateToAllAfterRemoveOne = new TreeMap<MessageQueue, String>();
+ List<String> cidAfterRemoveOne = new ArrayList<String>(cidAll);
//test allocate remove one cid
{
String removeCID = cidAfterRemoveOne.remove(0);
//System.out.println("removing one cid "+removeCID);
- List<MessageQueue> mqShouldOnlyChanged = new ArrayList<>();
+ List<MessageQueue> mqShouldOnlyChanged = new ArrayList<MessageQueue>();
Iterator<Map.Entry<MessageQueue, String>> it = allocateToAllOrigin.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<MessageQueue, String> entry = it.next();
@@ -151,7 +152,7 @@ public class AllocateMessageQueueConsitentHashTest {
}
//System.out.println("cidAll:" + cidAfterRemoveOne.toString());
- List<MessageQueue> allocatedResAllAfterRemove = new ArrayList<>();
+ List<MessageQueue> allocatedResAllAfterRemove = new ArrayList<MessageQueue>();
for (String cid : cidAfterRemoveOne) {
List<MessageQueue> rs = allocateMessageQueueConsistentHash.allocate("testConsumerGroup", cid, mqAll, cidAfterRemoveOne);
allocatedResAllAfterRemove.addAll(rs);
@@ -166,16 +167,16 @@ public class AllocateMessageQueueConsitentHashTest {
verifyAfterRemove(allocateToAllOrigin, allocateToAllAfterRemoveOne, removeCID);
}
- List<String> cidAfterAdd = new ArrayList<>(cidAfterRemoveOne);
+ List<String> cidAfterAdd = new ArrayList<String>(cidAfterRemoveOne);
//test allocate add one more cid
{
String newCid = CID_PREFIX+"NEW";
//System.out.println("add one more cid "+newCid);
cidAfterAdd.add(newCid);
- List<MessageQueue> mqShouldOnlyChanged = new ArrayList<>();
+ List<MessageQueue> mqShouldOnlyChanged = new ArrayList<MessageQueue>();
//System.out.println("cidAll:" + cidAfterAdd.toString());
- List<MessageQueue> allocatedResAllAfterAdd = new ArrayList<>();
- Map<MessageQueue, String> allocateToAll3 = new TreeMap<>();
+ List<MessageQueue> allocatedResAllAfterAdd = new ArrayList<MessageQueue>();
+ Map<MessageQueue, String> allocateToAll3 = new TreeMap<MessageQueue, String>();
for (String cid : cidAfterAdd) {
List<MessageQueue> rs = allocateMessageQueueConsistentHash.allocate("testConsumerGroup", cid, mqAll, cidAfterAdd);
allocatedResAllAfterAdd.addAll(rs);
@@ -225,7 +226,7 @@ public class AllocateMessageQueueConsitentHashTest {
}
private List<String> createConsumerIdList(int size) {
- List<String> consumerIdList = new ArrayList<>(size);
+ List<String> consumerIdList = new ArrayList<String>(size);
for (int i = 0; i < size; i++) {
consumerIdList.add(CID_PREFIX + String.valueOf(i));
}
@@ -233,7 +234,7 @@ public class AllocateMessageQueueConsitentHashTest {
}
private List<MessageQueue> createMessageQueueList(int size) {
- List<MessageQueue> messageQueueList = new ArrayList<>(size);
+ List<MessageQueue> messageQueueList = new ArrayList<MessageQueue>(size);
for (int i = 0; i < size; i++) {
MessageQueue mq = new MessageQueue(topic, "brokerName", i);
messageQueueList.add(mq);
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/8d781757/common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java b/common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java
index 8606c43..a6fce51 100644
--- a/common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java
+++ b/common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java
@@ -30,7 +30,7 @@ import java.util.TreeMap;
* @param <T>
*/
public class ConsistentHashRouter<T extends Node> {
- private final SortedMap<Long, VirtualNode<T>> ring = new TreeMap<>();
+ private final SortedMap<Long, VirtualNode<T>> ring = new TreeMap<Long, VirtualNode<T>>();
private final HashFunction hashFunction;
public ConsistentHashRouter(Collection<T> pNodes, int vNodeCount) {
@@ -64,7 +64,7 @@ public class ConsistentHashRouter<T extends Node> {
if (vNodeCount < 0) throw new IllegalArgumentException("illegal virtual node counts :" + vNodeCount);
int existingReplicas = getExistingReplicas(pNode);
for (int i = 0; i < vNodeCount; i++) {
- VirtualNode<T> vNode = new VirtualNode<>(pNode, i + existingReplicas);
+ VirtualNode<T> vNode = new VirtualNode<T>(pNode, i + existingReplicas);
ring.put(hashFunction.hash(vNode.getKey()), vNode);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/8d781757/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
index 7b29263..b44211c 100644
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
@@ -722,7 +722,7 @@ public class CommitLog {
messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch));
- lockForPutMessage(); //spin...
+ putMessageLock.lock();
try {
long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
this.beginTimeInLock = beginLockTimestamp;
@@ -771,7 +771,7 @@ public class CommitLog {
eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
beginTimeInLock = 0;
} finally {
- releasePutMessageLock();
+ putMessageLock.unlock();
}
[05/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-121]Support
message filtering based on SQL92 closes apache/incubator-rocketmq#82
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java
index 33529da..3d33eaf 100644
--- a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java
@@ -16,6 +16,8 @@
*/
package org.apache.rocketmq.store;
+import java.util.Map;
+
public class DispatchRequest {
private final String topic;
private final int queueId;
@@ -30,6 +32,8 @@ public class DispatchRequest {
private final int sysFlag;
private final long preparedTransactionOffset;
+ private final Map<String, String> propertiesMap;
+ private byte[] bitMap;
public DispatchRequest(
final String topic,
@@ -42,7 +46,8 @@ public class DispatchRequest {
final String keys,
final String uniqKey,
final int sysFlag,
- final long preparedTransactionOffset
+ final long preparedTransactionOffset,
+ final Map<String, String> propertiesMap
) {
this.topic = topic;
this.queueId = queueId;
@@ -57,6 +62,7 @@ public class DispatchRequest {
this.sysFlag = sysFlag;
this.preparedTransactionOffset = preparedTransactionOffset;
this.success = true;
+ this.propertiesMap = propertiesMap;
}
public DispatchRequest(int size) {
@@ -81,6 +87,7 @@ public class DispatchRequest {
this.sysFlag = 0;
this.preparedTransactionOffset = 0;
this.success = false;
+ this.propertiesMap = null;
}
public DispatchRequest(int size, boolean success) {
@@ -105,6 +112,7 @@ public class DispatchRequest {
this.sysFlag = 0;
this.preparedTransactionOffset = 0;
this.success = success;
+ this.propertiesMap = null;
}
public String getTopic() {
@@ -155,4 +163,15 @@ public class DispatchRequest {
return uniqKey;
}
+ public Map<String, String> getPropertiesMap() {
+ return propertiesMap;
+ }
+
+ public byte[] getBitMap() {
+ return bitMap;
+ }
+
+ public void setBitMap(byte[] bitMap) {
+ this.bitMap = bitMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/MappedFile.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFile.java b/store/src/main/java/org/apache/rocketmq/store/MappedFile.java
index 550e578..a9a00a8 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MappedFile.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MappedFile.java
@@ -245,6 +245,31 @@ public class MappedFile extends ReferenceResource {
}
/**
+ * Content of data from offset to offset + length will be wrote to file.
+ *
+ * @param data
+ * @param offset The offset of the subarray to be used.
+ * @param length The length of the subarray to be used.
+ * @return
+ */
+ public boolean appendMessage(final byte[] data, final int offset, final int length) {
+ int currentPos = this.wrotePosition.get();
+
+ if ((currentPos + length) <= this.fileSize) {
+ try {
+ this.fileChannel.position(currentPos);
+ this.fileChannel.write(ByteBuffer.wrap(data, offset, length));
+ } catch (Throwable e) {
+ log.error("Error occurred when append message to mappedFile.", e);
+ }
+ this.wrotePosition.addAndGet(length);
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
* @param flushLeastPages
* @return The current flushed position
*/
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
index 5c6c62c..a8fa364 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
@@ -121,7 +121,7 @@ public class MappedFileQueue {
this.deleteExpiredFile(willRemoveFiles);
}
- private void deleteExpiredFile(List<MappedFile> files) {
+ void deleteExpiredFile(List<MappedFile> files) {
if (!files.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/MessageArrivingListener.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageArrivingListener.java b/store/src/main/java/org/apache/rocketmq/store/MessageArrivingListener.java
index 2523c1a..dee1bc7 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MessageArrivingListener.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MessageArrivingListener.java
@@ -17,6 +17,9 @@
package org.apache.rocketmq.store;
+import java.util.Map;
+
public interface MessageArrivingListener {
- void arriving(String topic, int queueId, long logicOffset, long tagsCode);
+ void arriving(String topic, int queueId, long logicOffset, long tagsCode,
+ long msgStoreTime, byte[] filterBitMap, Map<String, String> properties);
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/MessageFilter.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageFilter.java b/store/src/main/java/org/apache/rocketmq/store/MessageFilter.java
index 859ce99..6b34758 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MessageFilter.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MessageFilter.java
@@ -16,8 +16,30 @@
*/
package org.apache.rocketmq.store;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import java.nio.ByteBuffer;
+import java.util.Map;
public interface MessageFilter {
- boolean isMessageMatched(final SubscriptionData subscriptionData, final Long tagsCode);
+ /**
+ * match by tags code or filter bit map which is calculated when message received
+ * and stored in consume queue ext.
+ *
+ * @param tagsCode tagsCode
+ * @param cqExtUnit extend unit of consume queue
+ * @return
+ */
+ boolean isMatchedByConsumeQueue(final Long tagsCode,
+ final ConsumeQueueExt.CqExtUnit cqExtUnit);
+
+ /**
+ * match by message content which are stored in commit log.
+ * <br>{@code msgBuffer} and {@code properties} are not all null.If invoked in store,
+ * {@code properties} is null;If invoked in {@code PullRequestHoldService}, {@code msgBuffer} is null.
+ *
+ * @param msgBuffer message buffer in commit log, may be null if not invoked in store.
+ * @param properties message properties, should decode from buffer if null by yourself.
+ * @return
+ */
+ boolean isMatchedByCommitLog(final ByteBuffer msgBuffer,
+ final Map<String, String> properties);
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
index 65c546b..e841c08 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
@@ -17,10 +17,10 @@
package org.apache.rocketmq.store;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.Set;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageExtBatch;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
public interface MessageStore {
@@ -37,7 +37,7 @@ public interface MessageStore {
PutMessageResult putMessages(final MessageExtBatch messageExtBatch);
GetMessageResult getMessage(final String group, final String topic, final int queueId,
- final long offset, final int maxMsgNums, final SubscriptionData subscriptionData);
+ final long offset, final int maxMsgNums, final MessageFilter messageFilter);
long getMaxOffsetInQuque(final String topic, final int queueId);
@@ -105,4 +105,8 @@ public interface MessageStore {
long lockTimeMills();
boolean isTransientStorePoolDeficient();
+
+ LinkedList<CommitLogDispatcher> getDispatcherList();
+
+ ConsumeQueue getConsumeQueue(String topic, int queueId);
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
index 7ae2ab5..29f800c 100644
--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
+++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
@@ -34,6 +34,13 @@ public class MessageStoreConfig {
private int mapedFileSizeCommitLog = 1024 * 1024 * 1024;
// ConsumeQueue file size,default is 30W
private int mapedFileSizeConsumeQueue = 300000 * ConsumeQueue.CQ_STORE_UNIT_SIZE;
+ // enable consume queue ext
+ private boolean enableConsumeQueueExt = false;
+ // ConsumeQueue extend file size, 48M
+ private int mappedFileSizeConsumeQueueExt = 48 * 1024 * 1024;
+ // Bit count of filter bit map.
+ // this will be set by pipe of calculate filter bit map.
+ private int bitMapLengthConsumeQueueExt = 64;
// CommitLog flush interval
// flush data to disk
@@ -191,6 +198,30 @@ public class MessageStoreConfig {
this.mapedFileSizeConsumeQueue = mapedFileSizeConsumeQueue;
}
+ public boolean isEnableConsumeQueueExt() {
+ return enableConsumeQueueExt;
+ }
+
+ public void setEnableConsumeQueueExt(boolean enableConsumeQueueExt) {
+ this.enableConsumeQueueExt = enableConsumeQueueExt;
+ }
+
+ public int getMappedFileSizeConsumeQueueExt() {
+ return mappedFileSizeConsumeQueueExt;
+ }
+
+ public void setMappedFileSizeConsumeQueueExt(int mappedFileSizeConsumeQueueExt) {
+ this.mappedFileSizeConsumeQueueExt = mappedFileSizeConsumeQueueExt;
+ }
+
+ public int getBitMapLengthConsumeQueueExt() {
+ return bitMapLengthConsumeQueueExt;
+ }
+
+ public void setBitMapLengthConsumeQueueExt(int bitMapLengthConsumeQueueExt) {
+ this.bitMapLengthConsumeQueueExt = bitMapLengthConsumeQueueExt;
+ }
+
public int getFlushIntervalCommitLog() {
return flushIntervalCommitLog;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/config/StorePathConfigHelper.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/config/StorePathConfigHelper.java b/store/src/main/java/org/apache/rocketmq/store/config/StorePathConfigHelper.java
index aebebaf..ef1d670 100644
--- a/store/src/main/java/org/apache/rocketmq/store/config/StorePathConfigHelper.java
+++ b/store/src/main/java/org/apache/rocketmq/store/config/StorePathConfigHelper.java
@@ -24,6 +24,10 @@ public class StorePathConfigHelper {
return rootDir + File.separator + "consumequeue";
}
+ public static String getStorePathConsumeQueueExt(final String rootDir) {
+ return rootDir + File.separator + "consumequeue_ext";
+ }
+
public static String getStorePathIndex(final String rootDir) {
return rootDir + File.separator + "index";
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
index e08a6f5..d45b994 100644
--- a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
@@ -32,6 +32,7 @@ import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.running.RunningStats;
import org.apache.rocketmq.store.ConsumeQueue;
+import org.apache.rocketmq.store.ConsumeQueueExt;
import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.PutMessageResult;
@@ -248,11 +249,24 @@ public class ScheduleMessageService extends ConfigManager {
try {
long nextOffset = offset;
int i = 0;
+ ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
long offsetPy = bufferCQ.getByteBuffer().getLong();
int sizePy = bufferCQ.getByteBuffer().getInt();
long tagsCode = bufferCQ.getByteBuffer().getLong();
+ if (cq.isExtAddr(tagsCode)) {
+ if (cq.getExt(tagsCode, cqExtUnit)) {
+ tagsCode = cqExtUnit.getTagsCode();
+ } else {
+ //can't find ext content.So re compute tags code.
+ log.error("[BUG] can't find consume queue extend file content!addr={}, offsetPy={}, sizePy={}",
+ tagsCode, offsetPy, sizePy);
+ long msgStoreTime = defaultMessageStore.getCommitLog().pickupStoreTimestamp(offsetPy, sizePy);
+ tagsCode = computeDeliverTimestamp(delayLevel, msgStoreTime);
+ }
+ }
+
long now = System.currentTimeMillis();
long deliverTimestamp = this.correctDeliverTimestamp(now, tagsCode);
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueExtTest.java
----------------------------------------------------------------------
diff --git a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueExtTest.java b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueExtTest.java
new file mode 100644
index 0000000..5dbc584
--- /dev/null
+++ b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueExtTest.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store;
+
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Random;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ConsumeQueueExtTest {
+
+ private static final String topic = "abc";
+ private static final int queueId = 0;
+ private static final String storePath = "." + File.separator + "unit_test_store";
+ private static final int bitMapLength = 64;
+ private static final int unitSizeWithBitMap = ConsumeQueueExt.CqExtUnit.MIN_EXT_UNIT_SIZE + bitMapLength / Byte.SIZE;
+ private static final int cqExtFileSize = 10 * unitSizeWithBitMap;
+ private static final int unitCount = 20;
+
+
+ protected ConsumeQueueExt genExt() {
+ return new ConsumeQueueExt(
+ topic, queueId, storePath, cqExtFileSize, bitMapLength
+ );
+ }
+
+ protected byte[] genBitMap(int bitMapLength) {
+ byte[] bytes = new byte[bitMapLength / Byte.SIZE];
+
+ Random random = new Random(System.currentTimeMillis());
+ random.nextBytes(bytes);
+
+ return bytes;
+ }
+
+ protected ConsumeQueueExt.CqExtUnit genUnit(boolean hasBitMap) {
+ ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
+
+ cqExtUnit.setTagsCode(Math.abs((new Random(System.currentTimeMillis())).nextInt()));
+ cqExtUnit.setMsgStoreTime(System.currentTimeMillis());
+ if (hasBitMap) {
+ cqExtUnit.setFilterBitMap(genBitMap(bitMapLength));
+ }
+
+ return cqExtUnit;
+ }
+
+ protected void deleteDirectory(String rootPath) {
+ File file = new File(rootPath);
+ deleteFile(file);
+ }
+
+ protected void deleteFile(File file) {
+ File[] subFiles = file.listFiles();
+ if (subFiles != null) {
+ for (File sub : subFiles) {
+ deleteFile(sub);
+ }
+ }
+
+ file.delete();
+ }
+
+ protected void putSth(ConsumeQueueExt consumeQueueExt, boolean getAfterPut,
+ boolean unitSameSize, int unitCount) {
+ for (int i = 0; i < unitCount; i++) {
+ ConsumeQueueExt.CqExtUnit putUnit =
+ unitSameSize ? genUnit(true) : genUnit(i % 2 == 0);
+
+ long addr = consumeQueueExt.put(putUnit);
+ assertThat(addr).isLessThan(0);
+
+ if (getAfterPut) {
+ ConsumeQueueExt.CqExtUnit getUnit = consumeQueueExt.get(addr);
+
+ assertThat(getUnit).isNotNull();
+ assertThat(putUnit).isEqualTo(getUnit);
+ }
+
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ assertThat(false).isTrue();
+ }
+ }
+ }
+
+ @Test
+ public void testPut() {
+ ConsumeQueueExt consumeQueueExt = genExt();
+
+ try {
+ putSth(consumeQueueExt, true, false, unitCount);
+ } finally {
+ consumeQueueExt.destroy();
+ deleteDirectory(storePath);
+ }
+ }
+
+ @Test
+ public void testGet() {
+ ConsumeQueueExt consumeQueueExt = genExt();
+
+ putSth(consumeQueueExt, false, false, unitCount);
+
+ try {
+ // from start.
+ long addr = consumeQueueExt.decorate(0);
+
+ ConsumeQueueExt.CqExtUnit unit = new ConsumeQueueExt.CqExtUnit();
+ while (true) {
+ boolean ret = consumeQueueExt.get(addr, unit);
+
+ if (!ret) {
+ break;
+ }
+
+ assertThat(unit.getSize()).isGreaterThanOrEqualTo(ConsumeQueueExt.CqExtUnit.MIN_EXT_UNIT_SIZE);
+
+ addr += unit.getSize();
+ }
+ } finally {
+ consumeQueueExt.destroy();
+ deleteDirectory(storePath);
+ }
+ }
+
+ @Test
+ public void testGet_invalidAddress() {
+ ConsumeQueueExt consumeQueueExt = genExt();
+
+ putSth(consumeQueueExt, false, true, unitCount);
+
+ try {
+ ConsumeQueueExt.CqExtUnit unit = consumeQueueExt.get(0);
+
+ assertThat(unit).isNull();
+
+ long addr = (cqExtFileSize / unitSizeWithBitMap) * unitSizeWithBitMap;
+ addr += unitSizeWithBitMap;
+
+ unit = consumeQueueExt.get(addr);
+ assertThat(unit).isNull();
+ } finally {
+ consumeQueueExt.destroy();
+ deleteDirectory(storePath);
+ }
+ }
+
+ @Test
+ public void testRecovery() {
+ ConsumeQueueExt putCqExt = genExt();
+
+ putSth(putCqExt, false, true, unitCount);
+
+ ConsumeQueueExt loadCqExt = genExt();
+
+ loadCqExt.load();
+
+ loadCqExt.recover();
+
+ try {
+ assertThat(loadCqExt.getMinAddress()).isEqualTo(Long.MIN_VALUE);
+
+ // same unit size.
+ int countPerFile = (cqExtFileSize - ConsumeQueueExt.END_BLANK_DATA_LENGTH) / unitSizeWithBitMap;
+
+ int lastFileUnitCount = unitCount % countPerFile;
+
+ int fileCount = unitCount / countPerFile + 1;
+ if (lastFileUnitCount == 0) {
+ fileCount -= 1;
+ }
+
+ if (lastFileUnitCount == 0) {
+ assertThat(loadCqExt.unDecorate(loadCqExt.getMaxAddress()) % cqExtFileSize).isEqualTo(0);
+ } else {
+ assertThat(loadCqExt.unDecorate(loadCqExt.getMaxAddress()))
+ .isEqualTo(lastFileUnitCount * unitSizeWithBitMap + (fileCount - 1) * cqExtFileSize);
+ }
+ } finally {
+ putCqExt.destroy();
+ loadCqExt.destroy();
+ deleteDirectory(storePath);
+ }
+ }
+
+ @Test
+ public void testTruncateByMinOffset() {
+ ConsumeQueueExt consumeQueueExt = genExt();
+
+ putSth(consumeQueueExt, false, true, unitCount * 2);
+
+ try {
+ // truncate first one file.
+ long address = consumeQueueExt.decorate((long) (cqExtFileSize * 1.5));
+
+ long expectMinAddress = consumeQueueExt.decorate(cqExtFileSize);
+
+ consumeQueueExt.truncateByMinAddress(address);
+
+ long minAddress = consumeQueueExt.getMinAddress();
+
+ assertThat(expectMinAddress).isEqualTo(minAddress);
+ } finally {
+ consumeQueueExt.destroy();
+ deleteDirectory(storePath);
+ }
+ }
+
+ @Test
+ public void testTruncateByMaxOffset() {
+ ConsumeQueueExt consumeQueueExt = genExt();
+
+ putSth(consumeQueueExt, false, true, unitCount * 2);
+
+ try {
+ // truncate, only first 3 files exist.
+ long address = consumeQueueExt.decorate(cqExtFileSize * 2 + unitSizeWithBitMap);
+
+ long expectMaxAddress = address + unitSizeWithBitMap;
+
+ consumeQueueExt.truncateByMaxAddress(address);
+
+ long maxAddress = consumeQueueExt.getMaxAddress();
+
+ assertThat(expectMaxAddress).isEqualTo(maxAddress);
+ } finally {
+ consumeQueueExt.destroy();
+ deleteDirectory(storePath);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java
----------------------------------------------------------------------
diff --git a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java
new file mode 100644
index 0000000..9c42fb9
--- /dev/null
+++ b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store;
+
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ConsumeQueueTest {
+
+ private static final String msg = "Once, there was a chance for me!";
+ private static final byte[] msgBody = msg.getBytes();
+
+ private static final String topic = "abc";
+ private static final int queueId = 0;
+ private static final String storePath = "." + File.separator + "unit_test_store";
+ private static final int commitLogFileSize = 1024 * 8;
+ private static final int cqFileSize = 10 * 20;
+ private static final int cqExtFileSize = 10 * (ConsumeQueueExt.CqExtUnit.MIN_EXT_UNIT_SIZE + 64);
+
+ private static SocketAddress BornHost;
+
+ private static SocketAddress StoreHost;
+
+ static {
+ try {
+ StoreHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ }
+ try {
+ BornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0);
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public MessageExtBrokerInner buildMessage() {
+ MessageExtBrokerInner msg = new MessageExtBrokerInner();
+ msg.setTopic(topic);
+ msg.setTags("TAG1");
+ msg.setKeys("Hello");
+ msg.setBody(msgBody);
+ msg.setKeys(String.valueOf(System.currentTimeMillis()));
+ msg.setQueueId(queueId);
+ msg.setSysFlag(0);
+ msg.setBornTimestamp(System.currentTimeMillis());
+ msg.setStoreHost(StoreHost);
+ msg.setBornHost(BornHost);
+ for (int i = 0; i < 1; i++) {
+ msg.putUserProperty(String.valueOf(i), "imagoodperson" + i);
+ }
+ msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
+
+ return msg;
+ }
+
+
+ public MessageStoreConfig buildStoreConfig(int commitLogFileSize, int cqFileSize,
+ boolean enableCqExt, int cqExtFileSize) {
+ MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+ messageStoreConfig.setMapedFileSizeCommitLog(commitLogFileSize);
+ messageStoreConfig.setMapedFileSizeConsumeQueue(cqFileSize);
+ messageStoreConfig.setMappedFileSizeConsumeQueueExt(cqExtFileSize);
+ messageStoreConfig.setMessageIndexEnable(false);
+ messageStoreConfig.setEnableConsumeQueueExt(enableCqExt);
+
+ messageStoreConfig.setStorePathRootDir(storePath);
+ messageStoreConfig.setStorePathCommitLog(storePath + File.separator + "commitlog");
+
+ return messageStoreConfig;
+ }
+
+ protected DefaultMessageStore gen() throws Exception {
+ MessageStoreConfig messageStoreConfig = buildStoreConfig(
+ commitLogFileSize, cqFileSize, true, cqExtFileSize
+ );
+
+ BrokerConfig brokerConfig = new BrokerConfig();
+
+ DefaultMessageStore master = new DefaultMessageStore(
+ messageStoreConfig,
+ new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
+ new MessageArrivingListener() {
+ @Override
+ public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
+ long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
+ }
+ }
+ , brokerConfig);
+
+ assertThat(master.load()).isTrue();
+
+ master.start();
+
+ return master;
+ }
+
+ protected void putMsg(DefaultMessageStore master) throws Exception {
+ long totalMsgs = 200;
+
+ for (long i = 0; i < totalMsgs; i++) {
+ master.putMessage(buildMessage());
+ }
+ }
+
+ protected void deleteDirectory(String rootPath) {
+ File file = new File(rootPath);
+ deleteFile(file);
+ }
+
+ protected void deleteFile(File file) {
+ File[] subFiles = file.listFiles();
+ if (subFiles != null) {
+ for (File sub : subFiles) {
+ deleteFile(sub);
+ }
+ }
+
+ file.delete();
+ }
+
+ @Test
+ public void testConsumeQueueWithExtendData() {
+ DefaultMessageStore master = null;
+ try {
+ master = gen();
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+
+ master.getDispatcherList().addFirst(new CommitLogDispatcher() {
+
+ @Override
+ public void dispatch(DispatchRequest request) {
+ runCount++;
+ }
+
+ private int runCount = 0;
+ });
+
+ try {
+ try {
+ putMsg(master);
+ // wait build consume queue
+ Thread.sleep(1000);
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+
+ ConsumeQueue cq = master.getConsumeQueueTable().get(topic).get(queueId);
+
+ assertThat(cq).isNotNull();
+
+ long index = 0;
+
+ while (index < cq.getMaxOffsetInQueue()) {
+ SelectMappedBufferResult bufferResult = cq.getIndexBuffer(index);
+
+ assertThat(bufferResult).isNotNull();
+
+ ByteBuffer buffer = bufferResult.getByteBuffer();
+
+ assertThat(buffer).isNotNull();
+ try {
+ ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
+ for (int i = 0; i < bufferResult.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
+ long phyOffset = buffer.getLong();
+ int size = buffer.getInt();
+ long tagsCode = buffer.getLong();
+
+ assertThat(phyOffset).isGreaterThanOrEqualTo(0);
+ assertThat(size).isGreaterThan(0);
+ assertThat(tagsCode).isLessThan(0);
+
+ boolean ret = cq.getExt(tagsCode, cqExtUnit);
+
+ assertThat(ret).isTrue();
+ assertThat(cqExtUnit).isNotNull();
+ assertThat(cqExtUnit.getSize()).isGreaterThan((short) 0);
+ assertThat(cqExtUnit.getMsgStoreTime()).isGreaterThan(0);
+ assertThat(cqExtUnit.getTagsCode()).isGreaterThan(0);
+ }
+
+ } finally {
+ bufferResult.release();
+ }
+
+ index += cqFileSize / ConsumeQueue.CQ_STORE_UNIT_SIZE;
+ }
+ } finally {
+ master.shutdown();
+ master.destroy();
+ deleteDirectory(storePath);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
----------------------------------------------------------------------
diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
index 5c9c46f..75f1de9 100644
--- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
@@ -20,6 +20,7 @@ package org.apache.rocketmq.store;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.store.config.FlushDiskType;
@@ -124,7 +125,8 @@ public class DefaultMessageStoreTest {
private class MyMessageArrivingListener implements MessageArrivingListener {
@Override
- public void arriving(String topic, int queueId, long logicOffset, long tagsCode) {
+ public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,
+ byte[] filterBitMap, Map<String, String> properties) {
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
index 19bff89..409ea33 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
@@ -41,6 +41,7 @@ import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
import org.apache.rocketmq.common.protocol.body.GroupList;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
@@ -469,4 +470,12 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt {
UnsupportedEncodingException {
return this.defaultMQAdminExtImpl.getNameServerConfig(nameServers);
}
+
+ @Override
+ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String topic, int queueId, long index, int count, String consumerGroup)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
+ return this.defaultMQAdminExtImpl.queryConsumeQueue(
+ brokerAddr, topic, queueId, index, count, consumerGroup
+ );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
index a31b69d..157ae21 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
@@ -63,6 +63,7 @@ import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
import org.apache.rocketmq.common.protocol.body.GroupList;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
@@ -955,4 +956,11 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner {
return this.mqClientInstance.getMQClientAPIImpl().getNameServerConfig(nameServers, timeoutMillis);
}
+ @Override
+ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String topic, int queueId, long index, int count, String consumerGroup)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
+ return this.mqClientInstance.getMQClientAPIImpl().queryConsumeQueue(
+ brokerAddr, topic, queueId, index, count, consumerGroup, timeoutMillis
+ );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
index 493cf54..82add92 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
@@ -39,6 +39,7 @@ import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
import org.apache.rocketmq.common.protocol.body.GroupList;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
@@ -241,4 +242,25 @@ public interface MQAdminExt extends MQAdmin {
Map<String, Properties> getNameServerConfig(final List<String> nameServers) throws InterruptedException,
RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,
MQClientException, UnsupportedEncodingException;
+
+ /**
+ * query consume queue data
+ *
+ * @param brokerAddr broker ip address
+ * @param topic topic
+ * @param queueId id of queue
+ * @param index start offset
+ * @param count how many
+ * @param consumerGroup group
+ * @return
+ * @throws InterruptedException
+ * @throws RemotingTimeoutException
+ * @throws RemotingSendRequestException
+ * @throws RemotingConnectException
+ * @throws MQClientException
+ */
+ QueryConsumeQueueResponseBody queryConsumeQueue(final String brokerAddr,
+ final String topic, final int queueId,
+ final long index, final int count, final String consumerGroup)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
index 9bd37e8..6398291 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
@@ -59,6 +59,7 @@ import org.apache.rocketmq.tools.command.namesrv.UpdateNamesrvConfigCommand;
import org.apache.rocketmq.tools.command.namesrv.WipeWritePermSubCommand;
import org.apache.rocketmq.tools.command.offset.CloneGroupOffsetCommand;
import org.apache.rocketmq.tools.command.offset.ResetOffsetByTimeCommand;
+import org.apache.rocketmq.tools.command.queue.QueryConsumeQueueCommand;
import org.apache.rocketmq.tools.command.stats.StatsAllSubCommand;
import org.apache.rocketmq.tools.command.topic.AllocateMQSubCommand;
import org.apache.rocketmq.tools.command.topic.DeleteTopicSubCommand;
@@ -189,6 +190,8 @@ public class MQAdminStartup {
initCommand(new GetNamesrvConfigCommand());
initCommand(new UpdateNamesrvConfigCommand());
initCommand(new GetBrokerConfigCommand());
+
+ initCommand(new QueryConsumeQueueCommand());
}
private static void initLogback() throws JoranException {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/QueryConsumeQueueCommand.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/queue/QueryConsumeQueueCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/QueryConsumeQueueCommand.java
new file mode 100644
index 0000000..611addd
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/QueryConsumeQueueCommand.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.tools.command.queue;
+
+import com.alibaba.fastjson.JSON;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.PosixParser;
+import org.apache.rocketmq.common.protocol.body.ConsumeQueueData;
+import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.SubCommand;
+
+public class QueryConsumeQueueCommand implements SubCommand {
+
+ public static void main(String[] args) {
+ QueryConsumeQueueCommand cmd = new QueryConsumeQueueCommand();
+
+ Options options = ServerUtil.buildCommandlineOptions(new Options());
+ String[] subargs = new String[]{"-t TopicTest", "-q 0", "-i 6447", "-b 100.81.165.119:10911"};
+ final CommandLine commandLine =
+ ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, cmd.buildCommandlineOptions(options),
+ new PosixParser());
+ cmd.execute(commandLine, options, null);
+ }
+
+ @Override
+ public String commandName() {
+ return "queryCq";
+ }
+
+ @Override
+ public String commandDesc() {
+ return "Query cq command.";
+ }
+
+ @Override
+ public Options buildCommandlineOptions(Options options) {
+ Option opt = new Option("t", "topic", true, "topic name");
+ opt.setRequired(true);
+ options.addOption(opt);
+
+ opt = new Option("q", "queue", true, "queue num, ie. 1");
+ opt.setRequired(true);
+ options.addOption(opt);
+
+ opt = new Option("i", "index", true, "start queue index.");
+ opt.setRequired(true);
+ options.addOption(opt);
+
+ opt = new Option("c", "count", true, "how many.");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("b", "broker", true, "broker addr.");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("g", "consumer", true, "consumer group.");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ return options;
+ }
+
+ @Override
+ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) {
+ DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+
+ defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+ try {
+ defaultMQAdminExt.start();
+
+ String topic = commandLine.getOptionValue("t").trim();
+ int queueId = Integer.valueOf(commandLine.getOptionValue("q").trim());
+ long index = Long.valueOf(commandLine.getOptionValue("i").trim());
+ int count = Integer.valueOf(commandLine.getOptionValue("c", "10").trim());
+ String broker = null;
+ if (commandLine.hasOption("b")) {
+ broker = commandLine.getOptionValue("b").trim();
+ }
+ String consumerGroup = null;
+ if (commandLine.hasOption("g")) {
+ consumerGroup = commandLine.getOptionValue("g").trim();
+ }
+
+ if (broker == null || broker == "") {
+ TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
+
+ if (topicRouteData == null || topicRouteData.getBrokerDatas() == null
+ || topicRouteData.getBrokerDatas().isEmpty()) {
+ throw new Exception("No topic route data!");
+ }
+
+ broker = topicRouteData.getBrokerDatas().get(0).getBrokerAddrs().get(0L);
+ }
+
+ QueryConsumeQueueResponseBody queryConsumeQueueResponseBody = defaultMQAdminExt.queryConsumeQueue(
+ broker, topic, queueId, index, count, consumerGroup
+ );
+
+ if (queryConsumeQueueResponseBody.getSubscriptionData() != null) {
+ System.out.printf("Subscription data: \n%s\n", JSON.toJSONString(queryConsumeQueueResponseBody.getSubscriptionData(), true));
+ System.out.print("======================================\n");
+ }
+
+ if (queryConsumeQueueResponseBody.getFilterData() != null) {
+ System.out.printf("Filter data: \n%s\n", queryConsumeQueueResponseBody.getFilterData());
+ System.out.print("======================================\n");
+ }
+
+ System.out.printf("Queue data: \nmax: %d, min: %d\n", queryConsumeQueueResponseBody.getMaxQueueIndex(),
+ queryConsumeQueueResponseBody.getMinQueueIndex());
+ System.out.print("======================================\n");
+
+ if (queryConsumeQueueResponseBody.getQueueData() != null) {
+
+ long i = index;
+ for (ConsumeQueueData queueData : queryConsumeQueueResponseBody.getQueueData()) {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("idx: " + i + "\n");
+
+ stringBuilder.append(queueData.toString() + "\n");
+
+ stringBuilder.append("======================================\n");
+
+ System.out.print(stringBuilder.toString());
+ i++;
+ }
+
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ defaultMQAdminExt.shutdown();
+ }
+ }
+}
[16/50] [abbrv] incubator-rocketmq git commit: Changed list creation
at DynaCode() to a singleton.
Posted by do...@apache.org.
Changed list creation at DynaCode() to a singleton.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/6898d96c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/6898d96c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/6898d96c
Branch: refs/heads/release-4.1.0-incubating
Commit: 6898d96c0a355a7256cc74d558b660d77c2871c7
Parents: d72addf
Author: shroman <rs...@yahoo.com>
Authored: Thu May 4 21:03:38 2017 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Thu May 4 21:03:38 2017 +0900
----------------------------------------------------------------------
.../java/org/apache/rocketmq/filtersrv/filter/DynaCode.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6898d96c/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java
----------------------------------------------------------------------
diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java
index a57b57f..e0a94d7 100644
--- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java
+++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/DynaCode.java
@@ -28,7 +28,7 @@ import java.net.URL;
import java.net.URLClassLoader;
import java.net.URLDecoder;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -71,9 +71,8 @@ public class DynaCode {
private String target;
- @SuppressWarnings("unchecked")
public DynaCode(String code) {
- this(Thread.currentThread().getContextClassLoader(), Arrays.asList(code));
+ this(Thread.currentThread().getContextClassLoader(), Collections.singletonList(code));
}
public DynaCode(ClassLoader parentClassLoader, List<String> codeStrs) {
[15/50] [abbrv] incubator-rocketmq git commit: Removed author info
and formatted.
Posted by do...@apache.org.
Removed author info and formatted.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/d72addff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/d72addff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/d72addff
Branch: refs/heads/release-4.1.0-incubating
Commit: d72addffb3c31a76da5c96f11c183402d8d4323f
Parents: 8feb88d
Author: shroman <rs...@yahoo.com>
Authored: Thu May 4 20:45:11 2017 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Thu May 4 20:45:11 2017 +0900
----------------------------------------------------------------------
.../common/MessageEncodeDecodeTest.java | 30 ++++++++------------
1 file changed, 12 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/d72addff/common/src/test/java/org/apache/rocketmq/common/MessageEncodeDecodeTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/rocketmq/common/MessageEncodeDecodeTest.java b/common/src/test/java/org/apache/rocketmq/common/MessageEncodeDecodeTest.java
index a219eda..42d3909 100644
--- a/common/src/test/java/org/apache/rocketmq/common/MessageEncodeDecodeTest.java
+++ b/common/src/test/java/org/apache/rocketmq/common/MessageEncodeDecodeTest.java
@@ -6,39 +6,33 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-
package org.apache.rocketmq.common;
-import org.apache.rocketmq.common.message.Message;
-import org.apache.rocketmq.common.message.MessageDecoder;
-import org.junit.Test;
-
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.junit.Test;
import static org.junit.Assert.assertTrue;
-/**
- * Created by liuzhendong on 16/12/21.
- */
public class MessageEncodeDecodeTest {
-
@Test
- public void testEncodeDecodeSingle() throws Exception{
+ public void testEncodeDecodeSingle() throws Exception {
Message message = new Message("topic", "body".getBytes());
message.setFlag(12);
- message.putUserProperty("key","value");
+ message.putUserProperty("key", "value");
byte[] bytes = MessageDecoder.encodeMessage(message);
ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
buffer.put(bytes);
@@ -59,7 +53,7 @@ public class MessageEncodeDecodeTest {
message.putUserProperty("key", "value" + i);
messages.add(message);
}
- byte[] bytes = MessageDecoder.encodeMessages(messages);
+ byte[] bytes = MessageDecoder.encodeMessages(messages);
ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
buffer.put(bytes);
[40/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-88] Polish
the developer list in pom.xml
Posted by do...@apache.org.
[ROCKETMQ-88] Polish the developer list in pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/de4c948c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/de4c948c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/de4c948c
Branch: refs/heads/release-4.1.0-incubating
Commit: de4c948cff561b9e20994947b32b0c328f30e194
Parents: 8d78175
Author: dongeforever <zh...@yeah.net>
Authored: Sat May 27 14:30:53 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 14:30:53 2017 +0800
----------------------------------------------------------------------
pom.xml | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/de4c948c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c60c93c..a77f21d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -76,8 +76,7 @@
<role>committer</role>
</roles>
<email>vintagewang@apache.org</email>
- <url>https://github.com/vintagewang</url>
- <timezone>8</timezone>
+ <timezone>+8</timezone>
</developer>
<developer>
<id>vongosling@apache.org</id>
@@ -87,14 +86,12 @@
<role>committer</role>
</roles>
<email>vongosling@apache.org</email>
- <url>https://github.com/vongosling@apache.org</url>
<timezone>+8</timezone>
</developer>
<developer>
<id>yukon</id>
<name>Xinyu Zhou</name>
<email>yukon@@apache.org</email>
- <url>https://github.com/zhouxinyu</url>
<roles>
<role>committer</role>
</roles>
@@ -130,6 +127,16 @@
</roles>
<timezone>+8</timezone>
</developer>
+ <developer>
+ <id>dongeforever</id>
+ <name>dongeforever</name>
+ <email>dongeforever@apache.org</email>
+ <roles>
+ <role>committer</role>
+ </roles>
+ <timezone>+8</timezone>
+ </developer>
+
</developers>
<licenses>
[06/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-121]Support
message filtering based on SQL92 closes apache/incubator-rocketmq#82
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/test/java/org/apache/rocketmq/filter/BitsArrayTest.java
----------------------------------------------------------------------
diff --git a/filter/src/test/java/org/apache/rocketmq/filter/BitsArrayTest.java b/filter/src/test/java/org/apache/rocketmq/filter/BitsArrayTest.java
new file mode 100644
index 0000000..ef81b29
--- /dev/null
+++ b/filter/src/test/java/org/apache/rocketmq/filter/BitsArrayTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter;
+
+import org.apache.rocketmq.filter.util.BitsArray;
+import org.junit.Test;
+
+import java.util.Random;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class BitsArrayTest {
+
+ BitsArray gen(int bitCount) {
+ BitsArray bitsArray = BitsArray.create(bitCount);
+
+ for (int i = 0; i < bitCount / Byte.SIZE; i++) {
+ bitsArray.setByte(i, (byte) (new Random(System.currentTimeMillis())).nextInt(0xff));
+ try {
+ Thread.sleep(2);
+ } catch (InterruptedException e) {
+ }
+ }
+
+ return bitsArray;
+ }
+
+ int bitLength = Byte.SIZE;
+
+ @Test
+ public void testConstructor() {
+ BitsArray bitsArray = BitsArray.create(8);
+
+ assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 8).isTrue();
+
+ bitsArray = BitsArray.create(9);
+
+ assertThat(bitsArray.byteLength() == 2 && bitsArray.bitLength() == 9).isTrue();
+
+ bitsArray = BitsArray.create(7);
+
+ assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 7).isTrue();
+ }
+
+ @Test
+ public void testSet() {
+ BitsArray bitsArray = gen(bitLength);
+ BitsArray backUp = bitsArray.clone();
+
+ boolean val = bitsArray.getBit(2);
+
+ bitsArray.setBit(2, !val);
+
+ bitsArray.xor(backUp);
+
+ assertThat(bitsArray.getBit(2)).isTrue();
+ }
+
+ @Test
+ public void testAndOr() {
+ BitsArray bitsArray = gen(bitLength);
+
+ boolean val = bitsArray.getBit(2);
+
+ if (val) {
+ bitsArray.and(2, false);
+ assertThat(!bitsArray.getBit(2)).isTrue();
+ } else {
+ bitsArray.or(2, true);
+ assertThat(bitsArray.getBit(2)).isTrue();
+ }
+ }
+
+ @Test
+ public void testXor() {
+ BitsArray bitsArray = gen(bitLength);
+
+ boolean val = bitsArray.getBit(2);
+
+ bitsArray.xor(2, !val);
+
+ assertThat(bitsArray.getBit(2)).isTrue();
+ }
+
+ @Test
+ public void testNot() {
+ BitsArray bitsArray = gen(bitLength);
+ BitsArray backUp = bitsArray.clone();
+
+ bitsArray.not(2);
+
+ bitsArray.xor(backUp);
+
+ assertThat(bitsArray.getBit(2)).isTrue();
+ }
+
+ @Test
+ public void testOr() {
+ BitsArray b1 = BitsArray.create(new byte[]{(byte) 0xff, 0x00});
+ BitsArray b2 = BitsArray.create(new byte[]{0x00, (byte) 0xff});
+
+ b1.or(b2);
+
+ for (int i = 0; i < b1.bitLength(); i++) {
+ assertThat(b1.getBit(i)).isTrue();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/test/java/org/apache/rocketmq/filter/BloomFilterTest.java
----------------------------------------------------------------------
diff --git a/filter/src/test/java/org/apache/rocketmq/filter/BloomFilterTest.java b/filter/src/test/java/org/apache/rocketmq/filter/BloomFilterTest.java
new file mode 100644
index 0000000..c6097ee
--- /dev/null
+++ b/filter/src/test/java/org/apache/rocketmq/filter/BloomFilterTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter;
+
+import org.apache.rocketmq.filter.util.BitsArray;
+import org.apache.rocketmq.filter.util.BloomFilter;
+import org.apache.rocketmq.filter.util.BloomFilterData;
+import org.junit.Test;
+
+import java.util.Random;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class BloomFilterTest {
+
+ @Test
+ public void testEquals() {
+ BloomFilter a = BloomFilter.createByFn(10, 20);
+
+ BloomFilter b = BloomFilter.createByFn(10, 20);
+
+ BloomFilter c = BloomFilter.createByFn(12, 20);
+
+ BloomFilter d = BloomFilter.createByFn(10, 30);
+
+ assertThat(a).isEqualTo(b);
+ assertThat(a).isNotEqualTo(c);
+ assertThat(a).isNotEqualTo(d);
+ assertThat(d).isNotEqualTo(c);
+
+ assertThat(a.hashCode()).isEqualTo(b.hashCode());
+ assertThat(a.hashCode()).isNotEqualTo(c.hashCode());
+ assertThat(a.hashCode()).isNotEqualTo(d.hashCode());
+ assertThat(c.hashCode()).isNotEqualTo(d.hashCode());
+ }
+
+ @Test
+ public void testHashTo() {
+ String cid = "CID_abc_efg";
+
+ BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);
+
+ BitsArray bits = BitsArray.create(bloomFilter.getM());
+
+ int[] bitPos = bloomFilter.calcBitPositions(cid);
+
+ bloomFilter.hashTo(cid, bits);
+
+ for (int bit : bitPos) {
+ assertThat(bits.getBit(bit)).isTrue();
+ }
+ }
+
+ @Test
+ public void testCalcBitPositions() {
+ String cid = "CID_abc_efg";
+
+ BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);
+
+ int[] bitPos = bloomFilter.calcBitPositions(cid);
+
+ assertThat(bitPos).isNotNull();
+ assertThat(bitPos.length).isEqualTo(bloomFilter.getK());
+
+ int[] bitPos2 = bloomFilter.calcBitPositions(cid);
+
+ assertThat(bitPos2).isNotNull();
+ assertThat(bitPos2.length).isEqualTo(bloomFilter.getK());
+
+ assertThat(bitPos).isEqualTo(bitPos2);
+ }
+
+ @Test
+ public void testIsHit() {
+ String cid = "CID_abc_efg";
+ String cid2 = "CID_abc_123";
+
+ BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);
+
+ BitsArray bits = BitsArray.create(bloomFilter.getM());
+
+ bloomFilter.hashTo(cid, bits);
+
+ assertThat(bloomFilter.isHit(cid, bits)).isTrue();
+ assertThat(!bloomFilter.isHit(cid2, bits)).isTrue();
+
+ bloomFilter.hashTo(cid2, bits);
+
+ assertThat(bloomFilter.isHit(cid, bits)).isTrue();
+ assertThat(bloomFilter.isHit(cid2, bits)).isTrue();
+ }
+
+ @Test
+ public void testBloomFilterData() {
+ BloomFilterData bloomFilterData = new BloomFilterData(new int[]{1, 2, 3}, 128);
+ BloomFilterData bloomFilterData1 = new BloomFilterData(new int[]{1, 2, 3}, 128);
+ BloomFilterData bloomFilterData2 = new BloomFilterData(new int[]{1, 2, 3}, 129);
+
+ assertThat(bloomFilterData).isEqualTo(bloomFilterData1);
+ assertThat(bloomFilterData2).isNotEqualTo(bloomFilterData);
+ assertThat(bloomFilterData2).isNotEqualTo(bloomFilterData1);
+
+ assertThat(bloomFilterData.hashCode()).isEqualTo(bloomFilterData1.hashCode());
+ assertThat(bloomFilterData2.hashCode()).isNotEqualTo(bloomFilterData.hashCode());
+ assertThat(bloomFilterData2.hashCode()).isNotEqualTo(bloomFilterData1.hashCode());
+
+ assertThat(bloomFilterData.getBitPos()).isEqualTo(bloomFilterData2.getBitPos());
+ assertThat(bloomFilterData.getBitNum()).isEqualTo(bloomFilterData1.getBitNum());
+ assertThat(bloomFilterData.getBitNum()).isNotEqualTo(bloomFilterData2.getBitNum());
+
+ bloomFilterData2.setBitNum(128);
+
+ assertThat(bloomFilterData).isEqualTo(bloomFilterData2);
+
+ bloomFilterData2.setBitPos(new int[]{1, 2, 3, 4});
+
+ assertThat(bloomFilterData).isNotEqualTo(bloomFilterData2);
+
+ BloomFilterData nullData = new BloomFilterData();
+
+ assertThat(nullData.getBitNum()).isEqualTo(0);
+ assertThat(nullData.getBitPos()).isNull();
+
+ BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
+
+ assertThat(bloomFilter).isNotNull();
+ assertThat(bloomFilter.isValid(bloomFilterData)).isFalse();
+ }
+
+ @Test
+ public void testCheckFalseHit() {
+ BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
+ BitsArray bits = BitsArray.create(bloomFilter.getM());
+ int falseHit = 0;
+ for (int i = 0; i < bloomFilter.getN(); i++) {
+ String str = randomString((new Random(System.nanoTime())).nextInt(127) + 10);
+ int[] bitPos = bloomFilter.calcBitPositions(str);
+
+ if (bloomFilter.checkFalseHit(bitPos, bits)) {
+ falseHit++;
+ }
+
+ bloomFilter.hashTo(bitPos, bits);
+ }
+
+ assertThat(falseHit).isLessThanOrEqualTo(bloomFilter.getF() * bloomFilter.getN() / 100);
+ }
+
+ private String randomString(int length) {
+ StringBuilder stringBuilder = new StringBuilder(length);
+ for (int i = 0; i < length; i++) {
+ stringBuilder.append((char) ((new Random(System.nanoTime())).nextInt(123 - 97) + 97));
+ }
+
+ return stringBuilder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/test/java/org/apache/rocketmq/filter/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/filter/src/test/java/org/apache/rocketmq/filter/ExpressionTest.java b/filter/src/test/java/org/apache/rocketmq/filter/ExpressionTest.java
new file mode 100644
index 0000000..0ee81c9
--- /dev/null
+++ b/filter/src/test/java/org/apache/rocketmq/filter/ExpressionTest.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter;
+
+import org.apache.rocketmq.filter.expression.ComparisonExpression;
+import org.apache.rocketmq.filter.expression.ConstantExpression;
+import org.apache.rocketmq.filter.expression.EvaluationContext;
+import org.apache.rocketmq.filter.expression.Expression;
+import org.apache.rocketmq.filter.expression.MQFilterException;
+import org.apache.rocketmq.filter.expression.PropertyExpression;
+import org.apache.rocketmq.filter.parser.SelectorParser;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ExpressionTest {
+
+ private static String andExpression = "a=3 and b<>4 And c>5 AND d<=4";
+ private static String orExpression = "a=3 or b<>4 Or c>5 OR d<=4";
+ private static String inExpression = "a in ('3', '4', '5')";
+ private static String notInExpression = "a not in ('3', '4', '5')";
+ private static String betweenExpression = "a between 2 and 10";
+ private static String notBetweenExpression = "a not between 2 and 10";
+ private static String isNullExpression = "a is null";
+ private static String isNotNullExpression = "a is not null";
+ private static String equalExpression = "a is not null and a='hello'";
+ private static String booleanExpression = "a=TRUE OR b=FALSE";
+ private static String nullOrExpression = "a is null OR a='hello'";
+ private static String stringHasString = "TAGS is not null and TAGS='''''tag'''''";
+
+ @Test
+ public void testEvaluate_stringHasString() {
+ Expression expr = genExp(stringHasString);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("TAGS", "''tag''")
+ );
+
+ eval(expr, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_now() {
+ EvaluationContext context = genContext(
+ KeyValue.c("a", System.currentTimeMillis())
+ );
+
+ Expression nowExpression = ConstantExpression.createNow();
+ Expression propertyExpression = new PropertyExpression("a");
+
+ Expression expression = ComparisonExpression.createLessThanEqual(propertyExpression,
+ nowExpression);
+
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_stringCompare() {
+ Expression expression = genExp("a between up and low");
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "3.14")
+ );
+
+ eval(expression, context, Boolean.FALSE);
+
+ {
+ context = genContext(
+ KeyValue.c("a", "3.14"),
+ KeyValue.c("up", "up"),
+ KeyValue.c("low", "low")
+ );
+
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ {
+ expression = genExp("key is not null and key between 0 and 100");
+
+ context = genContext(
+ KeyValue.c("key", "con")
+ );
+
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ {
+ expression = genExp("a between 0 and 100");
+
+ context = genContext(
+ KeyValue.c("a", "abc")
+ );
+
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ {
+ expression = genExp("a=b");
+
+ context = genContext(
+ KeyValue.c("a", "3.14"),
+ KeyValue.c("b", "3.14")
+ );
+
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ {
+ expression = genExp("a<>b");
+
+ context = genContext(
+ KeyValue.c("a", "3.14"),
+ KeyValue.c("b", "3.14")
+ );
+
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ {
+ expression = genExp("a<>b");
+
+ context = genContext(
+ KeyValue.c("a", "3.14"),
+ KeyValue.c("b", "3.141")
+ );
+
+ eval(expression, context, Boolean.TRUE);
+ }
+ }
+
+ @Test
+ public void testEvaluate_exponent() {
+ Expression expression = genExp("a > 3.1E10");
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", String.valueOf(3.1415 * Math.pow(10, 10)))
+ );
+
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_floatNumber() {
+ Expression expression = genExp("a > 3.14");
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", String.valueOf(3.1415))
+ );
+
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_twoVariable() {
+ Expression expression = genExp("a > b");
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", String.valueOf(10)),
+ KeyValue.c("b", String.valueOf(20))
+ );
+
+ eval(expression, context, Boolean.FALSE);
+
+ context = genContext(
+ KeyValue.c("b", String.valueOf(10)),
+ KeyValue.c("a", String.valueOf(20))
+ );
+
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_nullOr() {
+ Expression expression = genExp(nullOrExpression);
+
+ EvaluationContext context = genContext(
+ );
+
+ eval(expression, context, Boolean.TRUE);
+
+ context = genContext(
+ KeyValue.c("a", "hello")
+ );
+
+ eval(expression, context, Boolean.TRUE);
+
+ context = genContext(
+ KeyValue.c("a", "abc")
+ );
+
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_boolean() {
+ Expression expression = genExp(booleanExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "true"),
+ KeyValue.c("b", "false")
+ );
+
+ eval(expression, context, Boolean.TRUE);
+
+ context = genContext(
+ KeyValue.c("a", "false"),
+ KeyValue.c("b", "true")
+ );
+
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_equal() {
+ Expression expression = genExp(equalExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "hello")
+ );
+
+ eval(expression, context, Boolean.TRUE);
+
+ context = genContext(
+ );
+
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_andTrue() {
+ Expression expression = genExp(andExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", 3),
+ KeyValue.c("b", 5),
+ KeyValue.c("c", 6),
+ KeyValue.c("d", 1)
+ );
+
+ for (int i = 0; i < 500; i++) {
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ long start = System.currentTimeMillis();
+ for (int j = 0; j < 100; j++) {
+ for (int i = 0; i < 1000; i++) {
+ eval(expression, context, Boolean.TRUE);
+ }
+ }
+
+ // use string
+ context = genContext(
+ KeyValue.c("a", "3"),
+ KeyValue.c("b", "5"),
+ KeyValue.c("c", "6"),
+ KeyValue.c("d", "1")
+ );
+
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_andFalse() {
+ Expression expression = genExp(andExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", 4),
+ KeyValue.c("b", 5),
+ KeyValue.c("c", 6),
+ KeyValue.c("d", 1)
+ );
+
+ eval(expression, context, Boolean.FALSE);
+
+ // use string
+ context = genContext(
+ KeyValue.c("a", "4"),
+ KeyValue.c("b", "5"),
+ KeyValue.c("c", "6"),
+ KeyValue.c("d", "1")
+ );
+
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_orTrue() {
+ Expression expression = genExp(orExpression);
+
+ // first
+ EvaluationContext context = genContext(
+ KeyValue.c("a", 3)
+ );
+ eval(expression, context, Boolean.TRUE);
+
+ // second
+ context = genContext(
+ KeyValue.c("a", 4),
+ KeyValue.c("b", 5)
+ );
+ eval(expression, context, Boolean.TRUE);
+
+ // third
+ context = genContext(
+ KeyValue.c("a", 4),
+ KeyValue.c("b", 4),
+ KeyValue.c("c", 6)
+ );
+ eval(expression, context, Boolean.TRUE);
+
+ // forth
+ context = genContext(
+ KeyValue.c("a", 4),
+ KeyValue.c("b", 4),
+ KeyValue.c("c", 3),
+ KeyValue.c("d", 2)
+ );
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_orFalse() {
+ Expression expression = genExp(orExpression);
+ // forth
+ EvaluationContext context = genContext(
+ KeyValue.c("a", 4),
+ KeyValue.c("b", 4),
+ KeyValue.c("c", 3),
+ KeyValue.c("d", 10)
+ );
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_inTrue() {
+ Expression expression = genExp(inExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "3")
+ );
+ eval(expression, context, Boolean.TRUE);
+
+ context = genContext(
+ KeyValue.c("a", "4")
+ );
+ eval(expression, context, Boolean.TRUE);
+
+ context = genContext(
+ KeyValue.c("a", "5")
+ );
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_inFalse() {
+ Expression expression = genExp(inExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "8")
+ );
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_notInTrue() {
+ Expression expression = genExp(notInExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "8")
+ );
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_notInFalse() {
+ Expression expression = genExp(notInExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "3")
+ );
+ eval(expression, context, Boolean.FALSE);
+
+ context = genContext(
+ KeyValue.c("a", "4")
+ );
+ eval(expression, context, Boolean.FALSE);
+
+ context = genContext(
+ KeyValue.c("a", "5")
+ );
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_betweenTrue() {
+ Expression expression = genExp(betweenExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "2")
+ );
+ eval(expression, context, Boolean.TRUE);
+
+ context = genContext(
+ KeyValue.c("a", "10")
+ );
+ eval(expression, context, Boolean.TRUE);
+
+ context = genContext(
+ KeyValue.c("a", "3")
+ );
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_betweenFalse() {
+ Expression expression = genExp(betweenExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "1")
+ );
+ eval(expression, context, Boolean.FALSE);
+
+ context = genContext(
+ KeyValue.c("a", "11")
+ );
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_notBetweenTrue() {
+ Expression expression = genExp(notBetweenExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "1")
+ );
+ eval(expression, context, Boolean.TRUE);
+
+ context = genContext(
+ KeyValue.c("a", "11")
+ );
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_notBetweenFalse() {
+ Expression expression = genExp(notBetweenExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "2")
+ );
+ eval(expression, context, Boolean.FALSE);
+
+ context = genContext(
+ KeyValue.c("a", "10")
+ );
+ eval(expression, context, Boolean.FALSE);
+
+ context = genContext(
+ KeyValue.c("a", "3")
+ );
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_isNullTrue() {
+ Expression expression = genExp(isNullExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("abc", "2")
+ );
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_isNullFalse() {
+ Expression expression = genExp(isNullExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "2")
+ );
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ @Test
+ public void testEvaluate_isNotNullTrue() {
+ Expression expression = genExp(isNotNullExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("a", "2")
+ );
+ eval(expression, context, Boolean.TRUE);
+ }
+
+ @Test
+ public void testEvaluate_isNotNullFalse() {
+ Expression expression = genExp(isNotNullExpression);
+
+ EvaluationContext context = genContext(
+ KeyValue.c("abc", "2")
+ );
+ eval(expression, context, Boolean.FALSE);
+ }
+
+ protected void eval(Expression expression, EvaluationContext context, Boolean result) {
+ Object ret = null;
+ try {
+ ret = expression.evaluate(context);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+
+ if (ret == null || !(ret instanceof Boolean)) {
+ assertThat(result).isFalse();
+ } else {
+ assertThat(result).isEqualTo(ret);
+ }
+ }
+
+ protected EvaluationContext genContext(KeyValue... keyValues) {
+ if (keyValues == null || keyValues.length < 1) {
+ return new PropertyContext();
+ }
+
+ PropertyContext context = new PropertyContext();
+ for (KeyValue keyValue : keyValues) {
+ context.properties.put(keyValue.key, keyValue.value);
+ }
+
+ return context;
+ }
+
+ protected Expression genExp(String exp) {
+ Expression expression = null;
+
+ try {
+ expression = SelectorParser.parse(exp);
+
+ assertThat(expression).isNotNull();
+ } catch (MQFilterException e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+
+ return expression;
+ }
+
+ static class KeyValue {
+ public static KeyValue c(String key, Object value) {
+ return new KeyValue(key, value);
+ }
+
+ public KeyValue(String key, Object value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ public String key;
+ public Object value;
+ }
+
+ class PropertyContext implements EvaluationContext {
+
+ public Map<String, Object> properties = new HashMap<String, Object>(8);
+
+ @Override
+ public Object get(final String name) {
+ return properties.get(name);
+ }
+
+ @Override
+ public Map<String, Object> keyValues() {
+ return properties;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/test/java/org/apache/rocketmq/filter/FilterSpiTest.java
----------------------------------------------------------------------
diff --git a/filter/src/test/java/org/apache/rocketmq/filter/FilterSpiTest.java b/filter/src/test/java/org/apache/rocketmq/filter/FilterSpiTest.java
new file mode 100644
index 0000000..22eeb86
--- /dev/null
+++ b/filter/src/test/java/org/apache/rocketmq/filter/FilterSpiTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter;
+
+import org.apache.rocketmq.common.filter.ExpressionType;
+import org.apache.rocketmq.filter.expression.EmptyEvaluationContext;
+import org.apache.rocketmq.filter.expression.EvaluationContext;
+import org.apache.rocketmq.filter.expression.Expression;
+import org.apache.rocketmq.filter.expression.MQFilterException;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class FilterSpiTest {
+
+ static class NothingExpression implements Expression {
+
+ @Override
+ public Object evaluate(final EvaluationContext context) throws Exception {
+ return Boolean.TRUE;
+ }
+ }
+
+ static class NothingFilter implements FilterSpi {
+ @Override
+ public Expression compile(final String expr) throws MQFilterException {
+ return new NothingExpression();
+ }
+
+ @Override
+ public String ofType() {
+ return "Nothing";
+ }
+ }
+
+
+ @Test
+ public void testRegister() {
+ FilterFactory.INSTANCE.register(new NothingFilter());
+
+ Expression expr = null;
+ try {
+ expr = FilterFactory.INSTANCE.get("Nothing").compile("abc");
+ } catch (MQFilterException e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+
+ assertThat(expr).isNotNull();
+
+ try {
+ assertThat((Boolean) expr.evaluate(new EmptyEvaluationContext())).isTrue();
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+ }
+
+ @Test
+ public void testGet() {
+ try {
+ assertThat((Boolean) FilterFactory.INSTANCE.get(ExpressionType.SQL92).compile("a is not null and a > 0")
+ .evaluate(new EmptyEvaluationContext())).isFalse();
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java
----------------------------------------------------------------------
diff --git a/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java b/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java
new file mode 100644
index 0000000..36ef271
--- /dev/null
+++ b/filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter;
+
+import org.apache.rocketmq.filter.expression.Expression;
+import org.apache.rocketmq.filter.expression.MQFilterException;
+import org.apache.rocketmq.filter.parser.SelectorParser;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ParserTest {
+
+ private static String andExpression = "a=3 and b<>4 And c>5 AND d<=4";
+ private static String andExpressionHasBlank = "a=3 and b<>4 And c>5 AND d<=4";
+ private static String orExpression = "a=3 or b<>4 Or c>5 OR d<=4";
+ private static String inExpression = "a in ('3', '4', '5')";
+ private static String notInExpression = "(a not in ('6', '4', '5')) or (b in ('3', '4', '5'))";
+ private static String betweenExpression = "(a between 2 and 10) AND (b not between 6 and 9)";
+ private static String equalNullExpression = "a is null";
+ private static String notEqualNullExpression = "a is not null";
+ private static String nowExpression = "a <= now";
+
+ private static String invalidExpression = "a and between 2 and 10";
+ private static String illegalBetween = " a between 10 and 0";
+
+ @Test
+ public void testParse_valid() {
+ for (String expr : Arrays.asList(
+ andExpression, orExpression, inExpression, notInExpression, betweenExpression,
+ equalNullExpression, notEqualNullExpression, nowExpression
+ )) {
+
+ try {
+ Expression expression = SelectorParser.parse(expr);
+ assertThat(expression).isNotNull();
+ } catch (MQFilterException e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+
+ }
+ }
+
+ @Test
+ public void testParse_invalid() {
+ try {
+ SelectorParser.parse(invalidExpression);
+
+ assertThat(Boolean.TRUE).isFalse();
+ } catch (MQFilterException e) {
+ }
+ }
+
+ @Test
+ public void testParse_decimalOverFlow() {
+ try {
+ String str = "100000000000000000000000";
+
+ SelectorParser.parse("a > " + str);
+
+ assertThat(Boolean.TRUE).isFalse();
+ } catch (Exception e) {
+ }
+ }
+
+ @Test
+ public void testParse_floatOverFlow() {
+ try {
+ String str = "1";
+ for (int i = 0; i < 2048; i++) {
+ str += "111111111111111111111111111111111111111111111111111";
+ }
+ str += ".";
+ for (int i = 0; i < 2048; i++) {
+ str += "111111111111111111111111111111111111111111111111111";
+ }
+
+ SelectorParser.parse("a > " + str);
+
+ assertThat(Boolean.TRUE).isFalse();
+ } catch (Exception e) {
+ }
+ }
+
+ @Test
+ public void testParse_illegalBetween() {
+ try {
+ SelectorParser.parse(illegalBetween);
+
+ assertThat(Boolean.TRUE).isFalse();
+ } catch (Exception e) {
+ }
+ }
+
+ @Test
+ public void testEquals() {
+ try {
+ Expression expr1 = SelectorParser.parse(andExpression);
+
+ Expression expr2 = SelectorParser.parse(andExpressionHasBlank);
+
+ Expression expr3 = SelectorParser.parse(orExpression);
+
+ assertThat(expr1).isEqualTo(expr2);
+ assertThat(expr1).isNotEqualTo(expr3);
+ } catch (MQFilterException e) {
+ e.printStackTrace();
+ assertThat(Boolean.TRUE).isFalse();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 47df84d..feb8b14 100644
--- a/pom.xml
+++ b/pom.xml
@@ -178,6 +178,7 @@
<module>example</module>
<module>filtersrv</module>
<module>srvutil</module>
+ <module>filter</module>
<module>test</module>
<module>distribution</module>
</modules>
@@ -554,6 +555,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-filter</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>rocketmq-example</artifactId>
<version>${project.version}</version>
@@ -603,6 +609,11 @@
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>19.0</version>
+ </dependency>
</dependencies>
</dependencyManagement>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/srvutil/pom.xml
----------------------------------------------------------------------
diff --git a/srvutil/pom.xml b/srvutil/pom.xml
index 3269903..6dc0377 100644
--- a/srvutil/pom.xml
+++ b/srvutil/pom.xml
@@ -41,5 +41,9 @@
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
index 5be8258..7841feb 100644
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
@@ -314,10 +314,11 @@ public class CommitLog {
// 17 properties
short propertiesLength = byteBuffer.getShort();
+ Map<String, String> propertiesMap = null;
if (propertiesLength > 0) {
byteBuffer.get(bytesContent, 0, propertiesLength);
String properties = new String(bytesContent, 0, propertiesLength, MessageDecoder.CHARSET_UTF8);
- Map<String, String> propertiesMap = MessageDecoder.string2messageProperties(properties);
+ propertiesMap = MessageDecoder.string2messageProperties(properties);
keys = propertiesMap.get(MessageConst.PROPERTY_KEYS);
@@ -369,8 +370,9 @@ public class CommitLog {
queueOffset, // 7
keys, // 8
uniqKey, //9
- sysFlag, // 9
- preparedTransactionOffset// 10
+ sysFlag, // 10
+ preparedTransactionOffset, // 11
+ propertiesMap // 12
);
} catch (Exception e) {
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java
new file mode 100644
index 0000000..e1564a9
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store;
+
+/**
+ * Dispatcher of commit log.
+ */
+public interface CommitLogDispatcher {
+
+ void dispatch(final DispatchRequest request);
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
index 919c637..d03ff0f 100644
--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
+++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
@@ -20,6 +20,7 @@ import java.io.File;
import java.nio.ByteBuffer;
import java.util.List;
import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.store.config.StorePathConfigHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,6 +41,7 @@ public class ConsumeQueue {
private final int mappedFileSize;
private long maxPhysicOffset = -1;
private volatile long minLogicOffset = 0;
+ private ConsumeQueueExt consumeQueueExt = null;
public ConsumeQueue(
final String topic,
@@ -61,11 +63,24 @@ public class ConsumeQueue {
this.mappedFileQueue = new MappedFileQueue(queueDir, mappedFileSize, null);
this.byteBufferIndex = ByteBuffer.allocate(CQ_STORE_UNIT_SIZE);
+
+ if (defaultMessageStore.getMessageStoreConfig().isEnableConsumeQueueExt()) {
+ this.consumeQueueExt = new ConsumeQueueExt(
+ topic,
+ queueId,
+ StorePathConfigHelper.getStorePathConsumeQueueExt(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir()),
+ defaultMessageStore.getMessageStoreConfig().getMappedFileSizeConsumeQueueExt(),
+ defaultMessageStore.getMessageStoreConfig().getBitMapLengthConsumeQueueExt()
+ );
+ }
}
public boolean load() {
boolean result = this.mappedFileQueue.load();
log.info("load consume queue " + this.topic + "-" + this.queueId + " " + (result ? "OK" : "Failed"));
+ if (isExtReadEnable()) {
+ result &= this.consumeQueueExt.load();
+ }
return result;
}
@@ -82,6 +97,7 @@ public class ConsumeQueue {
ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();
long processOffset = mappedFile.getFileFromOffset();
long mappedFileOffset = 0;
+ long maxExtAddr = 1;
while (true) {
for (int i = 0; i < mappedFileSizeLogics; i += CQ_STORE_UNIT_SIZE) {
long offset = byteBuffer.getLong();
@@ -91,6 +107,9 @@ public class ConsumeQueue {
if (offset >= 0 && size > 0) {
mappedFileOffset = i + CQ_STORE_UNIT_SIZE;
this.maxPhysicOffset = offset;
+ if (isExtAddr(tagsCode)) {
+ maxExtAddr = tagsCode;
+ }
} else {
log.info("recover current consume queue file over, " + mappedFile.getFileName() + " "
+ offset + " " + size + " " + tagsCode);
@@ -123,6 +142,12 @@ public class ConsumeQueue {
this.mappedFileQueue.setFlushedWhere(processOffset);
this.mappedFileQueue.setCommittedWhere(processOffset);
this.mappedFileQueue.truncateDirtyFiles(processOffset);
+
+ if (isExtReadEnable()) {
+ this.consumeQueueExt.recover();
+ log.info("Truncate consume queue extend file by max {}", maxExtAddr);
+ this.consumeQueueExt.truncateByMaxAddress(maxExtAddr);
+ }
}
}
@@ -200,7 +225,7 @@ public class ConsumeQueue {
int logicFileSize = this.mappedFileSize;
this.maxPhysicOffset = phyOffet - 1;
-
+ long maxExtAddr = 1;
while (true) {
MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
if (mappedFile != null) {
@@ -213,7 +238,7 @@ public class ConsumeQueue {
for (int i = 0; i < logicFileSize; i += CQ_STORE_UNIT_SIZE) {
long offset = byteBuffer.getLong();
int size = byteBuffer.getInt();
- byteBuffer.getLong();
+ long tagsCode = byteBuffer.getLong();
if (0 == i) {
if (offset >= phyOffet) {
@@ -225,6 +250,10 @@ public class ConsumeQueue {
mappedFile.setCommittedPosition(pos);
mappedFile.setFlushedPosition(pos);
this.maxPhysicOffset = offset;
+ // This maybe not take effect, when not every consume queue has extend file.
+ if (isExtAddr(tagsCode)) {
+ maxExtAddr = tagsCode;
+ }
}
} else {
@@ -239,6 +268,9 @@ public class ConsumeQueue {
mappedFile.setCommittedPosition(pos);
mappedFile.setFlushedPosition(pos);
this.maxPhysicOffset = offset;
+ if (isExtAddr(tagsCode)) {
+ maxExtAddr = tagsCode;
+ }
if (pos == logicFileSize) {
return;
@@ -252,6 +284,10 @@ public class ConsumeQueue {
break;
}
}
+
+ if (isExtReadEnable()) {
+ this.consumeQueueExt.truncateByMaxAddress(maxExtAddr);
+ }
}
public long getLastOffset() {
@@ -285,7 +321,12 @@ public class ConsumeQueue {
}
public boolean flush(final int flushLeastPages) {
- return this.mappedFileQueue.flush(flushLeastPages);
+ boolean result = this.mappedFileQueue.flush(flushLeastPages);
+ if (isExtReadEnable()) {
+ result = result & this.consumeQueueExt.flush(flushLeastPages);
+ }
+
+ return result;
}
public int deleteExpiredFile(long offset) {
@@ -296,6 +337,7 @@ public class ConsumeQueue {
public void correctMinOffset(long phyMinOffset) {
MappedFile mappedFile = this.mappedFileQueue.getFirstMappedFile();
+ long minExtAddr = 1;
if (mappedFile != null) {
SelectMappedBufferResult result = mappedFile.selectMappedBuffer(0);
if (result != null) {
@@ -303,12 +345,16 @@ public class ConsumeQueue {
for (int i = 0; i < result.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
long offsetPy = result.getByteBuffer().getLong();
result.getByteBuffer().getInt();
- result.getByteBuffer().getLong();
+ long tagsCode = result.getByteBuffer().getLong();
if (offsetPy >= phyMinOffset) {
this.minLogicOffset = result.getMappedFile().getFileFromOffset() + i;
log.info("Compute logical min offset: {}, topic: {}, queueId: {}",
this.getMinOffsetInQueue(), this.topic, this.queueId);
+ // This maybe not take effect, when not every consume queue has extend file.
+ if (isExtAddr(tagsCode)) {
+ minExtAddr = tagsCode;
+ }
break;
}
}
@@ -319,24 +365,43 @@ public class ConsumeQueue {
}
}
}
+
+ if (isExtReadEnable()) {
+ this.consumeQueueExt.truncateByMinAddress(minExtAddr);
+ }
}
public long getMinOffsetInQueue() {
return this.minLogicOffset / CQ_STORE_UNIT_SIZE;
}
- public void putMessagePositionInfoWrapper(long offset, int size, long tagsCode, long storeTimestamp,
- long logicOffset) {
+ public void putMessagePositionInfoWrapper(DispatchRequest request) {
final int maxRetries = 30;
boolean canWrite = this.defaultMessageStore.getRunningFlags().isCQWriteable();
for (int i = 0; i < maxRetries && canWrite; i++) {
- boolean result = this.putMessagePositionInfo(offset, size, tagsCode, logicOffset);
+ long tagsCode = request.getTagsCode();
+ if (isExtWriteEnable()) {
+ ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
+ cqExtUnit.setFilterBitMap(request.getBitMap());
+ cqExtUnit.setMsgStoreTime(request.getStoreTimestamp());
+ cqExtUnit.setTagsCode(request.getTagsCode());
+
+ long extAddr = this.consumeQueueExt.put(cqExtUnit);
+ if (isExtAddr(extAddr)) {
+ tagsCode = extAddr;
+ } else {
+ log.warn("Save consume queue extend fail, So just save tagsCode! {}, topic:{}, queueId:{}, offset:{}", cqExtUnit,
+ topic, queueId, request.getCommitLogOffset());
+ }
+ }
+ boolean result = this.putMessagePositionInfo(request.getCommitLogOffset(),
+ request.getMsgSize(), tagsCode, request.getConsumeQueueOffset());
if (result) {
- this.defaultMessageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimestamp);
+ this.defaultMessageStore.getStoreCheckpoint().setLogicsMsgTimestamp(request.getStoreTimestamp());
return;
} else {
// XXX: warn and notify me
- log.warn("[BUG]put commit log position info to " + topic + ":" + queueId + " " + offset
+ log.warn("[BUG]put commit log position info to " + topic + ":" + queueId + " " + request.getCommitLogOffset()
+ " failed, retry " + i + " times");
try {
@@ -423,6 +488,20 @@ public class ConsumeQueue {
return null;
}
+ public ConsumeQueueExt.CqExtUnit getExt(final long offset) {
+ if (isExtReadEnable()) {
+ return this.consumeQueueExt.get(offset);
+ }
+ return null;
+ }
+
+ public boolean getExt(final long offset, ConsumeQueueExt.CqExtUnit cqExtUnit) {
+ if (isExtReadEnable()) {
+ return this.consumeQueueExt.get(offset, cqExtUnit);
+ }
+ return false;
+ }
+
public long getMinLogicOffset() {
return minLogicOffset;
}
@@ -457,6 +536,9 @@ public class ConsumeQueue {
this.maxPhysicOffset = -1;
this.minLogicOffset = 0;
this.mappedFileQueue.destroy();
+ if (isExtReadEnable()) {
+ this.consumeQueueExt.destroy();
+ }
}
public long getMessageTotalInQueue() {
@@ -469,5 +551,27 @@ public class ConsumeQueue {
public void checkSelf() {
mappedFileQueue.checkSelf();
+ if (isExtReadEnable()) {
+ this.consumeQueueExt.checkSelf();
+ }
+ }
+
+ protected boolean isExtReadEnable() {
+ return this.consumeQueueExt != null;
+ }
+
+ protected boolean isExtWriteEnable() {
+ return this.consumeQueueExt != null
+ && this.defaultMessageStore.getMessageStoreConfig().isEnableConsumeQueueExt();
+ }
+
+ /**
+ * Check {@code tagsCode} is address of extend file or tags code.
+ *
+ * @param tagsCode
+ * @return
+ */
+ public boolean isExtAddr(long tagsCode) {
+ return isExtReadEnable() && this.consumeQueueExt.isExtAddr(tagsCode);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java
new file mode 100644
index 0000000..1a177e9
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store;
+
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Extend of consume queue, to store something not important,
+ * such as message store time, filter bit map and etc.
+ * <p/>
+ * <li>1. This class is used only by {@link ConsumeQueue}</li>
+ * <li>2. And is week reliable.</li>
+ * <li>3. Be careful, address returned is always less than 0.</li>
+ * <li>4. Pls keep this file small.</li>
+ */
+public class ConsumeQueueExt {
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
+
+ private final MappedFileQueue mappedFileQueue;
+ private final String topic;
+ private final int queueId;
+
+ private final String storePath;
+ private final int mappedFileSize;
+ private ByteBuffer tempContainer;
+
+ public static final int END_BLANK_DATA_LENGTH = 4;
+
+ /**
+ * Addr can not exceed this value.For compatible.
+ */
+ public static final long MAX_ADDR = Integer.MIN_VALUE - 1L;
+ public static final long MAX_REAL_OFFSET = MAX_ADDR - Long.MIN_VALUE;
+
+ /**
+ * Constructor.
+ *
+ * @param topic topic
+ * @param queueId id of queue
+ * @param storePath root dir of files to store.
+ * @param mappedFileSize file size
+ * @param bitMapLength bit map length.
+ */
+ public ConsumeQueueExt(final String topic,
+ final int queueId,
+ final String storePath,
+ final int mappedFileSize,
+ final int bitMapLength) {
+
+ this.storePath = storePath;
+ this.mappedFileSize = mappedFileSize;
+
+ this.topic = topic;
+ this.queueId = queueId;
+
+ String queueDir = this.storePath
+ + File.separator + topic
+ + File.separator + queueId;
+
+ this.mappedFileQueue = new MappedFileQueue(queueDir, mappedFileSize, null);
+
+ if (bitMapLength > 0) {
+ this.tempContainer = ByteBuffer.allocate(
+ bitMapLength / Byte.SIZE
+ );
+ }
+ }
+
+ /**
+ * Check whether {@code address} point to extend file.
+ * <p>
+ * Just test {@code address} is less than 0.
+ * </p>
+ *
+ * @param address
+ * @return
+ */
+ public boolean isExtAddr(final long address) {
+ return address <= MAX_ADDR;
+ }
+
+ /**
+ * Transform {@code address}(decorated by {@link #decorate}) to offset in mapped file.
+ * <p>
+ * if {@code address} is less than 0, return {@code address} - {@link java.lang.Long#MIN_VALUE};
+ * else, just return {@code address}
+ * </p>
+ *
+ * @param address
+ * @return
+ */
+ public long unDecorate(final long address) {
+ if (isExtAddr(address)) {
+ return address - Long.MIN_VALUE;
+ }
+ return address;
+ }
+
+ /**
+ * Decorate {@code offset} from mapped file, in order to distinguish with tagsCode(saved in cq originally).
+ * <p>
+ * if {@code offset} is greater than or equal to 0, then return {@code offset} + {@link java.lang.Long#MIN_VALUE};
+ * else, just return {@code offset}
+ * </p>
+ *
+ * @param offset
+ * @return ext address(value is less than 0)
+ */
+ public long decorate(final long offset) {
+ if (!isExtAddr(offset)) {
+ return offset + Long.MIN_VALUE;
+ }
+ return offset;
+ }
+
+ /**
+ * Get data from buffer.
+ *
+ * @param address less than 0
+ * @return
+ */
+ public CqExtUnit get(final long address) {
+ CqExtUnit cqExtUnit = new CqExtUnit();
+ if (get(address, cqExtUnit)) {
+ return cqExtUnit;
+ }
+
+ return null;
+ }
+
+ /**
+ * Get data from buffer, and set to {@code cqExtUnit}
+ *
+ * @param address less than 0
+ * @param cqExtUnit
+ * @return
+ */
+ public boolean get(final long address, final CqExtUnit cqExtUnit) {
+ if (!isExtAddr(address)) {
+ return false;
+ }
+
+ final int mappedFileSize = this.mappedFileSize;
+ final long realOffset = unDecorate(address);
+
+ MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(realOffset, realOffset == 0);
+ if (mappedFile == null) {
+ return false;
+ }
+
+ int pos = (int) (realOffset % mappedFileSize);
+
+ SelectMappedBufferResult bufferResult = mappedFile.selectMappedBuffer(pos);
+ if (bufferResult == null) {
+ log.warn("[BUG] Consume queue extend unit({}) is not found!", realOffset);
+ return false;
+ }
+ boolean ret = false;
+ try {
+ ret = cqExtUnit.read(bufferResult.getByteBuffer());
+ } finally {
+ bufferResult.release();
+ }
+
+ return ret;
+ }
+
+ /**
+ * Save to mapped buffer of file and return address.
+ * <p>
+ * Be careful, this method is not thread safe.
+ * </p>
+ *
+ * @param cqExtUnit
+ * @return success: < 0: fail: >=0
+ */
+ public long put(final CqExtUnit cqExtUnit) {
+ final int retryTimes = 3;
+ try {
+ int size = cqExtUnit.calcUnitSize();
+ if (size > CqExtUnit.MAX_EXT_UNIT_SIZE) {
+ log.error("Size of cq ext unit is greater than {}, {}", CqExtUnit.MAX_EXT_UNIT_SIZE, cqExtUnit);
+ return 1;
+ }
+ if (this.mappedFileQueue.getMaxOffset() + size > MAX_REAL_OFFSET) {
+ log.warn("Capacity of ext is maximum!{}, {}", this.mappedFileQueue.getMaxOffset(), size);
+ return 1;
+ }
+ // unit size maybe change.but, the same most of the time.
+ if (this.tempContainer == null || this.tempContainer.capacity() < size) {
+ this.tempContainer = ByteBuffer.allocate(size);
+ }
+
+ for (int i = 0; i < retryTimes; i++) {
+ MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
+
+ if (mappedFile == null || mappedFile.isFull()) {
+ mappedFile = this.mappedFileQueue.getLastMappedFile(0);
+ }
+
+ if (mappedFile == null) {
+ log.error("Create mapped file when save consume queue extend, {}", cqExtUnit);
+ continue;
+ }
+ final int wrotePosition = mappedFile.getWrotePosition();
+ final int blankSize = this.mappedFileSize - wrotePosition - END_BLANK_DATA_LENGTH;
+
+ // check whether has enough space.
+ if (size > blankSize) {
+ fullFillToEnd(mappedFile, wrotePosition);
+ log.info("No enough space(need:{}, has:{}) of file {}, so fill to end",
+ size, blankSize, mappedFile.getFileName());
+ continue;
+ }
+
+ if (mappedFile.appendMessage(cqExtUnit.write(this.tempContainer), 0, size)) {
+ return decorate(wrotePosition + mappedFile.getFileFromOffset());
+ }
+ }
+ } catch (Throwable e) {
+ log.error("Save consume queue extend error, " + cqExtUnit, e);
+ }
+
+ return 1;
+ }
+
+ protected void fullFillToEnd(final MappedFile mappedFile, final int wrotePosition) {
+ ByteBuffer mappedFileBuffer = mappedFile.sliceByteBuffer();
+ mappedFileBuffer.position(wrotePosition);
+
+ // ending.
+ mappedFileBuffer.putShort((short) -1);
+
+ mappedFile.setWrotePosition(this.mappedFileSize);
+ }
+
+ /**
+ * Load data from file when startup.
+ *
+ * @return
+ */
+ public boolean load() {
+ boolean result = this.mappedFileQueue.load();
+ log.info("load consume queue extend" + this.topic + "-" + this.queueId + " " + (result ? "OK" : "Failed"));
+ return result;
+ }
+
+ /**
+ * Check whether the step size in mapped file queue is correct.
+ */
+ public void checkSelf() {
+ this.mappedFileQueue.checkSelf();
+ }
+
+ /**
+ * Recover.
+ */
+ public void recover() {
+ final List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();
+ if (mappedFiles == null || mappedFiles.isEmpty()) {
+ return;
+ }
+
+ // load all files, consume queue will truncate extend files.
+ int index = 0;
+
+ MappedFile mappedFile = mappedFiles.get(index);
+ ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();
+ long processOffset = mappedFile.getFileFromOffset();
+ long mappedFileOffset = 0;
+ CqExtUnit extUnit = new CqExtUnit();
+ while (true) {
+ extUnit.readBySkip(byteBuffer);
+
+ // check whether write sth.
+ if (extUnit.getSize() > 0) {
+ mappedFileOffset += extUnit.getSize();
+ continue;
+ }
+
+ index++;
+ if (index < mappedFiles.size()) {
+ mappedFile = mappedFiles.get(index);
+ byteBuffer = mappedFile.sliceByteBuffer();
+ processOffset = mappedFile.getFileFromOffset();
+ mappedFileOffset = 0;
+ log.info("Recover next consume queue extend file, " + mappedFile.getFileName());
+ continue;
+ }
+
+ log.info("All files of consume queue extend has been recovered over, last mapped file "
+ + mappedFile.getFileName());
+ break;
+ }
+
+ processOffset += mappedFileOffset;
+ this.mappedFileQueue.setFlushedWhere(processOffset);
+ this.mappedFileQueue.setCommittedWhere(processOffset);
+ this.mappedFileQueue.truncateDirtyFiles(processOffset);
+ }
+
+ /**
+ * Delete files before {@code minAddress}.
+ *
+ * @param minAddress less than 0
+ */
+ public void truncateByMinAddress(final long minAddress) {
+ if (!isExtAddr(minAddress)) {
+ return;
+ }
+
+ log.info("Truncate consume queue ext by min {}.", minAddress);
+
+ List<MappedFile> willRemoveFiles = new ArrayList<MappedFile>();
+
+ List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();
+ final long realOffset = unDecorate(minAddress);
+
+ for (MappedFile file : mappedFiles) {
+ long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize;
+
+ if (fileTailOffset < realOffset) {
+ log.info("Destroy consume queue ext by min: file={}, fileTailOffset={}, minOffset={}", file.getFileName(),
+ fileTailOffset, realOffset);
+ if (file.destroy(1000)) {
+ willRemoveFiles.add(file);
+ }
+ }
+ }
+
+ this.mappedFileQueue.deleteExpiredFile(willRemoveFiles);
+ }
+
+ /**
+ * Delete files after {@code maxAddress}, and reset wrote/commit/flush position to last file.
+ *
+ * @param maxAddress less than 0
+ */
+ public void truncateByMaxAddress(final long maxAddress) {
+ if (!isExtAddr(maxAddress)) {
+ return;
+ }
+
+ log.info("Truncate consume queue ext by max {}.", maxAddress);
+
+ CqExtUnit cqExtUnit = get(maxAddress);
+ if (cqExtUnit == null) {
+ log.error("[BUG] address {} of consume queue extend not found!", maxAddress);
+ return;
+ }
+
+ final long realOffset = unDecorate(maxAddress);
+
+ this.mappedFileQueue.truncateDirtyFiles(realOffset + cqExtUnit.getSize());
+ }
+
+ /**
+ * flush buffer to file.
+ *
+ * @param flushLeastPages
+ * @return
+ */
+ public boolean flush(final int flushLeastPages) {
+ return this.mappedFileQueue.flush(flushLeastPages);
+ }
+
+ /**
+ * delete files and directory.
+ */
+ public void destroy() {
+ this.mappedFileQueue.destroy();
+ }
+
+ /**
+ * Max address(value is less than 0).
+ * <p/>
+ * <p>
+ * Be careful: it's an address just when invoking this method.
+ * </p>
+ *
+ * @return
+ */
+ public long getMaxAddress() {
+ MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
+ if (mappedFile == null) {
+ return decorate(0);
+ }
+ return decorate(mappedFile.getFileFromOffset() + mappedFile.getWrotePosition());
+ }
+
+ /**
+ * Minus address saved in file.
+ *
+ * @return
+ */
+ public long getMinAddress() {
+ MappedFile firstFile = this.mappedFileQueue.getFirstMappedFile();
+ if (firstFile == null) {
+ return decorate(0);
+ }
+ return decorate(firstFile.getFileFromOffset());
+ }
+
+ /**
+ * Store unit.
+ */
+ public static class CqExtUnit {
+ public static final short MIN_EXT_UNIT_SIZE
+ = 2 * 1 // size, 32k max
+ + 8 * 2 // msg time + tagCode
+ + 2; // bitMapSize
+
+ public static final int MAX_EXT_UNIT_SIZE = Short.MAX_VALUE;
+
+ public CqExtUnit() {}
+
+ public CqExtUnit(Long tagsCode, long msgStoreTime, byte[] filterBitMap) {
+ this.tagsCode = tagsCode == null ? 0 : tagsCode;
+ this.msgStoreTime = msgStoreTime;
+ this.filterBitMap = filterBitMap;
+ this.bitMapSize = (short) (filterBitMap == null ? 0 : filterBitMap.length);
+ this.size = (short) (MIN_EXT_UNIT_SIZE + this.bitMapSize);
+ }
+
+ /**
+ * unit size
+ */
+ private short size;
+ /**
+ * has code of tags
+ */
+ private long tagsCode;
+ /**
+ * the time to store into commit log of message
+ */
+ private long msgStoreTime;
+ /**
+ * size of bit map
+ */
+ private short bitMapSize;
+ /**
+ * filter bit map
+ */
+ private byte[] filterBitMap;
+
+ /**
+ * build unit from buffer from current position.
+ *
+ * @param buffer
+ * @return
+ */
+ private boolean read(final ByteBuffer buffer) {
+ if (buffer.position() + 2 > buffer.limit()) {
+ return false;
+ }
+
+ this.size = buffer.getShort();
+
+ if (this.size < 1) {
+ return false;
+ }
+
+ this.tagsCode = buffer.getLong();
+ this.msgStoreTime = buffer.getLong();
+ this.bitMapSize = buffer.getShort();
+
+ if (this.bitMapSize < 1) {
+ return true;
+ }
+
+ if (this.filterBitMap == null || this.filterBitMap.length != this.bitMapSize) {
+ this.filterBitMap = new byte[bitMapSize];
+ }
+
+ buffer.get(this.filterBitMap);
+ return true;
+ }
+
+ /**
+ * Only read first 2 byte to get unit size.
+ * <p>
+ * if size > 0, then skip buffer position with size.
+ * </p>
+ * <p>
+ * if size <= 0, nothing to do.
+ * </p>
+ *
+ * @param buffer
+ */
+ private void readBySkip(final ByteBuffer buffer) {
+ ByteBuffer temp = buffer.slice();
+
+ short tempSize = temp.getShort();
+ this.size = tempSize;
+
+ if (tempSize > 0) {
+ buffer.position(buffer.position() + this.size);
+ }
+ }
+
+ /**
+ * Transform unit data to byte array.
+ * <p/>
+ * <li>1. @{code container} can be null, it will be created if null.</li>
+ * <li>2. if capacity of @{code container} is less than unit size, it will be created also.</li>
+ * <li>3. Pls be sure that size of unit is not greater than {@link #MAX_EXT_UNIT_SIZE}</li>
+ *
+ * @param container
+ * @return
+ */
+ private byte[] write(final ByteBuffer container) {
+ this.bitMapSize = (short) (filterBitMap == null ? 0 : filterBitMap.length);
+ this.size = (short) (MIN_EXT_UNIT_SIZE + this.bitMapSize);
+
+ ByteBuffer temp = container;
+
+ if (temp == null || temp.capacity() < this.size) {
+ temp = ByteBuffer.allocate(this.size);
+ }
+
+ temp.flip();
+ temp.limit(this.size);
+
+ temp.putShort(this.size);
+ temp.putLong(this.tagsCode);
+ temp.putLong(this.msgStoreTime);
+ temp.putShort(this.bitMapSize);
+ if (this.bitMapSize > 0) {
+ temp.put(this.filterBitMap);
+ }
+
+ return temp.array();
+ }
+
+ /**
+ * Calculate unit size by current data.
+ *
+ * @return
+ */
+ private int calcUnitSize() {
+ int sizeTemp = MIN_EXT_UNIT_SIZE + (filterBitMap == null ? 0 : filterBitMap.length);
+ return sizeTemp;
+ }
+
+ public long getTagsCode() {
+ return tagsCode;
+ }
+
+ public void setTagsCode(final long tagsCode) {
+ this.tagsCode = tagsCode;
+ }
+
+ public long getMsgStoreTime() {
+ return msgStoreTime;
+ }
+
+ public void setMsgStoreTime(final long msgStoreTime) {
+ this.msgStoreTime = msgStoreTime;
+ }
+
+ public byte[] getFilterBitMap() {
+ if (this.bitMapSize < 1) {
+ return null;
+ }
+ return filterBitMap;
+ }
+
+ public void setFilterBitMap(final byte[] filterBitMap) {
+ this.filterBitMap = filterBitMap;
+ // not safe transform, but size will be calculate by #calcUnitSize
+ this.bitMapSize = (short) (filterBitMap == null ? 0 : filterBitMap.length);
+ }
+
+ public short getSize() {
+ return size;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof CqExtUnit)) return false;
+
+ CqExtUnit cqExtUnit = (CqExtUnit) o;
+
+ if (bitMapSize != cqExtUnit.bitMapSize) return false;
+ if (msgStoreTime != cqExtUnit.msgStoreTime) return false;
+ if (size != cqExtUnit.size) return false;
+ if (tagsCode != cqExtUnit.tagsCode) return false;
+ if (!Arrays.equals(filterBitMap, cqExtUnit.filterBitMap)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = (int) size;
+ result = 31 * result + (int) (tagsCode ^ (tagsCode >>> 32));
+ result = 31 * result + (int) (msgStoreTime ^ (msgStoreTime >>> 32));
+ result = 31 * result + (int) bitMapSize;
+ result = 31 * result + (filterBitMap != null ? Arrays.hashCode(filterBitMap) : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "CqExtUnit{" +
+ "size=" + size +
+ ", tagsCode=" + tagsCode +
+ ", msgStoreTime=" + msgStoreTime +
+ ", bitMapSize=" + bitMapSize +
+ ", filterBitMap=" + Arrays.toString(filterBitMap) +
+ '}';
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageFilter.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageFilter.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageFilter.java
index 1350026..9db87f3 100644
--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageFilter.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageFilter.java
@@ -18,26 +18,33 @@ package org.apache.rocketmq.store;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import java.nio.ByteBuffer;
+import java.util.Map;
+
public class DefaultMessageFilter implements MessageFilter {
- @Override
- public boolean isMessageMatched(SubscriptionData subscriptionData, Long tagsCode) {
- if (tagsCode == null) {
- return true;
- }
+ private SubscriptionData subscriptionData;
- if (null == subscriptionData) {
- return true;
- }
+ public DefaultMessageFilter(final SubscriptionData subscriptionData) {
+ this.subscriptionData = subscriptionData;
+ }
- if (subscriptionData.isClassFilterMode())
+ @Override
+ public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
+ if (null == tagsCode || null == subscriptionData) {
return true;
+ }
- if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
+ if (subscriptionData.isClassFilterMode()) {
return true;
}
- return subscriptionData.getCodeSet().contains(tagsCode.intValue());
+ return subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)
+ || subscriptionData.getCodeSet().contains(tagsCode.intValue());
}
+ @Override
+ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
index 0edfeec..7bed62c 100644
--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -41,7 +42,6 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageExtBatch;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.running.RunningStats;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.store.config.BrokerRole;
@@ -60,8 +60,6 @@ import static org.apache.rocketmq.store.config.BrokerRole.SLAVE;
public class DefaultMessageStore implements MessageStore {
private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
- private final MessageFilter messageFilter = new DefaultMessageFilter();
-
private final MessageStoreConfig messageStoreConfig;
// CommitLog
private final CommitLog commitLog;
@@ -103,6 +101,8 @@ public class DefaultMessageStore implements MessageStore {
private AtomicLong printTimes = new AtomicLong(0);
+ private final LinkedList<CommitLogDispatcher> dispatcherList;
+
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException {
this.messageArrivingListener = messageArrivingListener;
@@ -133,6 +133,10 @@ public class DefaultMessageStore implements MessageStore {
this.allocateMappedFileService.start();
this.indexService.start();
+
+ this.dispatcherList = new LinkedList<>();
+ this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue());
+ this.dispatcherList.addLast(new CommitLogDispatcherBuildIndex());
}
public void truncateDirtyLogicFiles(long phyOffset) {
@@ -409,7 +413,7 @@ public class DefaultMessageStore implements MessageStore {
}
public GetMessageResult getMessage(final String group, final String topic, final int queueId, final long offset, final int maxMsgNums,
- final SubscriptionData subscriptionData) {
+ final MessageFilter messageFilter) {
if (this.shutdown) {
log.warn("message store has shutdown, so getMessage is forbidden");
return null;
@@ -464,6 +468,7 @@ public class DefaultMessageStore implements MessageStore {
int i = 0;
final int maxFilterMessageCount = Math.max(16000, maxMsgNums * ConsumeQueue.CQ_STORE_UNIT_SIZE);
final boolean diskFallRecorded = this.messageStoreConfig.isDiskFallRecorded();
+ ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
for (; i < bufferConsumeQueue.getSize() && i < maxFilterMessageCount; i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
long offsetPy = bufferConsumeQueue.getByteBuffer().getLong();
int sizePy = bufferConsumeQueue.getByteBuffer().getInt();
@@ -483,29 +488,51 @@ public class DefaultMessageStore implements MessageStore {
break;
}
- if (this.messageFilter.isMessageMatched(subscriptionData, tagsCode)) {
- SelectMappedBufferResult selectResult = this.commitLog.getMessage(offsetPy, sizePy);
- if (selectResult != null) {
- this.storeStatsService.getGetMessageTransferedMsgCount().incrementAndGet();
- getResult.addMessage(selectResult);
- status = GetMessageStatus.FOUND;
- nextPhyFileStartOffset = Long.MIN_VALUE;
+ boolean extRet = false;
+ if (consumeQueue.isExtAddr(tagsCode)) {
+ extRet = consumeQueue.getExt(tagsCode, cqExtUnit);
+ if (extRet) {
+ tagsCode = cqExtUnit.getTagsCode();
} else {
- if (getResult.getBufferTotalSize() == 0) {
- status = GetMessageStatus.MESSAGE_WAS_REMOVING;
- }
-
- nextPhyFileStartOffset = this.commitLog.rollNextFile(offsetPy);
+ // can't find ext content.Client will filter messages by tag also.
+ log.error("[BUG] can't find consume queue extend file content!addr={}, offsetPy={}, sizePy={}, topic={}, group={}",
+ tagsCode, offsetPy, sizePy, topic, group);
}
- } else {
+ }
+
+ if (messageFilter != null
+ && !messageFilter.isMatchedByConsumeQueue(tagsCode, extRet ? cqExtUnit : null)) {
if (getResult.getBufferTotalSize() == 0) {
status = GetMessageStatus.NO_MATCHED_MESSAGE;
}
- if (log.isDebugEnabled()) {
- log.debug("message type not matched, client: " + subscriptionData + " server: " + tagsCode);
+ continue;
+ }
+
+ SelectMappedBufferResult selectResult = this.commitLog.getMessage(offsetPy, sizePy);
+ if (null == selectResult) {
+ if (getResult.getBufferTotalSize() == 0) {
+ status = GetMessageStatus.MESSAGE_WAS_REMOVING;
+ }
+
+ nextPhyFileStartOffset = this.commitLog.rollNextFile(offsetPy);
+ continue;
+ }
+
+ if (messageFilter != null
+ && !messageFilter.isMatchedByCommitLog(selectResult.getByteBuffer().slice(), null)) {
+ if (getResult.getBufferTotalSize() == 0) {
+ status = GetMessageStatus.NO_MATCHED_MESSAGE;
}
+ // release...
+ selectResult.release();
+ continue;
}
+
+ this.storeStatsService.getGetMessageTransferedMsgCount().incrementAndGet();
+ getResult.addMessage(selectResult);
+ status = GetMessageStatus.FOUND;
+ nextPhyFileStartOffset = Long.MIN_VALUE;
}
if (diskFallRecorded) {
@@ -1318,27 +1345,14 @@ public class DefaultMessageStore implements MessageStore {
}
public void doDispatch(DispatchRequest req) {
- final int tranType = MessageSysFlag.getTransactionValue(req.getSysFlag());
- switch (tranType) {
- case MessageSysFlag.TRANSACTION_NOT_TYPE:
- case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
- DefaultMessageStore.this.putMessagePositionInfo(req.getTopic(), req.getQueueId(), req.getCommitLogOffset(), req.getMsgSize(),
- req.getTagsCode(), req.getStoreTimestamp(), req.getConsumeQueueOffset());
- break;
- case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
- case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
- break;
- }
-
- if (DefaultMessageStore.this.getMessageStoreConfig().isMessageIndexEnable()) {
- DefaultMessageStore.this.indexService.buildIndex(req);
+ for (CommitLogDispatcher dispatcher : this.dispatcherList) {
+ dispatcher.dispatch(req);
}
}
- public void putMessagePositionInfo(String topic, int queueId, long offset, int size, long tagsCode, long storeTimestamp,
- long logicOffset) {
- ConsumeQueue cq = this.findConsumeQueue(topic, queueId);
- cq.putMessagePositionInfoWrapper(offset, size, tagsCode, storeTimestamp, logicOffset);
+ public void putMessagePositionInfo(DispatchRequest dispatchRequest) {
+ ConsumeQueue cq = this.findConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId());
+ cq.putMessagePositionInfoWrapper(dispatchRequest);
}
public BrokerStatsManager getBrokerStatsManager() {
@@ -1354,6 +1368,20 @@ public class DefaultMessageStore implements MessageStore {
return remainTransientStoreBufferNumbs() == 0;
}
+ @Override
+ public LinkedList<CommitLogDispatcher> getDispatcherList() {
+ return this.dispatcherList;
+ }
+
+ @Override
+ public ConsumeQueue getConsumeQueue(String topic, int queueId) {
+ ConcurrentHashMap<Integer, ConsumeQueue> map = consumeQueueTable.get(topic);
+ if (map == null) {
+ return null;
+ }
+ return map.get(queueId);
+ }
+
public void unlockMappedFile(final MappedFile mappedFile) {
this.scheduledExecutorService.schedule(new Runnable() {
@Override
@@ -1363,6 +1391,33 @@ public class DefaultMessageStore implements MessageStore {
}, 6, TimeUnit.SECONDS);
}
+ class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher {
+
+ @Override
+ public void dispatch(DispatchRequest request) {
+ final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag());
+ switch (tranType) {
+ case MessageSysFlag.TRANSACTION_NOT_TYPE:
+ case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
+ DefaultMessageStore.this.putMessagePositionInfo(request);
+ break;
+ case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
+ case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
+ break;
+ }
+ }
+ }
+
+ class CommitLogDispatcherBuildIndex implements CommitLogDispatcher {
+
+ @Override
+ public void dispatch(DispatchRequest request) {
+ if (DefaultMessageStore.this.messageStoreConfig.isMessageIndexEnable()) {
+ DefaultMessageStore.this.indexService.buildIndex(request);
+ }
+ }
+ }
+
class CleanCommitLogService {
private final static int MAX_MANUAL_DELETE_FILE_TIMES = 20;
@@ -1695,7 +1750,8 @@ public class DefaultMessageStore implements MessageStore {
&& DefaultMessageStore.this.brokerConfig.isLongPollingEnable()) {
DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1,
- dispatchRequest.getTagsCode());
+ dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(),
+ dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap());
}
// FIXED BUG By shijia
this.reputFromOffset += size;
[22/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-186]
Implement the OpenMessaging specification 0.1.0-alpha version
Posted by do...@apache.org.
[ROCKETMQ-186] Implement the OpenMessaging specification 0.1.0-alpha version
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/1d966b50
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/1d966b50
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/1d966b50
Branch: refs/heads/release-4.1.0-incubating
Commit: 1d966b50c2ec189ca4f1bf81420959a33159a8ad
Parents: 1630f27
Author: yukon <yu...@apache.org>
Authored: Wed May 24 16:50:51 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Wed May 24 16:50:51 2017 +0800
----------------------------------------------------------------------
distribution/release-client.xml | 1 +
distribution/release.xml | 1 +
example/pom.xml | 9 +
.../example/openmessaging/SimpleProducer.java | 76 +++++++
.../openmessaging/SimplePullConsumer.java | 58 +++++
.../openmessaging/SimplePushConsumer.java | 59 +++++
openmessaging/pom.xml | 42 ++++
.../rocketmq/MessagingAccessPointImpl.java | 132 +++++++++++
.../rocketmq/config/ClientConfig.java | 194 ++++++++++++++++
.../rocketmq/consumer/LocalMessageCache.java | 213 +++++++++++++++++
.../rocketmq/consumer/PullConsumerImpl.java | 166 ++++++++++++++
.../rocketmq/consumer/PushConsumerImpl.java | 181 +++++++++++++++
.../rocketmq/domain/BytesMessageImpl.java | 108 +++++++++
.../rocketmq/domain/ConsumeRequest.java | 55 +++++
.../rocketmq/domain/NonStandardKeys.java | 30 +++
.../rocketmq/domain/SendResultImpl.java | 40 ++++
.../rocketmq/producer/AbstractOMSProducer.java | 138 +++++++++++
.../rocketmq/producer/ProducerImpl.java | 124 ++++++++++
.../rocketmq/producer/SequenceProducerImpl.java | 95 ++++++++
.../rocketmq/promise/DefaultPromise.java | 227 +++++++++++++++++++
.../rocketmq/promise/FutureState.java | 51 +++++
.../openmessaging/rocketmq/utils/BeanUtils.java | 185 +++++++++++++++
.../openmessaging/rocketmq/utils/OMSUtil.java | 182 +++++++++++++++
.../consumer/LocalMessageCacheTest.java | 89 ++++++++
.../rocketmq/consumer/PullConsumerImplTest.java | 96 ++++++++
.../rocketmq/consumer/PushConsumerImplTest.java | 87 +++++++
.../rocketmq/producer/ProducerImplTest.java | 101 +++++++++
.../producer/SequenceProducerImplTest.java | 86 +++++++
.../rocketmq/promise/DefaultPromiseTest.java | 136 +++++++++++
.../rocketmq/utils/BeanUtilsTest.java | 110 +++++++++
pom.xml | 6 +
31 files changed, 3078 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/distribution/release-client.xml
----------------------------------------------------------------------
diff --git a/distribution/release-client.xml b/distribution/release-client.xml
index 46563eb..84d33a0 100644
--- a/distribution/release-client.xml
+++ b/distribution/release-client.xml
@@ -47,6 +47,7 @@
<useAllReactorProjects>true</useAllReactorProjects>
<includes>
<include>org.apache.rocketmq:rocketmq-client</include>
+ <include>org.apache.rocketmq:rocketmq-openmessaging</include>
</includes>
<binaries>
<outputDirectory>./</outputDirectory>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/distribution/release.xml
----------------------------------------------------------------------
diff --git a/distribution/release.xml b/distribution/release.xml
index 9e4ef2a..c67d23e 100644
--- a/distribution/release.xml
+++ b/distribution/release.xml
@@ -68,6 +68,7 @@
<include>org.apache.rocketmq:rocketmq-filtersrv</include>
<include>org.apache.rocketmq:rocketmq-example</include>
<include>org.apache.rocketmq:rocketmq-filter</include>
+ <include>org.apache.rocketmq:rocketmq-openmessaging</include>
</includes>
<binaries>
<outputDirectory>lib/</outputDirectory>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/example/pom.xml
----------------------------------------------------------------------
diff --git a/example/pom.xml b/example/pom.xml
index 785a4ca..840fa36 100644
--- a/example/pom.xml
+++ b/example/pom.xml
@@ -48,5 +48,14 @@
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-openmessaging</artifactId>
+ <version>4.1.0-incubating-SNAPSHOT</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimpleProducer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimpleProducer.java b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimpleProducer.java
new file mode 100644
index 0000000..9d162ac
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimpleProducer.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.openmessaging;
+
+import io.openmessaging.Message;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.MessagingAccessPointFactory;
+import io.openmessaging.Producer;
+import io.openmessaging.Promise;
+import io.openmessaging.PromiseListener;
+import io.openmessaging.SendResult;
+import java.nio.charset.Charset;
+
+public class SimpleProducer {
+ public static void main(String[] args) {
+ final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory
+ .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace");
+
+ final Producer producer = messagingAccessPoint.createProducer();
+
+ messagingAccessPoint.startup();
+ System.out.printf("MessagingAccessPoint startup OK%n");
+
+ producer.startup();
+ System.out.printf("Producer startup OK%n");
+
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ producer.shutdown();
+ messagingAccessPoint.shutdown();
+ }
+ }));
+
+ {
+ Message message = producer.createBytesMessageToTopic("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8")));
+ SendResult sendResult = producer.send(message);
+ //final Void aVoid = result.get(3000L);
+ System.out.printf("Send async message OK, msgId: %s%n", sendResult.messageId());
+ }
+
+ {
+ final Promise<SendResult> result = producer.sendAsync(producer.createBytesMessageToTopic("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8"))));
+ result.addListener(new PromiseListener<SendResult>() {
+ @Override
+ public void operationCompleted(Promise<SendResult> promise) {
+ System.out.printf("Send async message OK, msgId: %s%n", promise.get().messageId());
+ }
+
+ @Override
+ public void operationFailed(Promise<SendResult> promise) {
+ System.out.printf("Send async message Failed, error: %s%n", promise.getThrowable().getMessage());
+ }
+ });
+ }
+
+ {
+ producer.sendOneway(producer.createBytesMessageToTopic("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8"))));
+ System.out.printf("Send oneway message OK%n");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePullConsumer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePullConsumer.java b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePullConsumer.java
new file mode 100644
index 0000000..8e06772
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePullConsumer.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.openmessaging;
+
+import io.openmessaging.Message;
+import io.openmessaging.MessageHeader;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.MessagingAccessPointFactory;
+import io.openmessaging.OMS;
+import io.openmessaging.PullConsumer;
+import io.openmessaging.rocketmq.domain.NonStandardKeys;
+
+public class SimplePullConsumer {
+ public static void main(String[] args) {
+ final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory
+ .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace");
+
+ final PullConsumer consumer = messagingAccessPoint.createPullConsumer("OMS_HELLO_TOPIC",
+ OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, "OMS_CONSUMER"));
+
+ messagingAccessPoint.startup();
+ System.out.printf("MessagingAccessPoint startup OK%n");
+
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ consumer.shutdown();
+ messagingAccessPoint.shutdown();
+ }
+ }));
+
+ consumer.startup();
+ System.out.printf("Consumer startup OK%n");
+
+ while (true) {
+ Message message = consumer.poll();
+ if (message != null) {
+ String msgId = message.headers().getString(MessageHeader.MESSAGE_ID);
+ System.out.printf("Received one message: %s%n", msgId);
+ consumer.ack(msgId);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePushConsumer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePushConsumer.java
new file mode 100644
index 0000000..b0935d4
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePushConsumer.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.openmessaging;
+
+import io.openmessaging.Message;
+import io.openmessaging.MessageHeader;
+import io.openmessaging.MessageListener;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.MessagingAccessPointFactory;
+import io.openmessaging.OMS;
+import io.openmessaging.PushConsumer;
+import io.openmessaging.ReceivedMessageContext;
+import io.openmessaging.rocketmq.domain.NonStandardKeys;
+
+public class SimplePushConsumer {
+ public static void main(String[] args) {
+ final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory
+ .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace");
+
+ final PushConsumer consumer = messagingAccessPoint.
+ createPushConsumer(OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, "OMS_CONSUMER"));
+
+ messagingAccessPoint.startup();
+ System.out.printf("MessagingAccessPoint startup OK%n");
+
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ consumer.shutdown();
+ messagingAccessPoint.shutdown();
+ }
+ }));
+
+ consumer.attachQueue("OMS_HELLO_TOPIC", new MessageListener() {
+ @Override
+ public void onMessage(final Message message, final ReceivedMessageContext context) {
+ System.out.printf("Received one message: %s%n", message.headers().getString(MessageHeader.MESSAGE_ID));
+ context.ack();
+ }
+ });
+
+ consumer.startup();
+ System.out.printf("Consumer startup OK%n");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/pom.xml
----------------------------------------------------------------------
diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml
new file mode 100644
index 0000000..e853642
--- /dev/null
+++ b/openmessaging/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>rocketmq-all</artifactId>
+ <groupId>org.apache.rocketmq</groupId>
+ <version>4.1.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>rocketmq-openmessaging</artifactId>
+ <name>rocketmq-openmessaging ${project.version}</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-client</artifactId>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/MessagingAccessPointImpl.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/MessagingAccessPointImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/MessagingAccessPointImpl.java
new file mode 100644
index 0000000..65caf84
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/MessagingAccessPointImpl.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq;
+
+import io.openmessaging.IterableConsumer;
+import io.openmessaging.KeyValue;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.Producer;
+import io.openmessaging.PullConsumer;
+import io.openmessaging.PushConsumer;
+import io.openmessaging.ResourceManager;
+import io.openmessaging.SequenceProducer;
+import io.openmessaging.ServiceEndPoint;
+import io.openmessaging.exception.OMSNotSupportedException;
+import io.openmessaging.observer.Observer;
+import io.openmessaging.rocketmq.consumer.PullConsumerImpl;
+import io.openmessaging.rocketmq.consumer.PushConsumerImpl;
+import io.openmessaging.rocketmq.producer.ProducerImpl;
+import io.openmessaging.rocketmq.producer.SequenceProducerImpl;
+import io.openmessaging.rocketmq.utils.OMSUtil;
+
+public class MessagingAccessPointImpl implements MessagingAccessPoint {
+ private final KeyValue accessPointProperties;
+
+ public MessagingAccessPointImpl(final KeyValue accessPointProperties) {
+ this.accessPointProperties = accessPointProperties;
+ }
+
+ @Override
+ public KeyValue properties() {
+ return accessPointProperties;
+ }
+
+ @Override
+ public Producer createProducer() {
+ return new ProducerImpl(this.accessPointProperties);
+ }
+
+ @Override
+ public Producer createProducer(KeyValue properties) {
+ return new ProducerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, properties));
+ }
+
+ @Override
+ public SequenceProducer createSequenceProducer() {
+ return new SequenceProducerImpl(this.accessPointProperties);
+ }
+
+ @Override
+ public SequenceProducer createSequenceProducer(KeyValue properties) {
+ return new SequenceProducerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, properties));
+ }
+
+ @Override
+ public PushConsumer createPushConsumer() {
+ return new PushConsumerImpl(accessPointProperties);
+ }
+
+ @Override
+ public PushConsumer createPushConsumer(KeyValue properties) {
+ return new PushConsumerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, properties));
+ }
+
+ @Override
+ public PullConsumer createPullConsumer(String queueName) {
+ return new PullConsumerImpl(queueName, accessPointProperties);
+ }
+
+ @Override
+ public PullConsumer createPullConsumer(String queueName, KeyValue properties) {
+ return new PullConsumerImpl(queueName, OMSUtil.buildKeyValue(this.accessPointProperties, properties));
+ }
+
+ @Override
+ public IterableConsumer createIterableConsumer(String queueName) {
+ throw new OMSNotSupportedException("-1", "IterableConsumer is not supported in current version");
+ }
+
+ @Override
+ public IterableConsumer createIterableConsumer(String queueName, KeyValue properties) {
+ throw new OMSNotSupportedException("-1", "IterableConsumer is not supported in current version");
+ }
+
+ @Override
+ public ResourceManager getResourceManager() {
+ throw new OMSNotSupportedException("-1", "ResourceManager is not supported in current version.");
+ }
+
+ @Override
+ public ServiceEndPoint createServiceEndPoint() {
+ throw new OMSNotSupportedException("-1", "ServiceEndPoint is not supported in current version.");
+ }
+
+ @Override
+ public ServiceEndPoint createServiceEndPoint(KeyValue properties) {
+ throw new OMSNotSupportedException("-1", "ServiceEndPoint is not supported in current version.");
+ }
+
+ @Override
+ public void addObserver(Observer observer) {
+ //Ignore
+ }
+
+ @Override
+ public void deleteObserver(Observer observer) {
+ //Ignore
+ }
+
+ @Override
+ public void startup() {
+ //Ignore
+ }
+
+ @Override
+ public void shutdown() {
+ //Ignore
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/config/ClientConfig.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/config/ClientConfig.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/config/ClientConfig.java
new file mode 100644
index 0000000..7077c6d
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/config/ClientConfig.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.config;
+
+import io.openmessaging.PropertyKeys;
+import io.openmessaging.rocketmq.domain.NonStandardKeys;
+
+public class ClientConfig implements PropertyKeys, NonStandardKeys {
+ private String omsDriverImpl;
+ private String omsAccessPoints;
+ private String omsNamespace;
+ private String omsProducerId;
+ private String omsConsumerId;
+ private int omsOperationTimeout = 5000;
+ private String omsRoutingName;
+ private String omsOperatorName;
+ private String omsDstQueue;
+ private String omsSrcTopic;
+ private String rmqConsumerGroup;
+ private String rmqProducerGroup = "__OMS_PRODUCER_DEFAULT_GROUP";
+ private int rmqMaxRedeliveryTimes = 16;
+ private int rmqMessageConsumeTimeout = 15; //In minutes
+ private int rmqMaxConsumeThreadNums = 64;
+ private int rmqMinConsumeThreadNums = 20;
+ private String rmqMessageDestination;
+ private int rmqPullMessageBatchNums = 32;
+ private int rmqPullMessageCacheCapacity = 1000;
+
+ public String getOmsDriverImpl() {
+ return omsDriverImpl;
+ }
+
+ public void setOmsDriverImpl(final String omsDriverImpl) {
+ this.omsDriverImpl = omsDriverImpl;
+ }
+
+ public String getOmsAccessPoints() {
+ return omsAccessPoints;
+ }
+
+ public void setOmsAccessPoints(final String omsAccessPoints) {
+ this.omsAccessPoints = omsAccessPoints;
+ }
+
+ public String getOmsNamespace() {
+ return omsNamespace;
+ }
+
+ public void setOmsNamespace(final String omsNamespace) {
+ this.omsNamespace = omsNamespace;
+ }
+
+ public String getOmsProducerId() {
+ return omsProducerId;
+ }
+
+ public void setOmsProducerId(final String omsProducerId) {
+ this.omsProducerId = omsProducerId;
+ }
+
+ public String getOmsConsumerId() {
+ return omsConsumerId;
+ }
+
+ public void setOmsConsumerId(final String omsConsumerId) {
+ this.omsConsumerId = omsConsumerId;
+ }
+
+ public int getOmsOperationTimeout() {
+ return omsOperationTimeout;
+ }
+
+ public void setOmsOperationTimeout(final int omsOperationTimeout) {
+ this.omsOperationTimeout = omsOperationTimeout;
+ }
+
+ public String getOmsRoutingName() {
+ return omsRoutingName;
+ }
+
+ public void setOmsRoutingName(final String omsRoutingName) {
+ this.omsRoutingName = omsRoutingName;
+ }
+
+ public String getOmsOperatorName() {
+ return omsOperatorName;
+ }
+
+ public void setOmsOperatorName(final String omsOperatorName) {
+ this.omsOperatorName = omsOperatorName;
+ }
+
+ public String getOmsDstQueue() {
+ return omsDstQueue;
+ }
+
+ public void setOmsDstQueue(final String omsDstQueue) {
+ this.omsDstQueue = omsDstQueue;
+ }
+
+ public String getOmsSrcTopic() {
+ return omsSrcTopic;
+ }
+
+ public void setOmsSrcTopic(final String omsSrcTopic) {
+ this.omsSrcTopic = omsSrcTopic;
+ }
+
+ public String getRmqConsumerGroup() {
+ return rmqConsumerGroup;
+ }
+
+ public void setRmqConsumerGroup(final String rmqConsumerGroup) {
+ this.rmqConsumerGroup = rmqConsumerGroup;
+ }
+
+ public String getRmqProducerGroup() {
+ return rmqProducerGroup;
+ }
+
+ public void setRmqProducerGroup(final String rmqProducerGroup) {
+ this.rmqProducerGroup = rmqProducerGroup;
+ }
+
+ public int getRmqMaxRedeliveryTimes() {
+ return rmqMaxRedeliveryTimes;
+ }
+
+ public void setRmqMaxRedeliveryTimes(final int rmqMaxRedeliveryTimes) {
+ this.rmqMaxRedeliveryTimes = rmqMaxRedeliveryTimes;
+ }
+
+ public int getRmqMessageConsumeTimeout() {
+ return rmqMessageConsumeTimeout;
+ }
+
+ public void setRmqMessageConsumeTimeout(final int rmqMessageConsumeTimeout) {
+ this.rmqMessageConsumeTimeout = rmqMessageConsumeTimeout;
+ }
+
+ public int getRmqMaxConsumeThreadNums() {
+ return rmqMaxConsumeThreadNums;
+ }
+
+ public void setRmqMaxConsumeThreadNums(final int rmqMaxConsumeThreadNums) {
+ this.rmqMaxConsumeThreadNums = rmqMaxConsumeThreadNums;
+ }
+
+ public int getRmqMinConsumeThreadNums() {
+ return rmqMinConsumeThreadNums;
+ }
+
+ public void setRmqMinConsumeThreadNums(final int rmqMinConsumeThreadNums) {
+ this.rmqMinConsumeThreadNums = rmqMinConsumeThreadNums;
+ }
+
+ public String getRmqMessageDestination() {
+ return rmqMessageDestination;
+ }
+
+ public void setRmqMessageDestination(final String rmqMessageDestination) {
+ this.rmqMessageDestination = rmqMessageDestination;
+ }
+
+ public int getRmqPullMessageBatchNums() {
+ return rmqPullMessageBatchNums;
+ }
+
+ public void setRmqPullMessageBatchNums(final int rmqPullMessageBatchNums) {
+ this.rmqPullMessageBatchNums = rmqPullMessageBatchNums;
+ }
+
+ public int getRmqPullMessageCacheCapacity() {
+ return rmqPullMessageCacheCapacity;
+ }
+
+ public void setRmqPullMessageCacheCapacity(final int rmqPullMessageCacheCapacity) {
+ this.rmqPullMessageCacheCapacity = rmqPullMessageCacheCapacity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java
new file mode 100644
index 0000000..90f9e03
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.consumer;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.PropertyKeys;
+import io.openmessaging.ServiceLifecycle;
+import io.openmessaging.rocketmq.config.ClientConfig;
+import io.openmessaging.rocketmq.domain.ConsumeRequest;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.consumer.ProcessQueue;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.common.ThreadFactoryImpl;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.utils.ThreadUtils;
+import org.slf4j.Logger;
+
+class LocalMessageCache implements ServiceLifecycle {
+ private final BlockingQueue<ConsumeRequest> consumeRequestCache;
+ private final Map<String, ConsumeRequest> consumedRequest;
+ private final ConcurrentHashMap<MessageQueue, Long> pullOffsetTable;
+ private final DefaultMQPullConsumer rocketmqPullConsumer;
+ private final ClientConfig clientConfig;
+ private final ScheduledExecutorService cleanExpireMsgExecutors;
+
+ private final static Logger log = ClientLogger.getLog();
+
+ LocalMessageCache(final DefaultMQPullConsumer rocketmqPullConsumer, final ClientConfig clientConfig) {
+ consumeRequestCache = new LinkedBlockingQueue<>(clientConfig.getRmqPullMessageCacheCapacity());
+ this.consumedRequest = new ConcurrentHashMap<>();
+ this.pullOffsetTable = new ConcurrentHashMap<>();
+ this.rocketmqPullConsumer = rocketmqPullConsumer;
+ this.clientConfig = clientConfig;
+ this.cleanExpireMsgExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(
+ "OMS_CleanExpireMsgScheduledThread_"));
+ }
+
+ int nextPullBatchNums() {
+ return Math.min(clientConfig.getRmqPullMessageBatchNums(), consumeRequestCache.remainingCapacity());
+ }
+
+ long nextPullOffset(MessageQueue remoteQueue) {
+ if (!pullOffsetTable.containsKey(remoteQueue)) {
+ try {
+ pullOffsetTable.putIfAbsent(remoteQueue,
+ rocketmqPullConsumer.fetchConsumeOffset(remoteQueue, false));
+ } catch (MQClientException e) {
+ log.error("A error occurred in fetch consume offset process.", e);
+ }
+ }
+ return pullOffsetTable.get(remoteQueue);
+ }
+
+ void updatePullOffset(MessageQueue remoteQueue, long nextPullOffset) {
+ pullOffsetTable.put(remoteQueue, nextPullOffset);
+ }
+
+ void submitConsumeRequest(ConsumeRequest consumeRequest) {
+ try {
+ consumeRequestCache.put(consumeRequest);
+ } catch (InterruptedException ignore) {
+ }
+ }
+
+ MessageExt poll() {
+ return poll(clientConfig.getOmsOperationTimeout());
+ }
+
+ MessageExt poll(final KeyValue properties) {
+ int currentPollTimeout = clientConfig.getOmsOperationTimeout();
+ if (properties.containsKey(PropertyKeys.OPERATION_TIMEOUT)) {
+ currentPollTimeout = properties.getInt(PropertyKeys.OPERATION_TIMEOUT);
+ }
+ return poll(currentPollTimeout);
+ }
+
+ private MessageExt poll(long timeout) {
+ try {
+ ConsumeRequest consumeRequest = consumeRequestCache.poll(timeout, TimeUnit.MILLISECONDS);
+ if (consumeRequest != null) {
+ MessageExt messageExt = consumeRequest.getMessageExt();
+ consumeRequest.setStartConsumeTimeMillis(System.currentTimeMillis());
+ MessageAccessor.setConsumeStartTimeStamp(messageExt, String.valueOf(consumeRequest.getStartConsumeTimeMillis()));
+ consumedRequest.put(messageExt.getMsgId(), consumeRequest);
+ return messageExt;
+ }
+ } catch (InterruptedException ignore) {
+ }
+ return null;
+ }
+
+ void ack(final String messageId) {
+ ConsumeRequest consumeRequest = consumedRequest.remove(messageId);
+ if (consumeRequest != null) {
+ long offset = consumeRequest.getProcessQueue().removeMessage(Collections.singletonList(consumeRequest.getMessageExt()));
+ try {
+ rocketmqPullConsumer.updateConsumeOffset(consumeRequest.getMessageQueue(), offset);
+ } catch (MQClientException e) {
+ log.error("A error occurred in update consume offset process.", e);
+ }
+ }
+ }
+
+ void ack(final MessageQueue messageQueue, final ProcessQueue processQueue, final MessageExt messageExt) {
+ consumedRequest.remove(messageExt.getMsgId());
+ long offset = processQueue.removeMessage(Collections.singletonList(messageExt));
+ try {
+ rocketmqPullConsumer.updateConsumeOffset(messageQueue, offset);
+ } catch (MQClientException e) {
+ log.error("A error occurred in update consume offset process.", e);
+ }
+ }
+
+ @Override
+ public void startup() {
+ this.cleanExpireMsgExecutors.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ cleanExpireMsg();
+ }
+ }, clientConfig.getRmqMessageConsumeTimeout(), clientConfig.getRmqMessageConsumeTimeout(), TimeUnit.MINUTES);
+ }
+
+ @Override
+ public void shutdown() {
+ ThreadUtils.shutdownGracefully(cleanExpireMsgExecutors, 5000, TimeUnit.MILLISECONDS);
+ }
+
+ private void cleanExpireMsg() {
+ for (final Map.Entry<MessageQueue, ProcessQueue> next : rocketmqPullConsumer.getDefaultMQPullConsumerImpl()
+ .getRebalanceImpl().getProcessQueueTable().entrySet()) {
+ ProcessQueue pq = next.getValue();
+ MessageQueue mq = next.getKey();
+ ReadWriteLock lockTreeMap = getLockInProcessQueue(pq);
+ if (lockTreeMap == null) {
+ log.error("Gets tree map lock in process queue error, may be has compatibility issue");
+ return;
+ }
+
+ TreeMap<Long, MessageExt> msgTreeMap = pq.getMsgTreeMap();
+
+ int loop = msgTreeMap.size();
+ for (int i = 0; i < loop; i++) {
+ MessageExt msg = null;
+ try {
+ lockTreeMap.readLock().lockInterruptibly();
+ try {
+ if (!msgTreeMap.isEmpty()) {
+ msg = msgTreeMap.firstEntry().getValue();
+ if (System.currentTimeMillis() - Long.parseLong(MessageAccessor.getConsumeStartTimeStamp(msg))
+ > clientConfig.getRmqMessageConsumeTimeout() * 60 * 1000) {
+ //Expired, ack and remove it.
+ } else {
+ break;
+ }
+ } else {
+ break;
+ }
+ } finally {
+ lockTreeMap.readLock().unlock();
+ }
+ } catch (InterruptedException e) {
+ log.error("Gets expired message exception", e);
+ }
+
+ try {
+ rocketmqPullConsumer.sendMessageBack(msg, 3);
+ log.info("Send expired msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}",
+ msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset());
+ ack(mq, pq, msg);
+ } catch (Exception e) {
+ log.error("Send back expired msg exception", e);
+ }
+ }
+ }
+ }
+
+ private ReadWriteLock getLockInProcessQueue(ProcessQueue pq) {
+ try {
+ return (ReadWriteLock) FieldUtils.readDeclaredField(pq, "lockTreeMap", true);
+ } catch (IllegalAccessException e) {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java
new file mode 100644
index 0000000..8d396d4
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.consumer;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.Message;
+import io.openmessaging.PropertyKeys;
+import io.openmessaging.PullConsumer;
+import io.openmessaging.exception.OMSRuntimeException;
+import io.openmessaging.rocketmq.config.ClientConfig;
+import io.openmessaging.rocketmq.domain.ConsumeRequest;
+import io.openmessaging.rocketmq.utils.BeanUtils;
+import io.openmessaging.rocketmq.utils.OMSUtil;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.client.consumer.MQPullConsumer;
+import org.apache.rocketmq.client.consumer.MQPullConsumerScheduleService;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullTaskCallback;
+import org.apache.rocketmq.client.consumer.PullTaskContext;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.consumer.ProcessQueue;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.slf4j.Logger;
+
+public class PullConsumerImpl implements PullConsumer {
+ private final DefaultMQPullConsumer rocketmqPullConsumer;
+ private final KeyValue properties;
+ private boolean started = false;
+ private String targetQueueName;
+ private final MQPullConsumerScheduleService pullConsumerScheduleService;
+ private final LocalMessageCache localMessageCache;
+ private final ClientConfig clientConfig;
+
+ final static Logger log = ClientLogger.getLog();
+
+ public PullConsumerImpl(final String queueName, final KeyValue properties) {
+ this.properties = properties;
+ this.targetQueueName = queueName;
+
+ this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);
+
+ String consumerGroup = clientConfig.getRmqConsumerGroup();
+ if (null == consumerGroup || consumerGroup.isEmpty()) {
+ throw new OMSRuntimeException("-1", "Consumer Group is necessary for RocketMQ, please set it.");
+ }
+ pullConsumerScheduleService = new MQPullConsumerScheduleService(consumerGroup);
+
+ this.rocketmqPullConsumer = pullConsumerScheduleService.getDefaultMQPullConsumer();
+
+ String accessPoints = clientConfig.getOmsAccessPoints();
+ if (accessPoints == null || accessPoints.isEmpty()) {
+ throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty.");
+ }
+ this.rocketmqPullConsumer.setNamesrvAddr(accessPoints.replace(',', ';'));
+
+ this.rocketmqPullConsumer.setConsumerGroup(consumerGroup);
+
+ int maxReDeliveryTimes = clientConfig.getRmqMaxRedeliveryTimes();
+ this.rocketmqPullConsumer.setMaxReconsumeTimes(maxReDeliveryTimes);
+
+ String consumerId = OMSUtil.buildInstanceName();
+ this.rocketmqPullConsumer.setInstanceName(consumerId);
+ properties.put(PropertyKeys.CONSUMER_ID, consumerId);
+
+ this.localMessageCache = new LocalMessageCache(this.rocketmqPullConsumer, clientConfig);
+ }
+
+ @Override
+ public KeyValue properties() {
+ return properties;
+ }
+
+ @Override
+ public Message poll() {
+ MessageExt rmqMsg = localMessageCache.poll();
+ return rmqMsg == null ? null : OMSUtil.msgConvert(rmqMsg);
+ }
+
+ @Override
+ public Message poll(final KeyValue properties) {
+ MessageExt rmqMsg = localMessageCache.poll(properties);
+ return rmqMsg == null ? null : OMSUtil.msgConvert(rmqMsg);
+ }
+
+ @Override
+ public void ack(final String messageId) {
+ localMessageCache.ack(messageId);
+ }
+
+ @Override
+ public void ack(final String messageId, final KeyValue properties) {
+ localMessageCache.ack(messageId);
+ }
+
+ @Override
+ public synchronized void startup() {
+ if (!started) {
+ try {
+ registerPullTaskCallback();
+ this.pullConsumerScheduleService.start();
+ this.localMessageCache.startup();
+ } catch (MQClientException e) {
+ throw new OMSRuntimeException("-1", e);
+ }
+ }
+ this.started = true;
+ }
+
+ private void registerPullTaskCallback() {
+ this.pullConsumerScheduleService.registerPullTaskCallback(targetQueueName, new PullTaskCallback() {
+ @Override
+ public void doPullTask(final MessageQueue mq, final PullTaskContext context) {
+ MQPullConsumer consumer = context.getPullConsumer();
+ try {
+ long offset = localMessageCache.nextPullOffset(mq);
+
+ PullResult pullResult = consumer.pull(mq, "*",
+ offset, localMessageCache.nextPullBatchNums());
+ ProcessQueue pq = rocketmqPullConsumer.getDefaultMQPullConsumerImpl().getRebalanceImpl()
+ .getProcessQueueTable().get(mq);
+ switch (pullResult.getPullStatus()) {
+ case FOUND:
+ if (pq != null) {
+ pq.putMessage(pullResult.getMsgFoundList());
+ for (final MessageExt messageExt : pullResult.getMsgFoundList()) {
+ localMessageCache.submitConsumeRequest(new ConsumeRequest(messageExt, mq, pq));
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ localMessageCache.updatePullOffset(mq, pullResult.getNextBeginOffset());
+ } catch (Exception e) {
+ log.error("A error occurred in pull message process.", e);
+ }
+ }
+ });
+ }
+
+ @Override
+ public synchronized void shutdown() {
+ if (this.started) {
+ this.localMessageCache.shutdown();
+ this.pullConsumerScheduleService.shutdown();
+ this.rocketmqPullConsumer.shutdown();
+ }
+ this.started = false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java
new file mode 100644
index 0000000..f9b8058
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.consumer;
+
+import io.openmessaging.BytesMessage;
+import io.openmessaging.KeyValue;
+import io.openmessaging.MessageListener;
+import io.openmessaging.OMS;
+import io.openmessaging.PropertyKeys;
+import io.openmessaging.PushConsumer;
+import io.openmessaging.ReceivedMessageContext;
+import io.openmessaging.exception.OMSRuntimeException;
+import io.openmessaging.rocketmq.config.ClientConfig;
+import io.openmessaging.rocketmq.domain.NonStandardKeys;
+import io.openmessaging.rocketmq.utils.BeanUtils;
+import io.openmessaging.rocketmq.utils.OMSUtil;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.message.MessageExt;
+
+public class PushConsumerImpl implements PushConsumer {
+ private final DefaultMQPushConsumer rocketmqPushConsumer;
+ private final KeyValue properties;
+ private boolean started = false;
+ private final Map<String, MessageListener> subscribeTable = new ConcurrentHashMap<>();
+ private final ClientConfig clientConfig;
+
+ public PushConsumerImpl(final KeyValue properties) {
+ this.rocketmqPushConsumer = new DefaultMQPushConsumer();
+ this.properties = properties;
+ this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);
+
+ String accessPoints = clientConfig.getOmsAccessPoints();
+ if (accessPoints == null || accessPoints.isEmpty()) {
+ throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty.");
+ }
+ this.rocketmqPushConsumer.setNamesrvAddr(accessPoints.replace(',', ';'));
+
+ String consumerGroup = clientConfig.getRmqConsumerGroup();
+ if (null == consumerGroup || consumerGroup.isEmpty()) {
+ throw new OMSRuntimeException("-1", "Consumer Group is necessary for RocketMQ, please set it.");
+ }
+ this.rocketmqPushConsumer.setConsumerGroup(consumerGroup);
+ this.rocketmqPushConsumer.setMaxReconsumeTimes(clientConfig.getRmqMaxRedeliveryTimes());
+ this.rocketmqPushConsumer.setConsumeTimeout(clientConfig.getRmqMessageConsumeTimeout());
+ this.rocketmqPushConsumer.setConsumeThreadMax(clientConfig.getRmqMaxConsumeThreadNums());
+ this.rocketmqPushConsumer.setConsumeThreadMin(clientConfig.getRmqMinConsumeThreadNums());
+
+ String consumerId = OMSUtil.buildInstanceName();
+ this.rocketmqPushConsumer.setInstanceName(consumerId);
+ properties.put(PropertyKeys.CONSUMER_ID, consumerId);
+
+ this.rocketmqPushConsumer.registerMessageListener(new MessageListenerImpl());
+ }
+
+ @Override
+ public KeyValue properties() {
+ return properties;
+ }
+
+ @Override
+ public void resume() {
+ this.rocketmqPushConsumer.resume();
+ }
+
+ @Override
+ public void suspend() {
+ this.rocketmqPushConsumer.suspend();
+ }
+
+ @Override
+ public boolean isSuspended() {
+ return this.rocketmqPushConsumer.getDefaultMQPushConsumerImpl().isPause();
+ }
+
+ @Override
+ public PushConsumer attachQueue(final String queueName, final MessageListener listener) {
+ this.subscribeTable.put(queueName, listener);
+ try {
+ this.rocketmqPushConsumer.subscribe(queueName, "*");
+ } catch (MQClientException e) {
+ throw new OMSRuntimeException("-1", String.format("RocketMQ push consumer can't attach to %s.", queueName));
+ }
+ return this;
+ }
+
+ @Override
+ public synchronized void startup() {
+ if (!started) {
+ try {
+ this.rocketmqPushConsumer.start();
+ } catch (MQClientException e) {
+ throw new OMSRuntimeException("-1", e);
+ }
+ }
+ this.started = true;
+ }
+
+ @Override
+ public synchronized void shutdown() {
+ if (this.started) {
+ this.rocketmqPushConsumer.shutdown();
+ }
+ this.started = false;
+ }
+
+ class MessageListenerImpl implements MessageListenerConcurrently {
+
+ @Override
+ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> rmqMsgList,
+ ConsumeConcurrentlyContext contextRMQ) {
+ MessageExt rmqMsg = rmqMsgList.get(0);
+ BytesMessage omsMsg = OMSUtil.msgConvert(rmqMsg);
+
+ MessageListener listener = PushConsumerImpl.this.subscribeTable.get(rmqMsg.getTopic());
+
+ if (listener == null) {
+ throw new OMSRuntimeException("-1",
+ String.format("The topic/queue %s isn't attached to this consumer", rmqMsg.getTopic()));
+ }
+
+ final KeyValue contextProperties = OMS.newKeyValue();
+ final CountDownLatch sync = new CountDownLatch(1);
+
+ contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, ConsumeConcurrentlyStatus.RECONSUME_LATER.name());
+
+ ReceivedMessageContext context = new ReceivedMessageContext() {
+ @Override
+ public KeyValue properties() {
+ return contextProperties;
+ }
+
+ @Override
+ public void ack() {
+ sync.countDown();
+ contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS,
+ ConsumeConcurrentlyStatus.CONSUME_SUCCESS.name());
+ }
+
+ @Override
+ public void ack(final KeyValue properties) {
+ sync.countDown();
+ contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS,
+ properties.getString(NonStandardKeys.MESSAGE_CONSUME_STATUS));
+ }
+ };
+ long begin = System.currentTimeMillis();
+ listener.onMessage(omsMsg, context);
+ long costs = System.currentTimeMillis() - begin;
+ long timeoutMills = clientConfig.getRmqMessageConsumeTimeout() * 60 * 1000;
+ try {
+ sync.await(Math.max(0, timeoutMills - costs), TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ignore) {
+ }
+
+ return ConsumeConcurrentlyStatus.valueOf(contextProperties.getString(NonStandardKeys.MESSAGE_CONSUME_STATUS));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java
new file mode 100644
index 0000000..43f80ae
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.domain;
+
+import io.openmessaging.BytesMessage;
+import io.openmessaging.KeyValue;
+import io.openmessaging.Message;
+import io.openmessaging.OMS;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+
+public class BytesMessageImpl implements BytesMessage {
+ private KeyValue headers;
+ private KeyValue properties;
+ private byte[] body;
+
+ public BytesMessageImpl() {
+ this.headers = OMS.newKeyValue();
+ this.properties = OMS.newKeyValue();
+ }
+
+ @Override
+ public byte[] getBody() {
+ return body;
+ }
+
+ @Override
+ public BytesMessage setBody(final byte[] body) {
+ this.body = body;
+ return this;
+ }
+
+ @Override
+ public KeyValue headers() {
+ return headers;
+ }
+
+ @Override
+ public KeyValue properties() {
+ return properties;
+ }
+
+ @Override
+ public Message putHeaders(final String key, final int value) {
+ headers.put(key, value);
+ return this;
+ }
+
+ @Override
+ public Message putHeaders(final String key, final long value) {
+ headers.put(key, value);
+ return this;
+ }
+
+ @Override
+ public Message putHeaders(final String key, final double value) {
+ headers.put(key, value);
+ return this;
+ }
+
+ @Override
+ public Message putHeaders(final String key, final String value) {
+ headers.put(key, value);
+ return this;
+ }
+
+ @Override
+ public Message putProperties(final String key, final int value) {
+ properties.put(key, value);
+ return this;
+ }
+
+ @Override
+ public Message putProperties(final String key, final long value) {
+ properties.put(key, value);
+ return this;
+ }
+
+ @Override
+ public Message putProperties(final String key, final double value) {
+ properties.put(key, value);
+ return this;
+ }
+
+ @Override
+ public Message putProperties(final String key, final String value) {
+ properties.put(key, value);
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return ToStringBuilder.reflectionToString(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/ConsumeRequest.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/ConsumeRequest.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/ConsumeRequest.java
new file mode 100644
index 0000000..7ce4a9b
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/ConsumeRequest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.domain;
+
+import org.apache.rocketmq.client.impl.consumer.ProcessQueue;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+
+public class ConsumeRequest {
+ private final MessageExt messageExt;
+ private final MessageQueue messageQueue;
+ private final ProcessQueue processQueue;
+ private long startConsumeTimeMillis;
+
+ public ConsumeRequest(final MessageExt messageExt, final MessageQueue messageQueue,
+ final ProcessQueue processQueue) {
+ this.messageExt = messageExt;
+ this.messageQueue = messageQueue;
+ this.processQueue = processQueue;
+ }
+
+ public MessageExt getMessageExt() {
+ return messageExt;
+ }
+
+ public MessageQueue getMessageQueue() {
+ return messageQueue;
+ }
+
+ public ProcessQueue getProcessQueue() {
+ return processQueue;
+ }
+
+ public long getStartConsumeTimeMillis() {
+ return startConsumeTimeMillis;
+ }
+
+ public void setStartConsumeTimeMillis(final long startConsumeTimeMillis) {
+ this.startConsumeTimeMillis = startConsumeTimeMillis;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/NonStandardKeys.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/NonStandardKeys.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/NonStandardKeys.java
new file mode 100644
index 0000000..3639a3f
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/NonStandardKeys.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.domain;
+
+public interface NonStandardKeys {
+ String CONSUMER_GROUP = "rmq.consumer.group";
+ String PRODUCER_GROUP = "rmq.producer.group";
+ String MAX_REDELIVERY_TIMES = "rmq.max.redelivery.times";
+ String MESSAGE_CONSUME_TIMEOUT = "rmq.message.consume.timeout";
+ String MAX_CONSUME_THREAD_NUMS = "rmq.max.consume.thread.nums";
+ String MIN_CONSUME_THREAD_NUMS = "rmq.min.consume.thread.nums";
+ String MESSAGE_CONSUME_STATUS = "rmq.message.consume.status";
+ String MESSAGE_DESTINATION = "rmq.message.destination";
+ String PULL_MESSAGE_BATCH_NUMS = "rmq.pull.message.batch.nums";
+ String PULL_MESSAGE_CACHE_CAPACITY = "rmq.pull.message.cache.capacity";
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/SendResultImpl.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/SendResultImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/SendResultImpl.java
new file mode 100644
index 0000000..228a9f0
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/domain/SendResultImpl.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.domain;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.SendResult;
+
+public class SendResultImpl implements SendResult {
+ private String messageId;
+ private KeyValue properties;
+
+ public SendResultImpl(final String messageId, final KeyValue properties) {
+ this.messageId = messageId;
+ this.properties = properties;
+ }
+
+ @Override
+ public String messageId() {
+ return messageId;
+ }
+
+ @Override
+ public KeyValue properties() {
+ return properties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java
new file mode 100644
index 0000000..8246bcd
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.producer;
+
+import io.openmessaging.BytesMessage;
+import io.openmessaging.KeyValue;
+import io.openmessaging.Message;
+import io.openmessaging.MessageFactory;
+import io.openmessaging.MessageHeader;
+import io.openmessaging.PropertyKeys;
+import io.openmessaging.ServiceLifecycle;
+import io.openmessaging.exception.OMSMessageFormatException;
+import io.openmessaging.exception.OMSNotSupportedException;
+import io.openmessaging.exception.OMSRuntimeException;
+import io.openmessaging.exception.OMSTimeOutException;
+import io.openmessaging.rocketmq.config.ClientConfig;
+import io.openmessaging.rocketmq.domain.BytesMessageImpl;
+import io.openmessaging.rocketmq.utils.BeanUtils;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.exception.RemotingConnectException;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
+import org.slf4j.Logger;
+
+import static io.openmessaging.rocketmq.utils.OMSUtil.buildInstanceName;
+
+abstract class AbstractOMSProducer implements ServiceLifecycle, MessageFactory {
+ final static Logger log = ClientLogger.getLog();
+ final KeyValue properties;
+ final DefaultMQProducer rocketmqProducer;
+ private boolean started = false;
+ final ClientConfig clientConfig;
+
+ AbstractOMSProducer(final KeyValue properties) {
+ this.properties = properties;
+ this.rocketmqProducer = new DefaultMQProducer();
+ this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);
+
+ String accessPoints = clientConfig.getOmsAccessPoints();
+ if (accessPoints == null || accessPoints.isEmpty()) {
+ throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty.");
+ }
+ this.rocketmqProducer.setNamesrvAddr(accessPoints.replace(',', ';'));
+ this.rocketmqProducer.setProducerGroup(clientConfig.getRmqProducerGroup());
+
+ String producerId = buildInstanceName();
+ this.rocketmqProducer.setSendMsgTimeout(clientConfig.getOmsOperationTimeout());
+ this.rocketmqProducer.setInstanceName(producerId);
+ this.rocketmqProducer.setMaxMessageSize(1024 * 1024 * 4);
+ properties.put(PropertyKeys.PRODUCER_ID, producerId);
+ }
+
+ @Override
+ public synchronized void startup() {
+ if (!started) {
+ try {
+ this.rocketmqProducer.start();
+ } catch (MQClientException e) {
+ throw new OMSRuntimeException("-1", e);
+ }
+ }
+ this.started = true;
+ }
+
+ @Override
+ public synchronized void shutdown() {
+ if (this.started) {
+ this.rocketmqProducer.shutdown();
+ }
+ this.started = false;
+ }
+
+ OMSRuntimeException checkProducerException(String topic, String msgId, Throwable e) {
+ if (e instanceof MQClientException) {
+ if (e.getCause() != null) {
+ if (e.getCause() instanceof RemotingTimeoutException) {
+ return new OMSTimeOutException("-1", String.format("Send message to broker timeout, %dms, Topic=%s, msgId=%s",
+ this.rocketmqProducer.getSendMsgTimeout(), topic, msgId), e);
+ } else if (e.getCause() instanceof MQBrokerException || e.getCause() instanceof RemotingConnectException) {
+ MQBrokerException brokerException = (MQBrokerException) e.getCause();
+ return new OMSRuntimeException("-1", String.format("Received a broker exception, Topic=%s, msgId=%s, %s",
+ topic, msgId, brokerException.getErrorMessage()), e);
+ }
+ }
+ // Exception thrown by local.
+ else {
+ MQClientException clientException = (MQClientException) e;
+ if (-1 == clientException.getResponseCode()) {
+ return new OMSRuntimeException("-1", String.format("Topic does not exist, Topic=%s, msgId=%s",
+ topic, msgId), e);
+ } else if (ResponseCode.MESSAGE_ILLEGAL == clientException.getResponseCode()) {
+ return new OMSMessageFormatException("-1", String.format("A illegal message for RocketMQ, Topic=%s, msgId=%s",
+ topic, msgId), e);
+ }
+ }
+ }
+ return new OMSRuntimeException("-1", "Send message to RocketMQ broker failed.", e);
+ }
+
+ protected void checkMessageType(Message message) {
+ if (!(message instanceof BytesMessage)) {
+ throw new OMSNotSupportedException("-1", "Only BytesMessage is supported.");
+ }
+ }
+
+ @Override
+ public BytesMessage createBytesMessageToTopic(final String topic, final byte[] body) {
+ BytesMessage bytesMessage = new BytesMessageImpl();
+ bytesMessage.setBody(body);
+ bytesMessage.headers().put(MessageHeader.TOPIC, topic);
+ return bytesMessage;
+ }
+
+ @Override
+ public BytesMessage createBytesMessageToQueue(final String queue, final byte[] body) {
+ BytesMessage bytesMessage = new BytesMessageImpl();
+ bytesMessage.setBody(body);
+ bytesMessage.headers().put(MessageHeader.QUEUE, queue);
+ return bytesMessage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/ProducerImpl.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/ProducerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/ProducerImpl.java
new file mode 100644
index 0000000..2c00c60
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/ProducerImpl.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.producer;
+
+import io.openmessaging.BytesMessage;
+import io.openmessaging.KeyValue;
+import io.openmessaging.Message;
+import io.openmessaging.MessageHeader;
+import io.openmessaging.Producer;
+import io.openmessaging.Promise;
+import io.openmessaging.PropertyKeys;
+import io.openmessaging.SendResult;
+import io.openmessaging.exception.OMSRuntimeException;
+import io.openmessaging.rocketmq.promise.DefaultPromise;
+import io.openmessaging.rocketmq.utils.OMSUtil;
+import org.apache.rocketmq.client.producer.SendCallback;
+import org.apache.rocketmq.client.producer.SendStatus;
+
+import static io.openmessaging.rocketmq.utils.OMSUtil.msgConvert;
+
+public class ProducerImpl extends AbstractOMSProducer implements Producer {
+
+ public ProducerImpl(final KeyValue properties) {
+ super(properties);
+ }
+
+ @Override
+ public KeyValue properties() {
+ return properties;
+ }
+
+ @Override
+ public SendResult send(final Message message) {
+ return send(message, this.rocketmqProducer.getSendMsgTimeout());
+ }
+
+ @Override
+ public SendResult send(final Message message, final KeyValue properties) {
+ long timeout = properties.containsKey(PropertyKeys.OPERATION_TIMEOUT)
+ ? properties.getInt(PropertyKeys.OPERATION_TIMEOUT) : this.rocketmqProducer.getSendMsgTimeout();
+ return send(message, timeout);
+ }
+
+ private SendResult send(final Message message, long timeout) {
+ checkMessageType(message);
+ org.apache.rocketmq.common.message.Message rmqMessage = msgConvert((BytesMessage) message);
+ try {
+ org.apache.rocketmq.client.producer.SendResult rmqResult = this.rocketmqProducer.send(rmqMessage, timeout);
+ if (!rmqResult.getSendStatus().equals(SendStatus.SEND_OK)) {
+ log.error(String.format("Send message to RocketMQ failed, %s", message));
+ throw new OMSRuntimeException("-1", "Send message to RocketMQ broker failed.");
+ }
+ message.headers().put(MessageHeader.MESSAGE_ID, rmqResult.getMsgId());
+ return OMSUtil.sendResultConvert(rmqResult);
+ } catch (Exception e) {
+ log.error(String.format("Send message to RocketMQ failed, %s", message), e);
+ throw checkProducerException(rmqMessage.getTopic(), message.headers().getString(MessageHeader.MESSAGE_ID), e);
+ }
+ }
+
+ @Override
+ public Promise<SendResult> sendAsync(final Message message) {
+ return sendAsync(message, this.rocketmqProducer.getSendMsgTimeout());
+ }
+
+ @Override
+ public Promise<SendResult> sendAsync(final Message message, final KeyValue properties) {
+ long timeout = properties.containsKey(PropertyKeys.OPERATION_TIMEOUT)
+ ? properties.getInt(PropertyKeys.OPERATION_TIMEOUT) : this.rocketmqProducer.getSendMsgTimeout();
+ return sendAsync(message, timeout);
+ }
+
+ private Promise<SendResult> sendAsync(final Message message, long timeout) {
+ checkMessageType(message);
+ org.apache.rocketmq.common.message.Message rmqMessage = msgConvert((BytesMessage) message);
+ final Promise<SendResult> promise = new DefaultPromise<>();
+ try {
+ this.rocketmqProducer.send(rmqMessage, new SendCallback() {
+ @Override
+ public void onSuccess(final org.apache.rocketmq.client.producer.SendResult rmqResult) {
+ message.headers().put(MessageHeader.MESSAGE_ID, rmqResult.getMsgId());
+ promise.set(OMSUtil.sendResultConvert(rmqResult));
+ }
+
+ @Override
+ public void onException(final Throwable e) {
+ promise.setFailure(e);
+ }
+ }, timeout);
+ } catch (Exception e) {
+ promise.setFailure(e);
+ }
+ return promise;
+ }
+
+ @Override
+ public void sendOneway(final Message message) {
+ checkMessageType(message);
+ org.apache.rocketmq.common.message.Message rmqMessage = msgConvert((BytesMessage) message);
+ try {
+ this.rocketmqProducer.sendOneway(rmqMessage);
+ } catch (Exception ignore) { //Ignore the oneway exception.
+ }
+ }
+
+ @Override
+ public void sendOneway(final Message message, final KeyValue properties) {
+ sendOneway(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/SequenceProducerImpl.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/SequenceProducerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/SequenceProducerImpl.java
new file mode 100644
index 0000000..05225cc
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/producer/SequenceProducerImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.producer;
+
+import io.openmessaging.BytesMessage;
+import io.openmessaging.KeyValue;
+import io.openmessaging.Message;
+import io.openmessaging.MessageHeader;
+import io.openmessaging.SequenceProducer;
+import io.openmessaging.rocketmq.utils.OMSUtil;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.rocketmq.client.Validators;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.SendResult;
+
+public class SequenceProducerImpl extends AbstractOMSProducer implements SequenceProducer {
+
+ private BlockingQueue<Message> msgCacheQueue;
+
+ public SequenceProducerImpl(final KeyValue properties) {
+ super(properties);
+ this.msgCacheQueue = new LinkedBlockingQueue<>();
+ }
+
+ @Override
+ public KeyValue properties() {
+ return properties;
+ }
+
+ @Override
+ public void send(final Message message) {
+ checkMessageType(message);
+ org.apache.rocketmq.common.message.Message rmqMessage = OMSUtil.msgConvert((BytesMessage) message);
+ try {
+ Validators.checkMessage(rmqMessage, this.rocketmqProducer);
+ } catch (MQClientException e) {
+ throw checkProducerException(rmqMessage.getTopic(), message.headers().getString(MessageHeader.MESSAGE_ID), e);
+ }
+ msgCacheQueue.add(message);
+ }
+
+ @Override
+ public void send(final Message message, final KeyValue properties) {
+ send(message);
+ }
+
+ @Override
+ public synchronized void commit() {
+ List<Message> messages = new ArrayList<>();
+ msgCacheQueue.drainTo(messages);
+
+ List<org.apache.rocketmq.common.message.Message> rmqMessages = new ArrayList<>();
+
+ for (Message message : messages) {
+ rmqMessages.add(OMSUtil.msgConvert((BytesMessage) message));
+ }
+
+ if (rmqMessages.size() == 0) {
+ return;
+ }
+
+ try {
+ SendResult sendResult = this.rocketmqProducer.send(rmqMessages);
+ String[] msgIdArray = sendResult.getMsgId().split(",");
+ for (int i = 0; i < messages.size(); i++) {
+ Message message = messages.get(i);
+ message.headers().put(MessageHeader.MESSAGE_ID, msgIdArray[i]);
+ }
+ } catch (Exception e) {
+ throw checkProducerException("", "", e);
+ }
+ }
+
+ @Override
+ public synchronized void rollback() {
+ msgCacheQueue.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java
new file mode 100644
index 0000000..c863ccf
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.promise;
+
+import io.openmessaging.Promise;
+import io.openmessaging.PromiseListener;
+import io.openmessaging.exception.OMSRuntimeException;
+import java.util.ArrayList;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultPromise<V> implements Promise<V> {
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultPromise.class);
+ private final Object lock = new Object();
+ private volatile FutureState state = FutureState.DOING;
+ private V result = null;
+ private long timeout;
+ private long createTime;
+ private Throwable exception = null;
+ private List<PromiseListener<V>> promiseListenerList;
+
+ public DefaultPromise() {
+ createTime = System.currentTimeMillis();
+ promiseListenerList = new ArrayList<>();
+ timeout = 5000;
+ }
+
+ @Override
+ public boolean cancel(final boolean mayInterruptIfRunning) {
+ return false;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return state.isCancelledState();
+ }
+
+ @Override
+ public boolean isDone() {
+ return state.isDoneState();
+ }
+
+ @Override
+ public V get() {
+ return result;
+ }
+
+ @Override
+ public V get(final long timeout) {
+ synchronized (lock) {
+ if (!isDoing()) {
+ return getValueOrThrowable();
+ }
+
+ if (timeout <= 0) {
+ try {
+ lock.wait();
+ } catch (Exception e) {
+ cancel(e);
+ }
+ return getValueOrThrowable();
+ } else {
+ long waitTime = timeout - (System.currentTimeMillis() - createTime);
+ if (waitTime > 0) {
+ for (;; ) {
+ try {
+ lock.wait(waitTime);
+ } catch (InterruptedException e) {
+ LOG.error("promise get value interrupted,excepiton:{}", e.getMessage());
+ }
+
+ if (!isDoing()) {
+ break;
+ } else {
+ waitTime = timeout - (System.currentTimeMillis() - createTime);
+ if (waitTime <= 0) {
+ break;
+ }
+ }
+ }
+ }
+
+ if (isDoing()) {
+ timeoutSoCancel();
+ }
+ }
+ return getValueOrThrowable();
+ }
+ }
+
+ @Override
+ public boolean set(final V value) {
+ if (value == null)
+ return false;
+ this.result = value;
+ return done();
+ }
+
+ @Override
+ public boolean setFailure(final Throwable cause) {
+ if (cause == null)
+ return false;
+ this.exception = cause;
+ return done();
+ }
+
+ @Override
+ public void addListener(final PromiseListener<V> listener) {
+ if (listener == null) {
+ throw new NullPointerException("FutureListener is null");
+ }
+
+ boolean notifyNow = false;
+ synchronized (lock) {
+ if (!isDoing()) {
+ notifyNow = true;
+ } else {
+ if (promiseListenerList == null) {
+ promiseListenerList = new ArrayList<>();
+ }
+ promiseListenerList.add(listener);
+ }
+ }
+
+ if (notifyNow) {
+ notifyListener(listener);
+ }
+ }
+
+ @Override
+ public Throwable getThrowable() {
+ return exception;
+ }
+
+ private void notifyListeners() {
+ if (promiseListenerList != null) {
+ for (PromiseListener<V> listener : promiseListenerList) {
+ notifyListener(listener);
+ }
+ }
+ }
+
+ private boolean isSuccess() {
+ return isDone() && (exception == null);
+ }
+
+ private void timeoutSoCancel() {
+ synchronized (lock) {
+ if (!isDoing()) {
+ return;
+ }
+ state = FutureState.CANCELLED;
+ exception = new RuntimeException("Get request result is timeout or interrupted");
+ lock.notifyAll();
+ }
+ notifyListeners();
+ }
+
+ private V getValueOrThrowable() {
+ if (exception != null) {
+ Throwable e = exception.getCause() != null ? exception.getCause() : exception;
+ throw new OMSRuntimeException("-1", e);
+ }
+ notifyListeners();
+ return result;
+ }
+
+ private boolean isDoing() {
+ return state.isDoingState();
+ }
+
+ private boolean done() {
+ synchronized (lock) {
+ if (!isDoing()) {
+ return false;
+ }
+
+ state = FutureState.DONE;
+ lock.notifyAll();
+ }
+
+ notifyListeners();
+ return true;
+ }
+
+ private void notifyListener(final PromiseListener<V> listener) {
+ try {
+ if (exception != null)
+ listener.operationFailed(this);
+ else
+ listener.operationCompleted(this);
+ } catch (Throwable t) {
+ LOG.error("notifyListener {} Error:{}", listener.getClass().getSimpleName(), t);
+ }
+ }
+
+ private boolean cancel(Exception e) {
+ synchronized (lock) {
+ if (!isDoing()) {
+ return false;
+ }
+
+ state = FutureState.CANCELLED;
+ exception = e;
+ lock.notifyAll();
+ }
+
+ notifyListeners();
+ return true;
+ }
+}
+
[30/50] [abbrv] incubator-rocketmq git commit: Merge branch
'ROCKETMQ-206' into develop
Posted by do...@apache.org.
Merge branch 'ROCKETMQ-206' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/e5d01b41
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/e5d01b41
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/e5d01b41
Branch: refs/heads/release-4.1.0-incubating
Commit: e5d01b4121c3e17be8073752510a7ca78dd2bf76
Parents: 37fbb7b ceeef8e
Author: dongeforever <zh...@yeah.net>
Authored: Fri May 26 16:34:11 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Fri May 26 16:34:11 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/rocketmq/common/MixAll.java | 26 ++++++++------------
.../org/apache/rocketmq/common/MixAllTest.java | 20 +++++++++++++++
2 files changed, 30 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
[23/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-206] Fix bug
when non-1byte character exists in JSON config files.
Posted by do...@apache.org.
[ROCKETMQ-206] Fix bug when non-1byte character exists in JSON config files.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/ceeef8ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/ceeef8ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/ceeef8ec
Branch: refs/heads/release-4.1.0-incubating
Commit: ceeef8ec25758eea490b39400328dd8a702b0175
Parents: 1d966b5
Author: yukon <yu...@apache.org>
Authored: Thu May 25 13:48:22 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Thu May 25 13:48:22 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/rocketmq/common/MixAll.java | 26 ++++++++------------
.../org/apache/rocketmq/common/MixAllTest.java | 20 +++++++++++++++
2 files changed, 30 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/ceeef8ec/common/src/main/java/org/apache/rocketmq/common/MixAll.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
index e75efd9..36d81d0 100644
--- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java
+++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
@@ -18,7 +18,7 @@ package org.apache.rocketmq.common;
import java.io.ByteArrayInputStream;
import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
@@ -187,30 +187,24 @@ public class MixAll {
}
}
- public static String file2String(final String fileName) {
+ public static String file2String(final String fileName) throws IOException {
File file = new File(fileName);
return file2String(file);
}
- public static String file2String(final File file) {
+ public static String file2String(final File file) throws IOException {
if (file.exists()) {
- char[] data = new char[(int) file.length()];
- boolean result = false;
+ byte[] data = new byte[(int) file.length()];
+ boolean result;
- FileReader fileReader = null;
+ FileInputStream inputStream = null;
try {
- fileReader = new FileReader(file);
- int len = fileReader.read(data);
+ inputStream = new FileInputStream(file);
+ int len = inputStream.read(data);
result = len == data.length;
- } catch (IOException e) {
- // e.printStackTrace();
} finally {
- if (fileReader != null) {
- try {
- fileReader.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
+ if (inputStream != null) {
+ inputStream.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/ceeef8ec/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java b/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java
index 8220981..218b36d 100644
--- a/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java
+++ b/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java
@@ -21,6 +21,10 @@ import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetAddress;
+import java.nio.ByteOrder;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.Test;
@@ -68,6 +72,22 @@ public class MixAllTest {
}
@Test
+ public void testFile2String_WithChinese() throws IOException {
+ String fileName = System.getProperty("java.io.tmpdir") + File.separator + "MixAllTest" + System.currentTimeMillis();
+ File file = new File(fileName);
+ if (file.exists()) {
+ file.delete();
+ }
+ file.createNewFile();
+ PrintWriter out = new PrintWriter(fileName);
+ out.write("TestForMixAll_中文");
+ out.close();
+ String string = MixAll.file2String(fileName);
+ assertThat(string).isEqualTo("TestForMixAll_中文");
+ file.delete();
+ }
+
+ @Test
public void testString2File() throws IOException {
String fileName = System.getProperty("java.io.tmpdir") + File.separator + "MixAllTest" + System.currentTimeMillis();
MixAll.string2File("MixAll_testString2File", fileName);
[17/50] [abbrv] incubator-rocketmq git commit: Add javadoc to message
store.
Posted by do...@apache.org.
Add javadoc to message store.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/e9814ad4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/e9814ad4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/e9814ad4
Branch: refs/heads/release-4.1.0-incubating
Commit: e9814ad47e3c2fb277ee37db1eb48c6c43848404
Parents: 6898d96
Author: Zhanhui Li <li...@apache.org>
Authored: Sun May 7 23:07:03 2017 +0800
Committer: Zhanhui Li <li...@apache.org>
Committed: Sun May 7 23:07:03 2017 +0800
----------------------------------------------------------------------
.../broker/client/net/Broker2Client.java | 2 +-
.../longpolling/PullRequestHoldService.java | 4 +-
.../broker/offset/ConsumerOffsetManager.java | 4 +-
.../plugin/AbstractPluginMessageStore.java | 20 +-
.../broker/processor/AdminBrokerProcessor.java | 16 +-
.../processor/ConsumerManageProcessor.java | 2 +-
.../rocketmq/store/DefaultMessageStore.java | 16 +-
.../org/apache/rocketmq/store/MessageStore.java | 239 ++++++++++++++++++-
.../store/schedule/ScheduleMessageService.java | 2 +-
9 files changed, 259 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e9814ad4/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
index c00898c..863da62 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
@@ -149,7 +149,7 @@ public class Broker2Client {
long timeStampOffset;
if (timeStamp == -1) {
- timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i);
+ timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
} else {
timeStampOffset = this.brokerController.getMessageStore().getOffsetInQueueByTime(topic, i, timeStamp);
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e9814ad4/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
index 1a53db1..71f56a4 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
@@ -98,7 +98,7 @@ public class PullRequestHoldService extends ServiceThread {
if (2 == kArray.length) {
String topic = kArray[0];
int queueId = Integer.parseInt(kArray[1]);
- final long offset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, queueId);
+ final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
try {
this.notifyMessageArriving(topic, queueId, offset);
} catch (Throwable e) {
@@ -124,7 +124,7 @@ public class PullRequestHoldService extends ServiceThread {
for (PullRequest request : requestList) {
long newestOffset = maxOffset;
if (newestOffset <= request.getPullFromThisOffset()) {
- newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, queueId);
+ newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
}
if (newestOffset > request.getPullFromThisOffset()) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e9814ad4/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
index bdcf30c..769c4ad 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
@@ -73,7 +73,7 @@ public class ConsumerOffsetManager extends ConfigManager {
while (it.hasNext() && result) {
Entry<Integer, Long> next = it.next();
- long minOffsetInStore = this.brokerController.getMessageStore().getMinOffsetInQuque(topic, next.getKey());
+ long minOffsetInStore = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, next.getKey());
long offsetInPersist = next.getValue();
result = offsetInPersist <= minOffsetInStore;
}
@@ -201,7 +201,7 @@ public class ConsumerOffsetManager extends ConfigManager {
String[] topicGroupArr = topicGroup.split(TOPIC_GROUP_SEPARATOR);
if (topic.equals(topicGroupArr[0])) {
for (Entry<Integer, Long> entry : offSetEntry.getValue().entrySet()) {
- long minOffset = this.brokerController.getMessageStore().getMinOffsetInQuque(topic, entry.getKey());
+ long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, entry.getKey());
if (entry.getValue() >= minOffset) {
Long offset = queueMinOffset.get(entry.getKey());
if (offset == null) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e9814ad4/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java b/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
index 8ded973..690f70b 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
@@ -92,18 +92,18 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
}
@Override
- public long getMaxOffsetInQuque(String topic, int queueId) {
- return next.getMaxOffsetInQuque(topic, queueId);
+ public long getMaxOffsetInQueue(String topic, int queueId) {
+ return next.getMaxOffsetInQueue(topic, queueId);
}
@Override
- public long getMinOffsetInQuque(String topic, int queueId) {
- return next.getMinOffsetInQuque(topic, queueId);
+ public long getMinOffsetInQueue(String topic, int queueId) {
+ return next.getMinOffsetInQueue(topic, queueId);
}
@Override
- public long getCommitLogOffsetInQueue(String topic, int queueId, long cqOffset) {
- return next.getCommitLogOffsetInQueue(topic, queueId, cqOffset);
+ public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) {
+ return next.getCommitLogOffsetInQueue(topic, queueId, consumeQueueOffset);
}
@Override
@@ -152,8 +152,8 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
}
@Override
- public long getMessageStoreTimeStamp(String topic, int queueId, long offset) {
- return next.getMessageStoreTimeStamp(topic, queueId, offset);
+ public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) {
+ return next.getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset);
}
@Override
@@ -172,8 +172,8 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
}
@Override
- public void excuteDeleteFilesManualy() {
- next.excuteDeleteFilesManualy();
+ public void executeDeleteFilesManually() {
+ next.executeDeleteFilesManually();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e9814ad4/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
index daea53c..f59d295 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
@@ -376,7 +376,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
final GetMaxOffsetRequestHeader requestHeader =
(GetMaxOffsetRequestHeader) request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class);
- long offset = this.brokerController.getMessageStore().getMaxOffsetInQuque(requestHeader.getTopic(), requestHeader.getQueueId());
+ long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
responseHeader.setOffset(offset);
@@ -391,7 +391,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
final GetMinOffsetRequestHeader requestHeader =
(GetMinOffsetRequestHeader) request.decodeCommandCustomHeader(GetMinOffsetRequestHeader.class);
- long offset = this.brokerController.getMessageStore().getMinOffsetInQuque(requestHeader.getTopic(), requestHeader.getQueueId());
+ long offset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
responseHeader.setOffset(offset);
response.setCode(ResponseCode.SUCCESS);
@@ -537,11 +537,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
mq.setQueueId(i);
TopicOffset topicOffset = new TopicOffset();
- long min = this.brokerController.getMessageStore().getMinOffsetInQuque(topic, i);
+ long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i);
if (min < 0)
min = 0;
- long max = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i);
+ long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
if (max < 0)
max = 0;
@@ -679,7 +679,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
OffsetWrapper offsetWrapper = new OffsetWrapper();
- long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i);
+ long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
if (brokerOffset < 0)
brokerOffset = 0;
@@ -862,7 +862,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
long minTime = this.brokerController.getMessageStore().getEarliestMessageTime(topic, i);
timeSpan.setMinTimeStamp(minTime);
- long max = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i);
+ long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
long maxTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
timeSpan.setMaxTimeStamp(maxTime);
@@ -876,7 +876,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
}
timeSpan.setConsumeTimeStamp(consumeTime);
- long maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(requestHeader.getTopic(), i);
+ long maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i);
if (consumerOffset < maxBrokerOffset) {
long nextTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset);
timeSpan.setDelayTime(System.currentTimeMillis() - nextTime);
@@ -1126,7 +1126,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
mq.setQueueId(i);
OffsetWrapper offsetWrapper = new OffsetWrapper();
- long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i);
+ long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
if (brokerOffset < 0)
brokerOffset = 0;
long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(//
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e9814ad4/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
index 2c1029c..bb42705 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
@@ -135,7 +135,7 @@ public class ConsumerManageProcessor implements NettyRequestProcessor {
response.setRemark(null);
} else {
long minOffset =
- this.brokerController.getMessageStore().getMinOffsetInQuque(requestHeader.getTopic(),
+ this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(),
requestHeader.getQueueId());
if (minOffset <= 0
&& !this.brokerController.getMessageStore().checkInDiskByConsumeOffset(
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e9814ad4/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
index 7bed62c..931edc7 100644
--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
@@ -580,7 +580,7 @@ public class DefaultMessageStore implements MessageStore {
/**
*/
- public long getMaxOffsetInQuque(String topic, int queueId) {
+ public long getMaxOffsetInQueue(String topic, int queueId) {
ConsumeQueue logic = this.findConsumeQueue(topic, queueId);
if (logic != null) {
long offset = logic.getMaxOffsetInQueue();
@@ -593,7 +593,7 @@ public class DefaultMessageStore implements MessageStore {
/**
*/
- public long getMinOffsetInQuque(String topic, int queueId) {
+ public long getMinOffsetInQueue(String topic, int queueId) {
ConsumeQueue logic = this.findConsumeQueue(topic, queueId);
if (logic != null) {
return logic.getMinOffsetInQueue();
@@ -603,10 +603,10 @@ public class DefaultMessageStore implements MessageStore {
}
@Override
- public long getCommitLogOffsetInQueue(String topic, int queueId, long cqOffset) {
+ public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) {
ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId);
if (consumeQueue != null) {
- SelectMappedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(cqOffset);
+ SelectMappedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(consumeQueueOffset);
if (bufferConsumeQueue != null) {
try {
long offsetPy = bufferConsumeQueue.getByteBuffer().getLong();
@@ -740,10 +740,10 @@ public class DefaultMessageStore implements MessageStore {
}
@Override
- public long getMessageStoreTimeStamp(String topic, int queueId, long offset) {
+ public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) {
ConsumeQueue logicQueue = this.findConsumeQueue(topic, queueId);
if (logicQueue != null) {
- SelectMappedBufferResult result = logicQueue.getIndexBuffer(offset);
+ SelectMappedBufferResult result = logicQueue.getIndexBuffer(consumeQueueOffset);
if (result != null) {
try {
final long phyOffset = result.getByteBuffer().getLong();
@@ -798,7 +798,7 @@ public class DefaultMessageStore implements MessageStore {
}
@Override
- public void excuteDeleteFilesManualy() {
+ public void executeDeleteFilesManually() {
this.cleanCommitLogService.excuteDeleteFilesManualy();
}
@@ -1434,7 +1434,7 @@ public class DefaultMessageStore implements MessageStore {
public void excuteDeleteFilesManualy() {
this.manualDeleteFileSeveralTimes = MAX_MANUAL_DELETE_FILE_TIMES;
- DefaultMessageStore.log.info("excuteDeleteFilesManualy was invoked");
+ DefaultMessageStore.log.info("executeDeleteFilesManually was invoked");
}
public void run() {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e9814ad4/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
index e841c08..55572ce 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
@@ -22,91 +22,304 @@ import java.util.Set;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageExtBatch;
+/**
+ * This class defines contracting interfaces to implement, allowing third-party vendor to use customized message store.
+ */
public interface MessageStore {
+ /**
+ * Load previously stored messages.
+ * @return true if success; false otherwise.
+ */
boolean load();
+ /**
+ * Launch this message store.
+ * @throws Exception if there is any error.
+ */
void start() throws Exception;
+ /**
+ * Shutdown this message store.
+ */
void shutdown();
+ /**
+ * Destroy this message store. Generally, all persistent files should be removed after invocation.
+ */
void destroy();
+ /**
+ * Store a message into store.
+ * @param msg Message instance to store
+ * @return result of store operation.
+ */
PutMessageResult putMessage(final MessageExtBrokerInner msg);
+ /**
+ * Store a batch of messages.
+ * @param messageExtBatch Message batch.
+ * @return result of storing batch messages.
+ */
PutMessageResult putMessages(final MessageExtBatch messageExtBatch);
+ /**
+ * Query at most <code>maxMsgNums</code> messages belonging to <code>topic</code> at <code>queueId</code> starting
+ * from given <code>offset</code>. Resulting messages will further be screened using provided message filter.
+ *
+ * @param group Consumer group that launches this query.
+ * @param topic Topic to query.
+ * @param queueId Queue ID to query.
+ * @param offset Logical offset to start from.
+ * @param maxMsgNums Maximum count of messages to query.
+ * @param messageFilter Message filter used to screen desired messages.
+ * @return Matched messages.
+ */
GetMessageResult getMessage(final String group, final String topic, final int queueId,
final long offset, final int maxMsgNums, final MessageFilter messageFilter);
- long getMaxOffsetInQuque(final String topic, final int queueId);
-
- long getMinOffsetInQuque(final String topic, final int queueId);
-
- long getCommitLogOffsetInQueue(final String topic, final int queueId, final long cqOffset);
-
+ /**
+ * Get maximum offset of the topic queue.
+ * @param topic Topic name.
+ * @param queueId Queue ID.
+ * @return Maximum offset at present.
+ */
+ long getMaxOffsetInQueue(final String topic, final int queueId);
+
+ /**
+ * Get the minimum offset of the topic queue.
+ * @param topic Topic name.
+ * @param queueId Queue ID.
+ * @return Minimum offset at present.
+ */
+ long getMinOffsetInQueue(final String topic, final int queueId);
+
+ /**
+ * Get the offset of the message in the commit log, which is also known as physical offset.
+ * @param topic Topic of the message to lookup.
+ * @param queueId Queue ID.
+ * @param consumeQueueOffset offset of consume queue.
+ * @return physical offset.
+ */
+ long getCommitLogOffsetInQueue(final String topic, final int queueId, final long consumeQueueOffset);
+
+ /**
+ * Look up the physical offset of the message whose store timestamp is as specified.
+ * @param topic Topic of the message.
+ * @param queueId Queue ID.
+ * @param timestamp Timestamp to look up.
+ * @return physical offset which matches.
+ */
long getOffsetInQueueByTime(final String topic, final int queueId, final long timestamp);
+ /**
+ * Look up the message by given commit log offset.
+ * @param commitLogOffset physical offset.
+ * @return Message whose physical offset is as specified.
+ */
MessageExt lookMessageByOffset(final long commitLogOffset);
+ /**
+ * Get one message from the specified commit log offset.
+ * @param commitLogOffset commit log offset.
+ * @return wrapped result of the message.
+ */
SelectMappedBufferResult selectOneMessageByOffset(final long commitLogOffset);
+ /**
+ * Get one message from the specified commit log offset.
+ * @param commitLogOffset commit log offset.
+ * @param msgSize message size.
+ * @return wrapped result of the message.
+ */
SelectMappedBufferResult selectOneMessageByOffset(final long commitLogOffset, final int msgSize);
+ /**
+ * Get the running information of this store.
+ * @return message store running info.
+ */
String getRunningDataInfo();
+ /**
+ * Message store runtime information, which should generally contains various statistical information.
+ * @return runtime information of the message store in format of key-value pairs.
+ */
HashMap<String, String> getRuntimeInfo();
+ /**
+ * Get the maximum commit log offset.
+ * @return maximum commit log offset.
+ */
long getMaxPhyOffset();
+ /**
+ * Get the minimum commit log offset.
+ * @return minimum commit log offset.
+ */
long getMinPhyOffset();
+ /**
+ * Get the store time of the earliest message in the given queue.
+ * @param topic Topic of the messages to query.
+ * @param queueId Queue ID to find.
+ * @return store time of the earliest message.
+ */
long getEarliestMessageTime(final String topic, final int queueId);
+ /**
+ * Get the store time of the earliest message in this store.
+ * @return timestamp of the earliest message in this store.
+ */
long getEarliestMessageTime();
- long getMessageStoreTimeStamp(final String topic, final int queueId, final long offset);
-
+ /**
+ * Get the store time of the message specified.
+ * @param topic message topic.
+ * @param queueId queue ID.
+ * @param consumeQueueOffset consume queue offset.
+ * @return store timestamp of the message.
+ */
+ long getMessageStoreTimeStamp(final String topic, final int queueId, final long consumeQueueOffset);
+
+ /**
+ * Get the total number of the messages in the specified queue.
+ * @param topic Topic
+ * @param queueId Queue ID.
+ * @return total number.
+ */
long getMessageTotalInQueue(final String topic, final int queueId);
+ /**
+ * Get the raw commit log data starting from the given offset, which should used for replication purpose.
+ * @param offset starting offset.
+ * @return commit log data.
+ */
SelectMappedBufferResult getCommitLogData(final long offset);
+ /**
+ * Append data to commit log.
+ * @param startOffset starting offset.
+ * @param data data to append.
+ * @return true if success; false otherwise.
+ */
boolean appendToCommitLog(final long startOffset, final byte[] data);
- void excuteDeleteFilesManualy();
-
- QueryMessageResult queryMessage(final String topic, final String key, final int maxNum,
- final long begin, final long end);
-
+ /**
+ * Execute file deletion manually.
+ */
+ void executeDeleteFilesManually();
+
+ /**
+ * Query messages by given key.
+ * @param topic topic of the message.
+ * @param key message key.
+ * @param maxNum maximum number of the messages possible.
+ * @param begin begin timestamp.
+ * @param end end timestamp.
+ * @return
+ */
+ QueryMessageResult queryMessage(final String topic, final String key, final int maxNum, final long begin,
+ final long end);
+
+ /**
+ * Update HA master address.
+ * @param newAddr new address.
+ */
void updateHaMasterAddress(final String newAddr);
+ /**
+ * Return how much the slave falls behind.
+ * @return number of bytes that slave falls behind.
+ */
long slaveFallBehindMuch();
+ /**
+ * Return the current timestamp of the store.
+ * @return current time in milliseconds since 1970-01-01.
+ */
long now();
+ /**
+ * Clean unused topics.
+ * @param topics all valid topics.
+ * @return number of the topics deleted.
+ */
int cleanUnusedTopic(final Set<String> topics);
+ /**
+ * Clean expired consume queues.
+ */
void cleanExpiredConsumerQueue();
+ /**
+ * Check if the given message has been swapped out of the memory.
+ * @param topic topic.
+ * @param queueId queue ID.
+ * @param consumeOffset consume queue offset.
+ * @return true if the message is no longer in memory; false otherwise.
+ */
boolean checkInDiskByConsumeOffset(final String topic, final int queueId, long consumeOffset);
+ /**
+ * Get number of the bytes that have been stored in commit log and not yet dispatched to consume queue.
+ * @return number of the bytes to dispatch.
+ */
long dispatchBehindBytes();
+ /**
+ * Flush the message store to persist all data.
+ * @return maximum offset flushed to persistent storage device.
+ */
long flush();
+ /**
+ * Reset written offset.
+ * @param phyOffset new offset.
+ * @return true if success; false otherwise.
+ */
boolean resetWriteOffset(long phyOffset);
+ /**
+ * Get confirm offset.
+ * @return confirm offset.
+ */
long getConfirmOffset();
+ /**
+ * Set confirm offset.
+ * @param phyOffset confirm offset to set.
+ */
void setConfirmOffset(long phyOffset);
+ /**
+ * Check if the operation system page cache is busy or not.
+ * @return true if the OS page cache is busy; false otherwise.
+ */
boolean isOSPageCacheBusy();
+ /**
+ * Get lock time in milliseconds of the store by far.
+ * @return lock time in milliseconds.
+ */
long lockTimeMills();
+ /**
+ * Check if the transient store pool is deficient.
+ * @return true if the transient store pool is running out; false otherwise.
+ */
boolean isTransientStorePoolDeficient();
+ /**
+ * Get the dispatcher list.
+ * @return list of the dispatcher.
+ */
LinkedList<CommitLogDispatcher> getDispatcherList();
+ /**
+ * Get consume queue of the topic/queue.
+ * @param topic Topic.
+ * @param queueId Queue ID.
+ * @return Consume queue.
+ */
ConsumeQueue getConsumeQueue(String topic, int queueId);
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e9814ad4/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
index d45b994..501876e 100644
--- a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
@@ -79,7 +79,7 @@ public class ScheduleMessageService extends ConfigManager {
Entry<Integer, Long> next = it.next();
int queueId = delayLevel2QueueId(next.getKey());
long delayOffset = next.getValue();
- long maxOffset = this.defaultMessageStore.getMaxOffsetInQuque(SCHEDULE_TOPIC, queueId);
+ long maxOffset = this.defaultMessageStore.getMaxOffsetInQueue(SCHEDULE_TOPIC, queueId);
String value = String.format("%d,%d", delayOffset, maxOffset);
String key = String.format("%s_%d", RunningStats.scheduleMessageOffset.name(), next.getKey());
stats.put(key, value);
[18/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-187] Measure
the code coverage for Integration Tests, and add sonar-apache profile,
closes apache/incubator-rocketmq#96
Posted by do...@apache.org.
[ROCKETMQ-187] Measure the code coverage for Integration Tests, and add sonar-apache profile, closes apache/incubator-rocketmq#96
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/f5a2ee0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/f5a2ee0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/f5a2ee0a
Branch: refs/heads/release-4.1.0-incubating
Commit: f5a2ee0a8fecff48064f19ea242c233475e9635f
Parents: e9814ad
Author: dongeforever <zh...@yeah.net>
Authored: Tue May 9 23:38:56 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Tue May 9 23:38:56 2017 +0800
----------------------------------------------------------------------
.travis.yml | 2 +-
pom.xml | 13 ++++++++++---
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/f5a2ee0a/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 916cac5..2bc2296 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,4 +40,4 @@ script:
after_success:
- mvn clean install -Pit-test
- - mvn sonar:sonar
+ - mvn sonar:sonar -Psonar-apache
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/f5a2ee0a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6fd59ac..05ead63 100644
--- a/pom.xml
+++ b/pom.xml
@@ -161,10 +161,10 @@
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
- <!-- URL of the ASF SonarQube server -->
- <sonar.host.url>https://builds.apache.org/analysis</sonar.host.url>
<!-- Exclude all generated code -->
- <sonar.exclusions>file:**/generated-sources/**</sonar.exclusions>
+ <sonar.jacoco.itReportPath>${project.basedir}/../test/target/jacoco-it.exec</sonar.jacoco.itReportPath>
+ <sonar.exclusions>file:**/generated-sources/**,**/test/**</sonar.exclusions>
+
</properties>
<modules>
@@ -475,6 +475,13 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>sonar-apache</id>
+ <properties>
+ <!-- URL of the ASF SonarQube server -->
+ <sonar.host.url>https://builds.apache.org/analysis</sonar.host.url>
+ </properties>
+ </profile>
</profiles>
<dependencies>
[44/50] [abbrv] incubator-rocketmq git commit: Remove develops from
pom
Posted by do...@apache.org.
Remove develops from pom
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/e068ec17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/e068ec17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/e068ec17
Branch: refs/heads/release-4.1.0-incubating
Commit: e068ec17ad994b4fad68ebb239be6652e419d1f5
Parents: 96cd2e4
Author: dongeforever <zh...@yeah.net>
Authored: Tue Jun 6 16:07:41 2017 +0800
Committer: dongeforever <do...@apache.org>
Committed: Tue Jun 6 16:28:15 2017 +0800
----------------------------------------------------------------------
pom.xml | 71 +++++-------------------------------------------------------
1 file changed, 5 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/e068ec17/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 75dbf5b..851a6cb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -43,8 +43,8 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-rocketmq.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-rocketmq.git
</developerConnection>
- <tag>HEAD</tag>
- </scm>
+ <tag>HEAD</tag>
+ </scm>
<mailingLists>
<mailingList>
@@ -69,71 +69,10 @@
<developers>
<developer>
- <id>vintagewang</id>
- <name>vintagewang</name>
- <roles>
- <role>architect</role>
- <role>committer</role>
- </roles>
- <email>vintagewang@apache.org</email>
- <timezone>+8</timezone>
+ <id>Apache RocketMQ</id>
+ <name>Apache RocketMQ of ASF</name>
+ <url>https://rocketmq.apache.org/</url>
</developer>
- <developer>
- <id>vongosling@apache.org</id>
- <name>vongosling@apache.org</name>
- <roles>
- <role>architect</role>
- <role>committer</role>
- </roles>
- <email>vongosling@apache.org</email>
- <timezone>+8</timezone>
- </developer>
- <developer>
- <id>yukon</id>
- <name>Xinyu Zhou</name>
- <email>yukon@@apache.org</email>
- <roles>
- <role>committer</role>
- </roles>
- <timezone>+8</timezone>
- </developer>
- <developer>
- <id>stevenschew</id>
- <name>Wei Zhou</name>
- <email>stevenschew@@apache.org</email>
- <roles>
- <role>committer</role>
- </roles>
- <timezone>+8</timezone>
- </developer>
- <developer>
- <id>lollipop</id>
- <name>Jixiang Jin</name>
- <email>lollipop@apache.org</email>
- <roles>
- <role>committer</role>
- </roles>
- <timezone>+8</timezone>
- </developer>
- <developer>
- <id>lizhanhui</id>
- <name>Zhanhui Li</name>
- <email>lizhanhui@apache.org</email>
- <roles>
- <role>committer</role>
- </roles>
- <timezone>+8</timezone>
- </developer>
- <developer>
- <id>dongeforever</id>
- <name>dongeforever</name>
- <email>dongeforever@apache.org</email>
- <roles>
- <role>committer</role>
- </roles>
- <timezone>+8</timezone>
- </developer>
-
</developers>
<licenses>
[03/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-176] Use new
maven central badge with the newest release version info in README.
Posted by do...@apache.org.
[ROCKETMQ-176] Use new maven central badge with the newest release version info in README.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/deb08207
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/deb08207
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/deb08207
Branch: refs/heads/release-4.1.0-incubating
Commit: deb082071779563f7698090f9154e0e69200ac89
Parents: f508f13
Author: yukon <yu...@apache.org>
Authored: Thu Apr 20 11:36:46 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Thu Apr 20 11:36:46 2017 +0800
----------------------------------------------------------------------
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/deb08207/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 81eded4..7a9abb1 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
## Apache RocketMQ [](https://travis-ci.org/apache/incubator-rocketmq) [](https://coveralls.io/github/apache/incubator-rocketmq?branch=master)
-[](http://search.maven.org/#search%7Cga%7C1%7Corg.apache.rocketmq)
+[](http://search.maven.org/#search%7Cga%7C1%7Corg.apache.rocketmq)
[](https://github.org/apache/rocketmqreleases)
[](https://www.apache.org/licenses/LICENSE-2.0.html)
[43/50] [abbrv] incubator-rocketmq git commit:
[ROCKETMQ-208]incompatibility problem found in enviroment of JDK 1.7 when
running client closes apache/incubator-rocketmq#10
Posted by do...@apache.org.
[ROCKETMQ-208]incompatibility problem found in enviroment of JDK 1.7 when running client closes apache/incubator-rocketmq#10
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/96cd2e4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/96cd2e4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/96cd2e4e
Branch: refs/heads/release-4.1.0-incubating
Commit: 96cd2e4ed03e2c47a7d79d7e10980d154d2acb93
Parents: 2c28baa
Author: Jaskey <li...@gmail.com>
Authored: Tue Jun 6 16:06:46 2017 +0800
Committer: dongeforever <do...@apache.org>
Committed: Tue Jun 6 16:28:15 2017 +0800
----------------------------------------------------------------------
.../broker/client/ConsumerGroupInfo.java | 9 ++--
.../rocketmq/broker/client/ConsumerManager.java | 7 +--
.../broker/client/net/Broker2Client.java | 6 +--
.../client/rebalance/RebalanceLockManager.java | 3 +-
.../broker/filter/ConsumerFilterManager.java | 9 ++--
.../broker/filtersrv/FilterServerManager.java | 3 +-
.../longpolling/PullRequestHoldService.java | 3 +-
.../broker/offset/ConsumerOffsetManager.java | 31 +++++++------
.../broker/processor/AdminBrokerProcessor.java | 4 +-
.../subscription/SubscriptionGroupManager.java | 5 +-
.../broker/topic/TopicConfigManager.java | 5 +-
.../consumer/MQPullConsumerScheduleService.java | 7 +--
.../consumer/store/LocalFileOffsetStore.java | 3 +-
.../consumer/store/OffsetSerializeWrapper.java | 7 +--
.../consumer/store/RemoteBrokerOffsetStore.java | 3 +-
.../rocketmq/client/impl/MQClientManager.java | 3 +-
.../consumer/DefaultMQPullConsumerImpl.java | 4 +-
.../consumer/DefaultMQPushConsumerImpl.java | 6 +--
.../client/impl/consumer/MessageQueueLock.java | 3 +-
.../client/impl/consumer/PullAPIWrapper.java | 5 +-
.../client/impl/consumer/RebalanceImpl.java | 13 +++---
.../client/impl/factory/MQClientInstance.java | 17 +++----
.../impl/producer/DefaultMQProducerImpl.java | 5 +-
.../protocol/body/ConsumerConnection.java | 5 +-
.../body/ConsumerOffsetSerializeWrapper.java | 9 ++--
.../protocol/body/SubscriptionGroupWrapper.java | 7 +--
.../body/TopicConfigSerializeWrapper.java | 7 +--
.../common/stats/MomentStatsItemSet.java | 5 +-
.../rocketmq/common/stats/StatsItemSet.java | 3 +-
.../filtersrv/filter/FilterClassManager.java | 3 +-
.../namesrv/routeinfo/RouteInfoManager.java | 4 +-
.../remoting/netty/NettyRemotingAbstract.java | 3 +-
.../remoting/netty/NettyRemotingClient.java | 3 +-
.../store/AllocateMappedFileService.java | 3 +-
.../rocketmq/store/DefaultMessageStore.java | 49 ++++++++++----------
.../schedule/DelayOffsetSerializeWrapper.java | 7 +--
.../store/schedule/ScheduleMessageService.java | 5 +-
37 files changed, 153 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java
index 6ce542a..91b6c81 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
@@ -34,9 +35,9 @@ import org.slf4j.LoggerFactory;
public class ConsumerGroupInfo {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final String groupName;
- private final ConcurrentHashMap<String/* Topic */, SubscriptionData> subscriptionTable =
+ private final ConcurrentMap<String/* Topic */, SubscriptionData> subscriptionTable =
new ConcurrentHashMap<String, SubscriptionData>();
- private final ConcurrentHashMap<Channel, ClientChannelInfo> channelInfoTable =
+ private final ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =
new ConcurrentHashMap<Channel, ClientChannelInfo>(16);
private volatile ConsumeType consumeType;
private volatile MessageModel messageModel;
@@ -63,11 +64,11 @@ public class ConsumerGroupInfo {
return null;
}
- public ConcurrentHashMap<String, SubscriptionData> getSubscriptionTable() {
+ public ConcurrentMap<String, SubscriptionData> getSubscriptionTable() {
return subscriptionTable;
}
- public ConcurrentHashMap<Channel, ClientChannelInfo> getChannelInfoTable() {
+ public ConcurrentMap<Channel, ClientChannelInfo> getChannelInfoTable() {
return channelInfoTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
index a5ddec8..4a262e5 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
@@ -22,6 +22,7 @@ import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
@@ -35,7 +36,7 @@ import org.slf4j.LoggerFactory;
public class ConsumerManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120;
- private final ConcurrentHashMap<String/* Group */, ConsumerGroupInfo> consumerTable =
+ private final ConcurrentMap<String/* Group */, ConsumerGroupInfo> consumerTable =
new ConcurrentHashMap<String, ConsumerGroupInfo>(1024);
private final ConsumerIdsChangeListener consumerIdsChangeListener;
@@ -145,7 +146,7 @@ public class ConsumerManager {
Entry<String, ConsumerGroupInfo> next = it.next();
String group = next.getKey();
ConsumerGroupInfo consumerGroupInfo = next.getValue();
- ConcurrentHashMap<Channel, ClientChannelInfo> channelInfoTable =
+ ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =
consumerGroupInfo.getChannelInfoTable();
Iterator<Entry<Channel, ClientChannelInfo>> itChannel = channelInfoTable.entrySet().iterator();
@@ -176,7 +177,7 @@ public class ConsumerManager {
Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
while (it.hasNext()) {
Entry<String, ConsumerGroupInfo> entry = it.next();
- ConcurrentHashMap<String, SubscriptionData> subscriptionTable =
+ ConcurrentMap<String, SubscriptionData> subscriptionTable =
entry.getValue().getSubscriptionTable();
if (subscriptionTable.containsKey(topic)) {
groups.add(entry.getKey());
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
index 863da62..65b444e 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
@@ -25,7 +25,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
@@ -189,7 +189,7 @@ public class Broker2Client {
this.brokerController.getConsumerManager().getConsumerGroupInfo(group);
if (consumerGroupInfo != null && !consumerGroupInfo.getAllChannel().isEmpty()) {
- ConcurrentHashMap<Channel, ClientChannelInfo> channelInfoTable =
+ ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =
consumerGroupInfo.getChannelInfoTable();
for (Map.Entry<Channel, ClientChannelInfo> entry : channelInfoTable.entrySet()) {
int version = entry.getValue().getVersion();
@@ -252,7 +252,7 @@ public class Broker2Client {
Map<String, Map<MessageQueue, Long>> consumerStatusTable =
new HashMap<String, Map<MessageQueue, Long>>();
- ConcurrentHashMap<Channel, ClientChannelInfo> channelInfoTable =
+ ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =
this.brokerController.getConsumerManager().getConsumerGroupInfo(group).getChannelInfoTable();
if (null == channelInfoTable || channelInfoTable.isEmpty()) {
result.setCode(ResponseCode.SYSTEM_ERROR);
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java
index 98aceb6..ed5a875 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java
@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.client.rebalance;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.rocketmq.common.constant.LoggerName;
@@ -31,7 +32,7 @@ public class RebalanceLockManager {
private final static long REBALANCE_LOCK_MAX_LIVE_TIME = Long.parseLong(System.getProperty(
"rocketmq.broker.rebalance.lockMaxLiveTime", "60000"));
private final Lock lock = new ReentrantLock();
- private final ConcurrentHashMap<String/* group */, ConcurrentHashMap<MessageQueue, LockEntry>> mqLockTable =
+ private final ConcurrentMap<String/* group */, ConcurrentHashMap<MessageQueue, LockEntry>> mqLockTable =
new ConcurrentHashMap<String, ConcurrentHashMap<MessageQueue, LockEntry>>(1024);
public boolean tryLock(final String group, final MessageQueue mq, final String clientId) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java
index 7f790af..f50db86 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.broker.filter;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.BrokerPathConfigHelper;
import org.apache.rocketmq.common.ConfigManager;
@@ -45,7 +46,7 @@ public class ConsumerFilterManager extends ConfigManager {
private static final long MS_24_HOUR = 24 * 3600 * 1000;
- private ConcurrentHashMap<String/*Topic*/, FilterDataMapByTopic>
+ private ConcurrentMap<String/*Topic*/, FilterDataMapByTopic>
filterDataByTopic = new ConcurrentHashMap<String/*consumer group*/, FilterDataMapByTopic>(256);
private transient BrokerController brokerController;
@@ -316,7 +317,7 @@ public class ConsumerFilterManager extends ConfigManager {
}
}
- public ConcurrentHashMap<String, FilterDataMapByTopic> getFilterDataByTopic() {
+ public ConcurrentMap<String, FilterDataMapByTopic> getFilterDataByTopic() {
return filterDataByTopic;
}
@@ -326,7 +327,7 @@ public class ConsumerFilterManager extends ConfigManager {
public static class FilterDataMapByTopic {
- private ConcurrentHashMap<String/*consumer group*/, ConsumerFilterData>
+ private ConcurrentMap<String/*consumer group*/, ConsumerFilterData>
groupFilterData = new ConcurrentHashMap<String, ConsumerFilterData>();
private String topic;
@@ -452,7 +453,7 @@ public class ConsumerFilterManager extends ConfigManager {
return this.groupFilterData.get(consumerGroup);
}
- public final ConcurrentHashMap<String, ConsumerFilterData> getGroupFilterData() {
+ public final ConcurrentMap<String, ConsumerFilterData> getGroupFilterData() {
return this.groupFilterData;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java
index b935bc8..52cb919 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java
@@ -23,6 +23,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -38,7 +39,7 @@ public class FilterServerManager {
public static final long FILTER_SERVER_MAX_IDLE_TIME_MILLS = 30000;
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
- private final ConcurrentHashMap<Channel, FilterServerInfo> filterServerTable =
+ private final ConcurrentMap<Channel, FilterServerInfo> filterServerTable =
new ConcurrentHashMap<Channel, FilterServerInfo>(16);
private final BrokerController brokerController;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
index 71f56a4..b1bd86f 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
@@ -20,6 +20,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.common.SystemClock;
@@ -33,7 +34,7 @@ public class PullRequestHoldService extends ServiceThread {
private static final String TOPIC_QUEUEID_SEPARATOR = "@";
private final BrokerController brokerController;
private final SystemClock systemClock = new SystemClock();
- private ConcurrentHashMap<String/* topic@queueId */, ManyPullRequest> pullRequestTable =
+ private ConcurrentMap<String/* topic@queueId */, ManyPullRequest> pullRequestTable =
new ConcurrentHashMap<String, ManyPullRequest>(1024);
public PullRequestHoldService(final BrokerController brokerController) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
index 769c4ad..57565a6 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.BrokerPathConfigHelper;
import org.apache.rocketmq.common.ConfigManager;
@@ -36,8 +37,8 @@ public class ConsumerOffsetManager extends ConfigManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final String TOPIC_GROUP_SEPARATOR = "@";
- private ConcurrentHashMap<String/* topic@group */, ConcurrentHashMap<Integer, Long>> offsetTable =
- new ConcurrentHashMap<String, ConcurrentHashMap<Integer, Long>>(512);
+ private ConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> offsetTable =
+ new ConcurrentHashMap<String, ConcurrentMap<Integer, Long>>(512);
private transient BrokerController brokerController;
@@ -49,9 +50,9 @@ public class ConsumerOffsetManager extends ConfigManager {
}
public void scanUnsubscribedTopic() {
- Iterator<Entry<String, ConcurrentHashMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
+ Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
- Entry<String, ConcurrentHashMap<Integer, Long>> next = it.next();
+ Entry<String, ConcurrentMap<Integer, Long>> next = it.next();
String topicAtGroup = next.getKey();
String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
if (arrays.length == 2) {
@@ -67,7 +68,7 @@ public class ConsumerOffsetManager extends ConfigManager {
}
}
- private boolean offsetBehindMuchThanData(final String topic, ConcurrentHashMap<Integer, Long> table) {
+ private boolean offsetBehindMuchThanData(final String topic, ConcurrentMap<Integer, Long> table) {
Iterator<Entry<Integer, Long>> it = table.entrySet().iterator();
boolean result = !table.isEmpty();
@@ -84,9 +85,9 @@ public class ConsumerOffsetManager extends ConfigManager {
public Set<String> whichTopicByConsumer(final String group) {
Set<String> topics = new HashSet<String>();
- Iterator<Entry<String, ConcurrentHashMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
+ Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
- Entry<String, ConcurrentHashMap<Integer, Long>> next = it.next();
+ Entry<String, ConcurrentMap<Integer, Long>> next = it.next();
String topicAtGroup = next.getKey();
String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
if (arrays.length == 2) {
@@ -102,9 +103,9 @@ public class ConsumerOffsetManager extends ConfigManager {
public Set<String> whichGroupByTopic(final String topic) {
Set<String> groups = new HashSet<String>();
- Iterator<Entry<String, ConcurrentHashMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
+ Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
- Entry<String, ConcurrentHashMap<Integer, Long>> next = it.next();
+ Entry<String, ConcurrentMap<Integer, Long>> next = it.next();
String topicAtGroup = next.getKey();
String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
if (arrays.length == 2) {
@@ -124,7 +125,7 @@ public class ConsumerOffsetManager extends ConfigManager {
}
private void commitOffset(final String clientHost, final String key, final int queueId, final long offset) {
- ConcurrentHashMap<Integer, Long> map = this.offsetTable.get(key);
+ ConcurrentMap<Integer, Long> map = this.offsetTable.get(key);
if (null == map) {
map = new ConcurrentHashMap<Integer, Long>(32);
map.put(queueId, offset);
@@ -140,7 +141,7 @@ public class ConsumerOffsetManager extends ConfigManager {
public long queryOffset(final String group, final String topic, final int queueId) {
// topic@group
String key = topic + TOPIC_GROUP_SEPARATOR + group;
- ConcurrentHashMap<Integer, Long> map = this.offsetTable.get(key);
+ ConcurrentMap<Integer, Long> map = this.offsetTable.get(key);
if (null != map) {
Long offset = map.get(queueId);
if (offset != null)
@@ -173,11 +174,11 @@ public class ConsumerOffsetManager extends ConfigManager {
return RemotingSerializable.toJson(this, prettyFormat);
}
- public ConcurrentHashMap<String, ConcurrentHashMap<Integer, Long>> getOffsetTable() {
+ public ConcurrentMap<String, ConcurrentMap<Integer, Long>> getOffsetTable() {
return offsetTable;
}
- public void setOffsetTable(ConcurrentHashMap<String, ConcurrentHashMap<Integer, Long>> offsetTable) {
+ public void setOffsetTable(ConcurrentHashMap<String, ConcurrentMap<Integer, Long>> offsetTable) {
this.offsetTable = offsetTable;
}
@@ -196,7 +197,7 @@ public class ConsumerOffsetManager extends ConfigManager {
}
}
- for (Map.Entry<String, ConcurrentHashMap<Integer, Long>> offSetEntry : this.offsetTable.entrySet()) {
+ for (Map.Entry<String, ConcurrentMap<Integer, Long>> offSetEntry : this.offsetTable.entrySet()) {
String topicGroup = offSetEntry.getKey();
String[] topicGroupArr = topicGroup.split(TOPIC_GROUP_SEPARATOR);
if (topic.equals(topicGroupArr[0])) {
@@ -224,7 +225,7 @@ public class ConsumerOffsetManager extends ConfigManager {
}
public void cloneOffset(final String srcGroup, final String destGroup, final String topic) {
- ConcurrentHashMap<Integer, Long> offsets = this.offsetTable.get(topic + TOPIC_GROUP_SEPARATOR + srcGroup);
+ ConcurrentMap<Integer, Long> offsets = this.offsetTable.get(topic + TOPIC_GROUP_SEPARATOR + srcGroup);
if (offsets != null) {
this.offsetTable.put(topic + TOPIC_GROUP_SEPARATOR + destGroup, new ConcurrentHashMap<Integer, Long>(offsets));
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
index f59d295..71fdda9 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
@@ -29,7 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
@@ -1084,7 +1084,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
GetConsumeStatsInBrokerHeader requestHeader =
(GetConsumeStatsInBrokerHeader) request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);
boolean isOrder = requestHeader.isOrder();
- ConcurrentHashMap<String, SubscriptionGroupConfig> subscriptionGroups =
+ ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroups =
brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable();
List<Map<String/* subscriptionGroupName */, List<ConsumeStats>>> brokerConsumeStatsList =
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
index bdf2a01..bd4a26e 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.subscription;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.BrokerPathConfigHelper;
import org.apache.rocketmq.common.ConfigManager;
@@ -33,7 +34,7 @@ import org.slf4j.LoggerFactory;
public class SubscriptionGroupManager extends ConfigManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
- private final ConcurrentHashMap<String, SubscriptionGroupConfig> subscriptionGroupTable =
+ private final ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable =
new ConcurrentHashMap<String, SubscriptionGroupConfig>(1024);
private final DataVersion dataVersion = new DataVersion();
private transient BrokerController brokerController;
@@ -169,7 +170,7 @@ public class SubscriptionGroupManager extends ConfigManager {
}
}
- public ConcurrentHashMap<String, SubscriptionGroupConfig> getSubscriptionGroupTable() {
+ public ConcurrentMap<String, SubscriptionGroupConfig> getSubscriptionGroupTable() {
return subscriptionGroupTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
index 93a631a..3bcafc0 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
@@ -22,6 +22,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -44,7 +45,7 @@ public class TopicConfigManager extends ConfigManager {
private static final long LOCK_TIMEOUT_MILLIS = 3000;
private transient final Lock lockTopicConfigTable = new ReentrantLock();
- private final ConcurrentHashMap<String, TopicConfig> topicConfigTable =
+ private final ConcurrentMap<String, TopicConfig> topicConfigTable =
new ConcurrentHashMap<String, TopicConfig>(1024);
private final DataVersion dataVersion = new DataVersion();
private final Set<String> systemTopicList = new HashSet<String>();
@@ -416,7 +417,7 @@ public class TopicConfigManager extends ConfigManager {
return dataVersion;
}
- public ConcurrentHashMap<String, TopicConfig> getTopicConfigTable() {
+ public ConcurrentMap<String, TopicConfig> getTopicConfigTable() {
return topicConfigTable;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java
index 6bae85a..e0b546d 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java
@@ -20,6 +20,7 @@ import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.client.exception.MQClientException;
@@ -35,11 +36,11 @@ import org.slf4j.Logger;
public class MQPullConsumerScheduleService {
private final Logger log = ClientLogger.getLog();
private final MessageQueueListener messageQueueListener = new MessageQueueListenerImpl();
- private final ConcurrentHashMap<MessageQueue, PullTaskImpl> taskTable =
+ private final ConcurrentMap<MessageQueue, PullTaskImpl> taskTable =
new ConcurrentHashMap<MessageQueue, PullTaskImpl>();
private DefaultMQPullConsumer defaultMQPullConsumer;
private int pullThreadNums = 20;
- private ConcurrentHashMap<String /* topic */, PullTaskCallback> callbackTable =
+ private ConcurrentMap<String /* topic */, PullTaskCallback> callbackTable =
new ConcurrentHashMap<String, PullTaskCallback>();
private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor;
@@ -100,7 +101,7 @@ public class MQPullConsumerScheduleService {
}
}
- public ConcurrentHashMap<String, PullTaskCallback> getCallbackTable() {
+ public ConcurrentMap<String, PullTaskCallback> getCallbackTable() {
return callbackTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java
index 6c81516..d4b19b2 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
@@ -45,7 +46,7 @@ public class LocalFileOffsetStore implements OffsetStore {
private final MQClientInstance mQClientFactory;
private final String groupName;
private final String storePath;
- private ConcurrentHashMap<MessageQueue, AtomicLong> offsetTable =
+ private ConcurrentMap<MessageQueue, AtomicLong> offsetTable =
new ConcurrentHashMap<MessageQueue, AtomicLong>();
public LocalFileOffsetStore(MQClientInstance mQClientFactory, String groupName) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/consumer/store/OffsetSerializeWrapper.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/OffsetSerializeWrapper.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/OffsetSerializeWrapper.java
index 32bcc9f..7dfd97a 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/OffsetSerializeWrapper.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/OffsetSerializeWrapper.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.client.consumer.store;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
@@ -25,14 +26,14 @@ import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
* Wrapper class for offset serialization
*/
public class OffsetSerializeWrapper extends RemotingSerializable {
- private ConcurrentHashMap<MessageQueue, AtomicLong> offsetTable =
+ private ConcurrentMap<MessageQueue, AtomicLong> offsetTable =
new ConcurrentHashMap<MessageQueue, AtomicLong>();
- public ConcurrentHashMap<MessageQueue, AtomicLong> getOffsetTable() {
+ public ConcurrentMap<MessageQueue, AtomicLong> getOffsetTable() {
return offsetTable;
}
- public void setOffsetTable(ConcurrentHashMap<MessageQueue, AtomicLong> offsetTable) {
+ public void setOffsetTable(ConcurrentMap<MessageQueue, AtomicLong> offsetTable) {
this.offsetTable = offsetTable;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java
index 60ad101..5bd5749 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java
@@ -21,6 +21,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
@@ -42,7 +43,7 @@ public class RemoteBrokerOffsetStore implements OffsetStore {
private final static Logger log = ClientLogger.getLog();
private final MQClientInstance mQClientFactory;
private final String groupName;
- private ConcurrentHashMap<MessageQueue, AtomicLong> offsetTable =
+ private ConcurrentMap<MessageQueue, AtomicLong> offsetTable =
new ConcurrentHashMap<MessageQueue, AtomicLong>();
public RemoteBrokerOffsetStore(MQClientInstance mQClientFactory, String groupName) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java
index f596b83..25877d7 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.client.impl;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
@@ -28,7 +29,7 @@ public class MQClientManager {
private final static Logger log = ClientLogger.getLog();
private static MQClientManager instance = new MQClientManager();
private AtomicInteger factoryIndexGenerator = new AtomicInteger();
- private ConcurrentHashMap<String/* clientId */, MQClientInstance> factoryTable =
+ private ConcurrentMap<String/* clientId */, MQClientInstance> factoryTable =
new ConcurrentHashMap<String, MQClientInstance>();
private MQClientManager() {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
index 7d43b37..35ee16f 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.client.QueryResult;
import org.apache.rocketmq.client.Validators;
import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
@@ -115,7 +115,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
throw new IllegalArgumentException("topic is null");
}
- ConcurrentHashMap<MessageQueue, ProcessQueue> mqTable = this.rebalanceImpl.getProcessQueueTable();
+ ConcurrentMap<MessageQueue, ProcessQueue> mqTable = this.rebalanceImpl.getProcessQueueTable();
Set<MessageQueue> mqResult = new HashSet<MessageQueue>();
for (MessageQueue mq : mqTable.keySet()) {
if (mq.getTopic().equals(topic)) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
index 8767964..9bf34be 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
@@ -26,7 +26,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.client.QueryResult;
import org.apache.rocketmq.client.Validators;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
@@ -805,7 +805,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
}
}
- public ConcurrentHashMap<String, SubscriptionData> getSubscriptionInner() {
+ public ConcurrentMap<String, SubscriptionData> getSubscriptionInner() {
return this.rebalanceImpl.getSubscriptionInner();
}
@@ -1060,7 +1060,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
private long computeAccumulationTotal() {
long msgAccTotal = 0;
- ConcurrentHashMap<MessageQueue, ProcessQueue> processQueueTable = this.rebalanceImpl.getProcessQueueTable();
+ ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = this.rebalanceImpl.getProcessQueueTable();
Iterator<Entry<MessageQueue, ProcessQueue>> it = processQueueTable.entrySet().iterator();
while (it.hasNext()) {
Entry<MessageQueue, ProcessQueue> next = it.next();
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MessageQueueLock.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MessageQueueLock.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MessageQueueLock.java
index c25e41b..a02f1b6 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MessageQueueLock.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MessageQueueLock.java
@@ -17,13 +17,14 @@
package org.apache.rocketmq.client.impl.consumer;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.message.MessageQueue;
/**
* Message lock,strictly ensure the single queue only one thread at a time consuming
*/
public class MessageQueueLock {
- private ConcurrentHashMap<MessageQueue, Object> mqLockTable =
+ private ConcurrentMap<MessageQueue, Object> mqLockTable =
new ConcurrentHashMap<MessageQueue, Object>();
public Object fetchLockObject(final MessageQueue mq) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java
index 304a44a..bbdf27d 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.rocketmq.client.consumer.PullCallback;
import org.apache.rocketmq.client.consumer.PullResult;
@@ -53,7 +54,7 @@ public class PullAPIWrapper {
private final MQClientInstance mQClientFactory;
private final String consumerGroup;
private final boolean unitMode;
- private ConcurrentHashMap<MessageQueue, AtomicLong/* brokerId */> pullFromWhichNodeTable =
+ private ConcurrentMap<MessageQueue, AtomicLong/* brokerId */> pullFromWhichNodeTable =
new ConcurrentHashMap<MessageQueue, AtomicLong>(32);
private volatile boolean connectBrokerByUser = false;
private volatile long defaultBrokerId = MixAll.MASTER_ID;
@@ -247,7 +248,7 @@ public class PullAPIWrapper {
private String computPullFromWhichFilterServer(final String topic, final String brokerAddr)
throws MQClientException {
- ConcurrentHashMap<String, TopicRouteData> topicRouteTable = this.mQClientFactory.getTopicRouteTable();
+ ConcurrentMap<String, TopicRouteData> topicRouteTable = this.mQClientFactory.getTopicRouteTable();
if (topicRouteTable != null) {
TopicRouteData topicRouteData = topicRouteTable.get(topic);
List<String> list = topicRouteData.getFilterServerTable().get(brokerAddr);
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
index 6b12221..634e0f0 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
import org.apache.rocketmq.client.impl.FindBrokerResult;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
@@ -44,10 +45,10 @@ import org.slf4j.Logger;
*/
public abstract class RebalanceImpl {
protected static final Logger log = ClientLogger.getLog();
- protected final ConcurrentHashMap<MessageQueue, ProcessQueue> processQueueTable = new ConcurrentHashMap<MessageQueue, ProcessQueue>(64);
- protected final ConcurrentHashMap<String/* topic */, Set<MessageQueue>> topicSubscribeInfoTable =
+ protected final ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = new ConcurrentHashMap<MessageQueue, ProcessQueue>(64);
+ protected final ConcurrentMap<String/* topic */, Set<MessageQueue>> topicSubscribeInfoTable =
new ConcurrentHashMap<String, Set<MessageQueue>>();
- protected final ConcurrentHashMap<String /* topic */, SubscriptionData> subscriptionInner =
+ protected final ConcurrentMap<String /* topic */, SubscriptionData> subscriptionInner =
new ConcurrentHashMap<String, SubscriptionData>();
protected String consumerGroup;
protected MessageModel messageModel;
@@ -232,7 +233,7 @@ public abstract class RebalanceImpl {
this.truncateMessageQueueNotMyTopic();
}
- public ConcurrentHashMap<String, SubscriptionData> getSubscriptionInner() {
+ public ConcurrentMap<String, SubscriptionData> getSubscriptionInner() {
return subscriptionInner;
}
@@ -421,11 +422,11 @@ public abstract class RebalanceImpl {
}
}
- public ConcurrentHashMap<MessageQueue, ProcessQueue> getProcessQueueTable() {
+ public ConcurrentMap<MessageQueue, ProcessQueue> getProcessQueueTable() {
return processQueueTable;
}
- public ConcurrentHashMap<String, Set<MessageQueue>> getTopicSubscribeInfoTable() {
+ public ConcurrentMap<String, Set<MessageQueue>> getTopicSubscribeInfoTable() {
return topicSubscribeInfoTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
index 1b075ee..f146be9 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
@@ -28,6 +28,7 @@ import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
@@ -88,18 +89,18 @@ public class MQClientInstance {
private final int instanceIndex;
private final String clientId;
private final long bootTimestamp = System.currentTimeMillis();
- private final ConcurrentHashMap<String/* group */, MQProducerInner> producerTable = new ConcurrentHashMap<String, MQProducerInner>();
- private final ConcurrentHashMap<String/* group */, MQConsumerInner> consumerTable = new ConcurrentHashMap<String, MQConsumerInner>();
- private final ConcurrentHashMap<String/* group */, MQAdminExtInner> adminExtTable = new ConcurrentHashMap<String, MQAdminExtInner>();
+ private final ConcurrentMap<String/* group */, MQProducerInner> producerTable = new ConcurrentHashMap<String, MQProducerInner>();
+ private final ConcurrentMap<String/* group */, MQConsumerInner> consumerTable = new ConcurrentHashMap<String, MQConsumerInner>();
+ private final ConcurrentMap<String/* group */, MQAdminExtInner> adminExtTable = new ConcurrentHashMap<String, MQAdminExtInner>();
private final NettyClientConfig nettyClientConfig;
private final MQClientAPIImpl mQClientAPIImpl;
private final MQAdminImpl mQAdminImpl;
- private final ConcurrentHashMap<String/* Topic */, TopicRouteData> topicRouteTable = new ConcurrentHashMap<String, TopicRouteData>();
+ private final ConcurrentMap<String/* Topic */, TopicRouteData> topicRouteTable = new ConcurrentHashMap<String, TopicRouteData>();
private final Lock lockNamesrv = new ReentrantLock();
private final Lock lockHeartbeat = new ReentrantLock();
- private final ConcurrentHashMap<String/* Broker Name */, HashMap<Long/* brokerId */, String/* address */>> brokerAddrTable =
+ private final ConcurrentMap<String/* Broker Name */, HashMap<Long/* brokerId */, String/* address */>> brokerAddrTable =
new ConcurrentHashMap<String, HashMap<Long, String>>();
- private final ConcurrentHashMap<String/* Broker Name */, HashMap<String/* address */, Integer>> brokerVersionTable =
+ private final ConcurrentMap<String/* Broker Name */, HashMap<String/* address */, Integer>> brokerVersionTable =
new ConcurrentHashMap<String, HashMap<String, Integer>>();
private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
@Override
@@ -1088,7 +1089,7 @@ public class MQClientInstance {
}
consumer.suspend();
- ConcurrentHashMap<MessageQueue, ProcessQueue> processQueueTable = consumer.getRebalanceImpl().getProcessQueueTable();
+ ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = consumer.getRebalanceImpl().getProcessQueueTable();
for (Map.Entry<MessageQueue, ProcessQueue> entry : processQueueTable.entrySet()) {
MessageQueue mq = entry.getKey();
if (topic.equals(mq.getTopic()) && offsetTable.containsKey(mq)) {
@@ -1166,7 +1167,7 @@ public class MQClientInstance {
return defaultMQProducer;
}
- public ConcurrentHashMap<String, TopicRouteData> getTopicRouteTable() {
+ public ConcurrentMap<String, TopicRouteData> getTopicRouteTable() {
return topicRouteTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
index d828875..12f8a36 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
@@ -26,6 +26,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -84,7 +85,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
private final Logger log = ClientLogger.getLog();
private final Random random = new Random();
private final DefaultMQProducer defaultMQProducer;
- private final ConcurrentHashMap<String/* topic */, TopicPublishInfo> topicPublishInfoTable =
+ private final ConcurrentMap<String/* topic */, TopicPublishInfo> topicPublishInfoTable =
new ConcurrentHashMap<String, TopicPublishInfo>();
private final ArrayList<SendMessageHook> sendMessageHookList = new ArrayList<SendMessageHook>();
private final RPCHook rpcHook;
@@ -1057,7 +1058,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout);
}
- public ConcurrentHashMap<String, TopicPublishInfo> getTopicPublishInfoTable() {
+ public ConcurrentMap<String, TopicPublishInfo> getTopicPublishInfoTable() {
return topicPublishInfoTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerConnection.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerConnection.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerConnection.java
index 7478dd2..3a0356c 100644
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerConnection.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerConnection.java
@@ -19,6 +19,7 @@ package org.apache.rocketmq.common.protocol.body;
import java.util.HashSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
@@ -27,7 +28,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class ConsumerConnection extends RemotingSerializable {
private HashSet<Connection> connectionSet = new HashSet<Connection>();
- private ConcurrentHashMap<String/* Topic */, SubscriptionData> subscriptionTable =
+ private ConcurrentMap<String/* Topic */, SubscriptionData> subscriptionTable =
new ConcurrentHashMap<String, SubscriptionData>();
private ConsumeType consumeType;
private MessageModel messageModel;
@@ -52,7 +53,7 @@ public class ConsumerConnection extends RemotingSerializable {
this.connectionSet = connectionSet;
}
- public ConcurrentHashMap<String, SubscriptionData> getSubscriptionTable() {
+ public ConcurrentMap<String, SubscriptionData> getSubscriptionTable() {
return subscriptionTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java
index 02bf811..5b08d78 100644
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java
@@ -18,17 +18,18 @@
package org.apache.rocketmq.common.protocol.body;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class ConsumerOffsetSerializeWrapper extends RemotingSerializable {
- private ConcurrentHashMap<String/* topic@group */, ConcurrentHashMap<Integer, Long>> offsetTable =
- new ConcurrentHashMap<String, ConcurrentHashMap<Integer, Long>>(512);
+ private ConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> offsetTable =
+ new ConcurrentHashMap<String, ConcurrentMap<Integer, Long>>(512);
- public ConcurrentHashMap<String, ConcurrentHashMap<Integer, Long>> getOffsetTable() {
+ public ConcurrentMap<String, ConcurrentMap<Integer, Long>> getOffsetTable() {
return offsetTable;
}
- public void setOffsetTable(ConcurrentHashMap<String, ConcurrentHashMap<Integer, Long>> offsetTable) {
+ public void setOffsetTable(ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable) {
this.offsetTable = offsetTable;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/common/src/main/java/org/apache/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java
index 92c15eb..e05f759 100644
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java
@@ -18,21 +18,22 @@
package org.apache.rocketmq.common.protocol.body;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class SubscriptionGroupWrapper extends RemotingSerializable {
- private ConcurrentHashMap<String, SubscriptionGroupConfig> subscriptionGroupTable =
+ private ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable =
new ConcurrentHashMap<String, SubscriptionGroupConfig>(1024);
private DataVersion dataVersion = new DataVersion();
- public ConcurrentHashMap<String, SubscriptionGroupConfig> getSubscriptionGroupTable() {
+ public ConcurrentMap<String, SubscriptionGroupConfig> getSubscriptionGroupTable() {
return subscriptionGroupTable;
}
public void setSubscriptionGroupTable(
- ConcurrentHashMap<String, SubscriptionGroupConfig> subscriptionGroupTable) {
+ ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable) {
this.subscriptionGroupTable = subscriptionGroupTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java
index c471d1a..ce12302 100644
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java
@@ -18,20 +18,21 @@
package org.apache.rocketmq.common.protocol.body;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class TopicConfigSerializeWrapper extends RemotingSerializable {
- private ConcurrentHashMap<String, TopicConfig> topicConfigTable =
+ private ConcurrentMap<String, TopicConfig> topicConfigTable =
new ConcurrentHashMap<String, TopicConfig>();
private DataVersion dataVersion = new DataVersion();
- public ConcurrentHashMap<String, TopicConfig> getTopicConfigTable() {
+ public ConcurrentMap<String, TopicConfig> getTopicConfigTable() {
return topicConfigTable;
}
- public void setTopicConfigTable(ConcurrentHashMap<String, TopicConfig> topicConfigTable) {
+ public void setTopicConfigTable(ConcurrentMap<String, TopicConfig> topicConfigTable) {
this.topicConfigTable = topicConfigTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java
index 5498d34..57dfc38 100644
--- a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java
+++ b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java
@@ -20,13 +20,14 @@ package org.apache.rocketmq.common.stats;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.common.UtilAll;
import org.slf4j.Logger;
public class MomentStatsItemSet {
- private final ConcurrentHashMap<String/* key */, MomentStatsItem> statsItemTable =
+ private final ConcurrentMap<String/* key */, MomentStatsItem> statsItemTable =
new ConcurrentHashMap<String, MomentStatsItem>(128);
private final String statsName;
private final ScheduledExecutorService scheduledExecutorService;
@@ -39,7 +40,7 @@ public class MomentStatsItemSet {
this.init();
}
- public ConcurrentHashMap<String, MomentStatsItem> getStatsItemTable() {
+ public ConcurrentMap<String, MomentStatsItem> getStatsItemTable() {
return statsItemTable;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java
index 8633d68..17dbf0d 100644
--- a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java
+++ b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java
@@ -20,13 +20,14 @@ package org.apache.rocketmq.common.stats;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.common.UtilAll;
import org.slf4j.Logger;
public class StatsItemSet {
- private final ConcurrentHashMap<String/* key */, StatsItem> statsItemTable =
+ private final ConcurrentMap<String/* key */, StatsItem> statsItemTable =
new ConcurrentHashMap<String, StatsItem>(128);
private final String statsName;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java
----------------------------------------------------------------------
diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java
index 2c31538..490c582 100644
--- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java
+++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/filter/FilterClassManager.java
@@ -20,6 +20,7 @@ package org.apache.rocketmq.filtersrv.filter;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -40,7 +41,7 @@ public class FilterClassManager {
private final ScheduledExecutorService scheduledExecutorService = Executors
.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSGetClassScheduledThread"));
- private ConcurrentHashMap<String/* topic@consumerGroup */, FilterClassInfo> filterClassTable =
+ private ConcurrentMap<String/* topic@consumerGroup */, FilterClassInfo> filterClassTable =
new ConcurrentHashMap<String, FilterClassInfo>(128);
private FilterClassFetchMethod filterClassFetchMethod;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
----------------------------------------------------------------------
diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
index 5a953a9..7479fcc 100644
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
@@ -25,7 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.rocketmq.common.DataVersion;
@@ -135,7 +135,7 @@ public class RouteInfoManager {
&& MixAll.MASTER_ID == brokerId) {
if (this.isBrokerTopicConfigChanged(brokerAddr, topicConfigWrapper.getDataVersion())//
|| registerFirst) {
- ConcurrentHashMap<String, TopicConfig> tcTable =
+ ConcurrentMap<String, TopicConfig> tcTable =
topicConfigWrapper.getTopicConfigTable();
if (tcTable != null) {
for (Map.Entry<String, TopicConfig> entry : tcTable.entrySet()) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
----------------------------------------------------------------------
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
index 15586cb..0ba714a 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
@@ -27,6 +27,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
@@ -67,7 +68,7 @@ public abstract class NettyRemotingAbstract {
/**
* This map caches all on-going requests.
*/
- protected final ConcurrentHashMap<Integer /* opaque */, ResponseFuture> responseTable =
+ protected final ConcurrentMap<Integer /* opaque */, ResponseFuture> responseTable =
new ConcurrentHashMap<Integer, ResponseFuture>(256);
/**
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
----------------------------------------------------------------------
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
index 52ca47e..1c3da9a 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
@@ -41,6 +41,7 @@ import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
@@ -73,7 +74,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
private final Bootstrap bootstrap = new Bootstrap();
private final EventLoopGroup eventLoopGroupWorker;
private final Lock lockChannelTables = new ReentrantLock();
- private final ConcurrentHashMap<String /* addr */, ChannelWrapper> channelTables = new ConcurrentHashMap<String, ChannelWrapper>();
+ private final ConcurrentMap<String /* addr */, ChannelWrapper> channelTables = new ConcurrentHashMap<String, ChannelWrapper>();
private final Timer timer = new Timer("ClientHouseKeepingService", true);
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
index 0993a5f..abb8385 100644
--- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
@@ -20,6 +20,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ServiceLoader;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -36,7 +37,7 @@ import org.slf4j.LoggerFactory;
public class AllocateMappedFileService extends ServiceThread {
private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
private static int waitTimeOut = 1000 * 5;
- private ConcurrentHashMap<String, AllocateRequest> requestTable =
+ private ConcurrentMap<String, AllocateRequest> requestTable =
new ConcurrentHashMap<String, AllocateRequest>();
private PriorityBlockingQueue<AllocateRequest> requestQueue =
new PriorityBlockingQueue<AllocateRequest>();
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
index 931edc7..4549f1e 100644
--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -64,7 +65,7 @@ public class DefaultMessageStore implements MessageStore {
// CommitLog
private final CommitLog commitLog;
- private final ConcurrentHashMap<String/* topic */, ConcurrentHashMap<Integer/* queueId */, ConsumeQueue>> consumeQueueTable;
+ private final ConcurrentMap<String/* topic */, ConcurrentMap<Integer/* queueId */, ConsumeQueue>> consumeQueueTable;
private final FlushConsumeQueueService flushConsumeQueueService;
@@ -140,9 +141,9 @@ public class DefaultMessageStore implements MessageStore {
}
public void truncateDirtyLogicFiles(long phyOffset) {
- ConcurrentHashMap<String, ConcurrentHashMap<Integer, ConsumeQueue>> tables = DefaultMessageStore.this.consumeQueueTable;
+ ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueue>> tables = DefaultMessageStore.this.consumeQueueTable;
- for (ConcurrentHashMap<Integer, ConsumeQueue> maps : tables.values()) {
+ for (ConcurrentMap<Integer, ConsumeQueue> maps : tables.values()) {
for (ConsumeQueue logic : maps.values()) {
logic.truncateDirtyLogicFiles(phyOffset);
}
@@ -267,7 +268,7 @@ public class DefaultMessageStore implements MessageStore {
}
public void destroyLogics() {
- for (ConcurrentHashMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) {
+ for (ConcurrentMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) {
for (ConsumeQueue logic : maps.values()) {
logic.destroy();
}
@@ -885,13 +886,13 @@ public class DefaultMessageStore implements MessageStore {
@Override
public int cleanUnusedTopic(Set<String> topics) {
- Iterator<Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
+ Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
while (it.hasNext()) {
- Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>> next = it.next();
+ Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
String topic = next.getKey();
if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
- ConcurrentHashMap<Integer, ConsumeQueue> queueTable = next.getValue();
+ ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
for (ConsumeQueue cq : queueTable.values()) {
cq.destroy();
log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned", //
@@ -913,12 +914,12 @@ public class DefaultMessageStore implements MessageStore {
public void cleanExpiredConsumerQueue() {
long minCommitLogOffset = this.commitLog.getMinOffset();
- Iterator<Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
+ Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
while (it.hasNext()) {
- Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>> next = it.next();
+ Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
String topic = next.getKey();
if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
- ConcurrentHashMap<Integer, ConsumeQueue> queueTable = next.getValue();
+ ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
Iterator<Entry<Integer, ConsumeQueue>> itQT = queueTable.entrySet().iterator();
while (itQT.hasNext()) {
Entry<Integer, ConsumeQueue> nextQT = itQT.next();
@@ -1061,10 +1062,10 @@ public class DefaultMessageStore implements MessageStore {
}
public ConsumeQueue findConsumeQueue(String topic, int queueId) {
- ConcurrentHashMap<Integer, ConsumeQueue> map = consumeQueueTable.get(topic);
+ ConcurrentMap<Integer, ConsumeQueue> map = consumeQueueTable.get(topic);
if (null == map) {
- ConcurrentHashMap<Integer, ConsumeQueue> newMap = new ConcurrentHashMap<Integer, ConsumeQueue>(128);
- ConcurrentHashMap<Integer, ConsumeQueue> oldMap = consumeQueueTable.putIfAbsent(topic, newMap);
+ ConcurrentMap<Integer, ConsumeQueue> newMap = new ConcurrentHashMap<Integer, ConsumeQueue>(128);
+ ConcurrentMap<Integer, ConsumeQueue> oldMap = consumeQueueTable.putIfAbsent(topic, newMap);
if (oldMap != null) {
map = oldMap;
} else {
@@ -1205,9 +1206,9 @@ public class DefaultMessageStore implements MessageStore {
private void checkSelf() {
this.commitLog.checkSelf();
- Iterator<Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
+ Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
while (it.hasNext()) {
- Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>> next = it.next();
+ Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
Iterator<Entry<Integer, ConsumeQueue>> itNext = next.getValue().entrySet().iterator();
while (itNext.hasNext()) {
Entry<Integer, ConsumeQueue> cq = itNext.next();
@@ -1280,7 +1281,7 @@ public class DefaultMessageStore implements MessageStore {
}
private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueue consumeQueue) {
- ConcurrentHashMap<Integer/* queueId */, ConsumeQueue> map = this.consumeQueueTable.get(topic);
+ ConcurrentMap<Integer/* queueId */, ConsumeQueue> map = this.consumeQueueTable.get(topic);
if (null == map) {
map = new ConcurrentHashMap<Integer/* queueId */, ConsumeQueue>();
map.put(queueId, consumeQueue);
@@ -1291,7 +1292,7 @@ public class DefaultMessageStore implements MessageStore {
}
private void recoverConsumeQueue() {
- for (ConcurrentHashMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) {
+ for (ConcurrentMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) {
for (ConsumeQueue logic : maps.values()) {
logic.recover();
}
@@ -1301,7 +1302,7 @@ public class DefaultMessageStore implements MessageStore {
private void recoverTopicQueueTable() {
HashMap<String/* topic-queueid */, Long/* offset */> table = new HashMap<String, Long>(1024);
long minPhyOffset = this.commitLog.getMinOffset();
- for (ConcurrentHashMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) {
+ for (ConcurrentMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) {
for (ConsumeQueue logic : maps.values()) {
String key = logic.getTopic() + "-" + logic.getQueueId();
table.put(key, logic.getMaxOffsetInQueue());
@@ -1324,7 +1325,7 @@ public class DefaultMessageStore implements MessageStore {
return runningFlags;
}
- public ConcurrentHashMap<String, ConcurrentHashMap<Integer, ConsumeQueue>> getConsumeQueueTable() {
+ public ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueue>> getConsumeQueueTable() {
return consumeQueueTable;
}
@@ -1375,7 +1376,7 @@ public class DefaultMessageStore implements MessageStore {
@Override
public ConsumeQueue getConsumeQueue(String topic, int queueId) {
- ConcurrentHashMap<Integer, ConsumeQueue> map = consumeQueueTable.get(topic);
+ ConcurrentMap<Integer, ConsumeQueue> map = consumeQueueTable.get(topic);
if (map == null) {
return null;
}
@@ -1594,9 +1595,9 @@ public class DefaultMessageStore implements MessageStore {
if (minOffset > this.lastPhysicalMinOffset) {
this.lastPhysicalMinOffset = minOffset;
- ConcurrentHashMap<String, ConcurrentHashMap<Integer, ConsumeQueue>> tables = DefaultMessageStore.this.consumeQueueTable;
+ ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueue>> tables = DefaultMessageStore.this.consumeQueueTable;
- for (ConcurrentHashMap<Integer, ConsumeQueue> maps : tables.values()) {
+ for (ConcurrentMap<Integer, ConsumeQueue> maps : tables.values()) {
for (ConsumeQueue logic : maps.values()) {
int deleteCount = logic.deleteExpiredFile(minOffset);
@@ -1639,9 +1640,9 @@ public class DefaultMessageStore implements MessageStore {
logicsMsgTimestamp = DefaultMessageStore.this.getStoreCheckpoint().getLogicsMsgTimestamp();
}
- ConcurrentHashMap<String, ConcurrentHashMap<Integer, ConsumeQueue>> tables = DefaultMessageStore.this.consumeQueueTable;
+ ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueue>> tables = DefaultMessageStore.this.consumeQueueTable;
- for (ConcurrentHashMap<Integer, ConsumeQueue> maps : tables.values()) {
+ for (ConcurrentMap<Integer, ConsumeQueue> maps : tables.values()) {
for (ConsumeQueue cq : maps.values()) {
boolean result = false;
for (int i = 0; i < retryTimes && !result; i++) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/store/src/main/java/org/apache/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java b/store/src/main/java/org/apache/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java
index efb6aa8..7021992 100644
--- a/store/src/main/java/org/apache/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java
+++ b/store/src/main/java/org/apache/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java
@@ -17,17 +17,18 @@
package org.apache.rocketmq.store.schedule;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class DelayOffsetSerializeWrapper extends RemotingSerializable {
- private ConcurrentHashMap<Integer /* level */, Long/* offset */> offsetTable =
+ private ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable =
new ConcurrentHashMap<Integer, Long>(32);
- public ConcurrentHashMap<Integer, Long> getOffsetTable() {
+ public ConcurrentMap<Integer, Long> getOffsetTable() {
return offsetTable;
}
- public void setOffsetTable(ConcurrentHashMap<Integer, Long> offsetTable) {
+ public void setOffsetTable(ConcurrentMap<Integer, Long> offsetTable) {
this.offsetTable = offsetTable;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/96cd2e4e/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
----------------------------------------------------------------------
diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
index 501876e..172954d 100644
--- a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java
@@ -23,6 +23,7 @@ import java.util.Map.Entry;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.ConfigManager;
import org.apache.rocketmq.common.TopicFilterType;
import org.apache.rocketmq.common.constant.LoggerName;
@@ -49,10 +50,10 @@ public class ScheduleMessageService extends ConfigManager {
private static final long DELAY_FOR_A_WHILE = 100L;
private static final long DELAY_FOR_A_PERIOD = 10000L;
- private final ConcurrentHashMap<Integer /* level */, Long/* delay timeMillis */> delayLevelTable =
+ private final ConcurrentMap<Integer /* level */, Long/* delay timeMillis */> delayLevelTable =
new ConcurrentHashMap<Integer, Long>(32);
- private final ConcurrentHashMap<Integer /* level */, Long/* offset */> offsetTable =
+ private final ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable =
new ConcurrentHashMap<Integer, Long>(32);
private final Timer timer = new Timer("ScheduleMessageTimerThread", true);
[37/50] [abbrv] incubator-rocketmq git commit:
[ROCKETMQ-160]SendHeartBeat may not be logged in the expected period closes
apache/incubator-rocketmq#86
Posted by do...@apache.org.
[ROCKETMQ-160]SendHeartBeat may not be logged in the expected period closes apache/incubator-rocketmq#86
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/04c8925d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/04c8925d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/04c8925d
Branch: refs/heads/release-4.1.0-incubating
Commit: 04c8925d6da77a41cef746a9c6478a407c4c9edd
Parents: adae162
Author: Jaskey <li...@gmail.com>
Authored: Sat May 27 12:38:00 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 12:38:00 2017 +0800
----------------------------------------------------------------------
.../client/impl/factory/MQClientInstance.java | 66 ++++++++++----------
1 file changed, 34 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/04c8925d/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
index a8c65b2..1b075ee 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
@@ -112,7 +112,7 @@ public class MQClientInstance {
private final RebalanceService rebalanceService;
private final DefaultMQProducer defaultMQProducer;
private final ConsumerStatsManager consumerStatsManager;
- private final AtomicLong storeTimesTotal = new AtomicLong(0);
+ private final AtomicLong sendHeartbeatTimesTotal = new AtomicLong(0);
private ServiceState serviceState = ServiceState.CREATE_JUST;
private DatagramSocket datagramSocket;
private Random random = new Random();
@@ -517,38 +517,40 @@ public class MQClientInstance {
return;
}
- long times = this.storeTimesTotal.getAndIncrement();
- Iterator<Entry<String, HashMap<Long, String>>> it = this.brokerAddrTable.entrySet().iterator();
- while (it.hasNext()) {
- Entry<String, HashMap<Long, String>> entry = it.next();
- String brokerName = entry.getKey();
- HashMap<Long, String> oneTable = entry.getValue();
- if (oneTable != null) {
- for (Map.Entry<Long, String> entry1 : oneTable.entrySet()) {
- Long id = entry1.getKey();
- String addr = entry1.getValue();
- if (addr != null) {
- if (consumerEmpty) {
- if (id != MixAll.MASTER_ID)
- continue;
- }
-
- try {
- int version = this.mQClientAPIImpl.sendHearbeat(addr, heartbeatData, 3000);
- if (!this.brokerVersionTable.containsKey(brokerName)) {
- this.brokerVersionTable.put(brokerName, new HashMap<String, Integer>(4));
- }
- this.brokerVersionTable.get(brokerName).put(addr, version);
- if (times % 20 == 0) {
- log.info("send heart beat to broker[{} {} {}] success", brokerName, id, addr);
- log.info(heartbeatData.toString());
+ if (!this.brokerAddrTable.isEmpty()) {
+ long times = this.sendHeartbeatTimesTotal.getAndIncrement();
+ Iterator<Entry<String, HashMap<Long, String>>> it = this.brokerAddrTable.entrySet().iterator();
+ while (it.hasNext()) {
+ Entry<String, HashMap<Long, String>> entry = it.next();
+ String brokerName = entry.getKey();
+ HashMap<Long, String> oneTable = entry.getValue();
+ if (oneTable != null) {
+ for (Map.Entry<Long, String> entry1 : oneTable.entrySet()) {
+ Long id = entry1.getKey();
+ String addr = entry1.getValue();
+ if (addr != null) {
+ if (consumerEmpty) {
+ if (id != MixAll.MASTER_ID)
+ continue;
}
- } catch (Exception e) {
- if (this.isBrokerInNameServer(addr)) {
- log.error("send heart beat to broker exception", e);
- } else {
- log.info("send heart beat to broker[{} {} {}] exception, because the broker not up, forget it", brokerName,
- id, addr);
+
+ try {
+ int version = this.mQClientAPIImpl.sendHearbeat(addr, heartbeatData, 3000);
+ if (!this.brokerVersionTable.containsKey(brokerName)) {
+ this.brokerVersionTable.put(brokerName, new HashMap<String, Integer>(4));
+ }
+ this.brokerVersionTable.get(brokerName).put(addr, version);
+ if (times % 20 == 0) {
+ log.info("send heart beat to broker[{} {} {}] success", brokerName, id, addr);
+ log.info(heartbeatData.toString());
+ }
+ } catch (Exception e) {
+ if (this.isBrokerInNameServer(addr)) {
+ log.info("send heart beat to broker[{} {} {}] failed", brokerName, id, addr);
+ } else {
+ log.info("send heart beat to broker[{} {} {}] exception, because the broker not up, forget it", brokerName,
+ id, addr);
+ }
}
}
}
[07/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-121]Support
message filtering based on SQL92 closes apache/incubator-rocketmq#82
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserConstants.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserConstants.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserConstants.java
new file mode 100644
index 0000000..915658c
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserConstants.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. SelectorParserConstants.java */
+package org.apache.rocketmq.filter.parser;
+
+/**
+ * Token literal values and constants.
+ * Generated by org.javacc.parser.OtherFilesGen#start()
+ */
+public interface SelectorParserConstants {
+
+ /**
+ * End of File.
+ */
+ int EOF = 0;
+ /**
+ * RegularExpression Id.
+ */
+ int LINE_COMMENT = 6;
+ /**
+ * RegularExpression Id.
+ */
+ int BLOCK_COMMENT = 7;
+ /**
+ * RegularExpression Id.
+ */
+ int NOT = 8;
+ /**
+ * RegularExpression Id.
+ */
+ int AND = 9;
+ /**
+ * RegularExpression Id.
+ */
+ int OR = 10;
+ /**
+ * RegularExpression Id.
+ */
+ int BETWEEN = 11;
+ /**
+ * RegularExpression Id.
+ */
+ int IN = 12;
+ /**
+ * RegularExpression Id.
+ */
+ int TRUE = 13;
+ /**
+ * RegularExpression Id.
+ */
+ int FALSE = 14;
+ /**
+ * RegularExpression Id.
+ */
+ int NULL = 15;
+ /**
+ * RegularExpression Id.
+ */
+ int IS = 16;
+ /**
+ * RegularExpression Id.
+ */
+ int DECIMAL_LITERAL = 17;
+ /**
+ * RegularExpression Id.
+ */
+ int FLOATING_POINT_LITERAL = 18;
+ /**
+ * RegularExpression Id.
+ */
+ int EXPONENT = 19;
+ /**
+ * RegularExpression Id.
+ */
+ int STRING_LITERAL = 20;
+ /**
+ * RegularExpression Id.
+ */
+ int ID = 21;
+
+ /**
+ * Lexical state.
+ */
+ int DEFAULT = 0;
+
+ /**
+ * Literal token values.
+ */
+ String[] TOKEN_IMAGE = {
+ "<EOF>",
+ "\" \"",
+ "\"\\t\"",
+ "\"\\n\"",
+ "\"\\r\"",
+ "\"\\f\"",
+ "<LINE_COMMENT>",
+ "<BLOCK_COMMENT>",
+ "\"NOT\"",
+ "\"AND\"",
+ "\"OR\"",
+ "\"BETWEEN\"",
+ "\"IN\"",
+ "\"TRUE\"",
+ "\"FALSE\"",
+ "\"NULL\"",
+ "\"IS\"",
+ "<DECIMAL_LITERAL>",
+ "<FLOATING_POINT_LITERAL>",
+ "<EXPONENT>",
+ "<STRING_LITERAL>",
+ "<ID>",
+ "\"=\"",
+ "\"<>\"",
+ "\">\"",
+ "\">=\"",
+ "\"<\"",
+ "\"<=\"",
+ "\"(\"",
+ "\",\"",
+ "\")\"",
+ "\"+\"",
+ "\"-\"",
+ };
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserTokenManager.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserTokenManager.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserTokenManager.java
new file mode 100644
index 0000000..354f5ba
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserTokenManager.java
@@ -0,0 +1,919 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. SelectorParserTokenManager.java */
+package org.apache.rocketmq.filter.parser;
+
+/**
+ * Token Manager.
+ */
+public class SelectorParserTokenManager implements SelectorParserConstants {
+
+ /**
+ * Debug output.
+ */
+ public java.io.PrintStream debugStream = System.out;
+
+ /**
+ * Set debug output.
+ */
+ public void setDebugStream(java.io.PrintStream ds) {
+ debugStream = ds;
+ }
+
+ private int jjStopAtPos(int pos, int kind) {
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ return pos + 1;
+ }
+
+ private int jjMoveStringLiteralDfa0_0() {
+ switch (curChar) {
+ case 9:
+ jjmatchedKind = 2;
+ return jjMoveNfa_0(5, 0);
+ case 10:
+ jjmatchedKind = 3;
+ return jjMoveNfa_0(5, 0);
+ case 12:
+ jjmatchedKind = 5;
+ return jjMoveNfa_0(5, 0);
+ case 13:
+ jjmatchedKind = 4;
+ return jjMoveNfa_0(5, 0);
+ case 32:
+ jjmatchedKind = 1;
+ return jjMoveNfa_0(5, 0);
+ case 40:
+ jjmatchedKind = 28;
+ return jjMoveNfa_0(5, 0);
+ case 41:
+ jjmatchedKind = 30;
+ return jjMoveNfa_0(5, 0);
+ case 43:
+ jjmatchedKind = 31;
+ return jjMoveNfa_0(5, 0);
+ case 44:
+ jjmatchedKind = 29;
+ return jjMoveNfa_0(5, 0);
+ case 45:
+ jjmatchedKind = 32;
+ return jjMoveNfa_0(5, 0);
+ case 60:
+ jjmatchedKind = 26;
+ return jjMoveStringLiteralDfa1_0(0x8800000L);
+ case 61:
+ jjmatchedKind = 22;
+ return jjMoveNfa_0(5, 0);
+ case 62:
+ jjmatchedKind = 24;
+ return jjMoveStringLiteralDfa1_0(0x2000000L);
+ case 65:
+ return jjMoveStringLiteralDfa1_0(0x200L);
+ case 66:
+ return jjMoveStringLiteralDfa1_0(0x800L);
+ case 70:
+ return jjMoveStringLiteralDfa1_0(0x4000L);
+ case 73:
+ return jjMoveStringLiteralDfa1_0(0x11000L);
+ case 78:
+ return jjMoveStringLiteralDfa1_0(0x8100L);
+ case 79:
+ return jjMoveStringLiteralDfa1_0(0x400L);
+ case 84:
+ return jjMoveStringLiteralDfa1_0(0x2000L);
+ case 97:
+ return jjMoveStringLiteralDfa1_0(0x200L);
+ case 98:
+ return jjMoveStringLiteralDfa1_0(0x800L);
+ case 102:
+ return jjMoveStringLiteralDfa1_0(0x4000L);
+ case 105:
+ return jjMoveStringLiteralDfa1_0(0x11000L);
+ case 110:
+ return jjMoveStringLiteralDfa1_0(0x8100L);
+ case 111:
+ return jjMoveStringLiteralDfa1_0(0x400L);
+ case 116:
+ return jjMoveStringLiteralDfa1_0(0x2000L);
+ default:
+ return jjMoveNfa_0(5, 0);
+ }
+ }
+
+ private int jjMoveStringLiteralDfa1_0(long active0) {
+ try {
+ curChar = inputStream.readChar();
+ } catch (java.io.IOException e) {
+ return jjMoveNfa_0(5, 0);
+ }
+ switch (curChar) {
+ case 61:
+ if ((active0 & 0x2000000L) != 0L) {
+ jjmatchedKind = 25;
+ jjmatchedPos = 1;
+ } else if ((active0 & 0x8000000L) != 0L) {
+ jjmatchedKind = 27;
+ jjmatchedPos = 1;
+ }
+ break;
+ case 62:
+ if ((active0 & 0x800000L) != 0L) {
+ jjmatchedKind = 23;
+ jjmatchedPos = 1;
+ }
+ break;
+ case 65:
+ return jjMoveStringLiteralDfa2_0(active0, 0x4000L);
+ case 69:
+ return jjMoveStringLiteralDfa2_0(active0, 0x800L);
+ case 78:
+ if ((active0 & 0x1000L) != 0L) {
+ jjmatchedKind = 12;
+ jjmatchedPos = 1;
+ }
+ return jjMoveStringLiteralDfa2_0(active0, 0x200L);
+ case 79:
+ return jjMoveStringLiteralDfa2_0(active0, 0x100L);
+ case 82:
+ if ((active0 & 0x400L) != 0L) {
+ jjmatchedKind = 10;
+ jjmatchedPos = 1;
+ }
+ return jjMoveStringLiteralDfa2_0(active0, 0x2000L);
+ case 83:
+ if ((active0 & 0x10000L) != 0L) {
+ jjmatchedKind = 16;
+ jjmatchedPos = 1;
+ }
+ break;
+ case 85:
+ return jjMoveStringLiteralDfa2_0(active0, 0x8000L);
+ case 97:
+ return jjMoveStringLiteralDfa2_0(active0, 0x4000L);
+ case 101:
+ return jjMoveStringLiteralDfa2_0(active0, 0x800L);
+ case 110:
+ if ((active0 & 0x1000L) != 0L) {
+ jjmatchedKind = 12;
+ jjmatchedPos = 1;
+ }
+ return jjMoveStringLiteralDfa2_0(active0, 0x200L);
+ case 111:
+ return jjMoveStringLiteralDfa2_0(active0, 0x100L);
+ case 114:
+ if ((active0 & 0x400L) != 0L) {
+ jjmatchedKind = 10;
+ jjmatchedPos = 1;
+ }
+ return jjMoveStringLiteralDfa2_0(active0, 0x2000L);
+ case 115:
+ if ((active0 & 0x10000L) != 0L) {
+ jjmatchedKind = 16;
+ jjmatchedPos = 1;
+ }
+ break;
+ case 117:
+ return jjMoveStringLiteralDfa2_0(active0, 0x8000L);
+ default:
+ break;
+ }
+ return jjMoveNfa_0(5, 1);
+ }
+
+ private int jjMoveStringLiteralDfa2_0(long old0, long active0) {
+ if (((active0 &= old0)) == 0L)
+ return jjMoveNfa_0(5, 1);
+ try {
+ curChar = inputStream.readChar();
+ } catch (java.io.IOException e) {
+ return jjMoveNfa_0(5, 1);
+ }
+ switch (curChar) {
+ case 68:
+ if ((active0 & 0x200L) != 0L) {
+ jjmatchedKind = 9;
+ jjmatchedPos = 2;
+ }
+ break;
+ case 76:
+ return jjMoveStringLiteralDfa3_0(active0, 0xc000L);
+ case 84:
+ if ((active0 & 0x100L) != 0L) {
+ jjmatchedKind = 8;
+ jjmatchedPos = 2;
+ }
+ return jjMoveStringLiteralDfa3_0(active0, 0x800L);
+ case 85:
+ return jjMoveStringLiteralDfa3_0(active0, 0x2000L);
+ case 100:
+ if ((active0 & 0x200L) != 0L) {
+ jjmatchedKind = 9;
+ jjmatchedPos = 2;
+ }
+ break;
+ case 108:
+ return jjMoveStringLiteralDfa3_0(active0, 0xc000L);
+ case 116:
+ if ((active0 & 0x100L) != 0L) {
+ jjmatchedKind = 8;
+ jjmatchedPos = 2;
+ }
+ return jjMoveStringLiteralDfa3_0(active0, 0x800L);
+ case 117:
+ return jjMoveStringLiteralDfa3_0(active0, 0x2000L);
+ default:
+ break;
+ }
+ return jjMoveNfa_0(5, 2);
+ }
+
+ private int jjMoveStringLiteralDfa3_0(long old0, long active0) {
+ if (((active0 &= old0)) == 0L)
+ return jjMoveNfa_0(5, 2);
+ try {
+ curChar = inputStream.readChar();
+ } catch (java.io.IOException e) {
+ return jjMoveNfa_0(5, 2);
+ }
+ switch (curChar) {
+ case 69:
+ if ((active0 & 0x2000L) != 0L) {
+ jjmatchedKind = 13;
+ jjmatchedPos = 3;
+ }
+ break;
+ case 76:
+ if ((active0 & 0x8000L) != 0L) {
+ jjmatchedKind = 15;
+ jjmatchedPos = 3;
+ }
+ break;
+ case 83:
+ return jjMoveStringLiteralDfa4_0(active0, 0x4000L);
+ case 87:
+ return jjMoveStringLiteralDfa4_0(active0, 0x800L);
+ case 101:
+ if ((active0 & 0x2000L) != 0L) {
+ jjmatchedKind = 13;
+ jjmatchedPos = 3;
+ }
+ break;
+ case 108:
+ if ((active0 & 0x8000L) != 0L) {
+ jjmatchedKind = 15;
+ jjmatchedPos = 3;
+ }
+ break;
+ case 115:
+ return jjMoveStringLiteralDfa4_0(active0, 0x4000L);
+ case 119:
+ return jjMoveStringLiteralDfa4_0(active0, 0x800L);
+ default:
+ break;
+ }
+ return jjMoveNfa_0(5, 3);
+ }
+
+ private int jjMoveStringLiteralDfa4_0(long old0, long active0) {
+ if (((active0 &= old0)) == 0L)
+ return jjMoveNfa_0(5, 3);
+ try {
+ curChar = inputStream.readChar();
+ } catch (java.io.IOException e) {
+ return jjMoveNfa_0(5, 3);
+ }
+ switch (curChar) {
+ case 69:
+ if ((active0 & 0x4000L) != 0L) {
+ jjmatchedKind = 14;
+ jjmatchedPos = 4;
+ }
+ return jjMoveStringLiteralDfa5_0(active0, 0x800L);
+ case 101:
+ if ((active0 & 0x4000L) != 0L) {
+ jjmatchedKind = 14;
+ jjmatchedPos = 4;
+ }
+ return jjMoveStringLiteralDfa5_0(active0, 0x800L);
+ default:
+ break;
+ }
+ return jjMoveNfa_0(5, 4);
+ }
+
+ private int jjMoveStringLiteralDfa5_0(long old0, long active0) {
+ if (((active0 &= old0)) == 0L)
+ return jjMoveNfa_0(5, 4);
+ try {
+ curChar = inputStream.readChar();
+ } catch (java.io.IOException e) {
+ return jjMoveNfa_0(5, 4);
+ }
+ switch (curChar) {
+ case 69:
+ return jjMoveStringLiteralDfa6_0(active0, 0x800L);
+ case 101:
+ return jjMoveStringLiteralDfa6_0(active0, 0x800L);
+ default:
+ break;
+ }
+ return jjMoveNfa_0(5, 5);
+ }
+
+ private int jjMoveStringLiteralDfa6_0(long old0, long active0) {
+ if (((active0 &= old0)) == 0L)
+ return jjMoveNfa_0(5, 5);
+ try {
+ curChar = inputStream.readChar();
+ } catch (java.io.IOException e) {
+ return jjMoveNfa_0(5, 5);
+ }
+ switch (curChar) {
+ case 78:
+ if ((active0 & 0x800L) != 0L) {
+ jjmatchedKind = 11;
+ jjmatchedPos = 6;
+ }
+ break;
+ case 110:
+ if ((active0 & 0x800L) != 0L) {
+ jjmatchedKind = 11;
+ jjmatchedPos = 6;
+ }
+ break;
+ default:
+ break;
+ }
+ return jjMoveNfa_0(5, 6);
+ }
+
+ static final long[] JJ_BIT_VEC_0 = {
+ 0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
+ };
+ static final long[] JJ_BIT_VEC_2 = {
+ 0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
+ };
+
+ private int jjMoveNfa_0(int startState, int curPos) {
+ int strKind = jjmatchedKind;
+ int strPos = jjmatchedPos;
+ int seenUpto;
+ inputStream.backup(seenUpto = curPos + 1);
+ try {
+ curChar = inputStream.readChar();
+ } catch (java.io.IOException e) {
+ throw new Error("Internal Error");
+ }
+ curPos = 0;
+ int startsAt = 0;
+ jjnewStateCnt = 40;
+ int i = 1;
+ jjstateSet[0] = startState;
+ int kind = 0x7fffffff;
+ for (;;) {
+ if (++jjround == 0x7fffffff)
+ ReInitRounds();
+ if (curChar < 64) {
+ long l = 1L << curChar;
+ do {
+ switch (jjstateSet[--i]) {
+ case 5:
+ if ((0x3ff000000000000L & l) != 0L)
+ jjCheckNAddStates(0, 3);
+ else if (curChar == 36) {
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAdd(28);
+ } else if (curChar == 39)
+ jjCheckNAddStates(4, 6);
+ else if (curChar == 46)
+ jjCheckNAdd(18);
+ else if (curChar == 47)
+ jjstateSet[jjnewStateCnt++] = 6;
+ else if (curChar == 45)
+ jjstateSet[jjnewStateCnt++] = 0;
+ if ((0x3fe000000000000L & l) != 0L) {
+ if (kind > 17)
+ kind = 17;
+ jjCheckNAddTwoStates(15, 16);
+ } else if (curChar == 48) {
+ if (kind > 17)
+ kind = 17;
+ }
+ break;
+ case 0:
+ if (curChar == 45)
+ jjCheckNAddStates(7, 9);
+ break;
+ case 1:
+ if ((0xffffffffffffdbffL & l) != 0L)
+ jjCheckNAddStates(7, 9);
+ break;
+ case 2:
+ if ((0x2400L & l) != 0L && kind > 6)
+ kind = 6;
+ break;
+ case 3:
+ if (curChar == 10 && kind > 6)
+ kind = 6;
+ break;
+ case 4:
+ if (curChar == 13)
+ jjstateSet[jjnewStateCnt++] = 3;
+ break;
+ case 6:
+ if (curChar == 42)
+ jjCheckNAddTwoStates(7, 8);
+ break;
+ case 7:
+ if ((0xfffffbffffffffffL & l) != 0L)
+ jjCheckNAddTwoStates(7, 8);
+ break;
+ case 8:
+ if (curChar == 42)
+ jjCheckNAddStates(10, 12);
+ break;
+ case 9:
+ if ((0xffff7bffffffffffL & l) != 0L)
+ jjCheckNAddTwoStates(10, 8);
+ break;
+ case 10:
+ if ((0xfffffbffffffffffL & l) != 0L)
+ jjCheckNAddTwoStates(10, 8);
+ break;
+ case 11:
+ if (curChar == 47 && kind > 7)
+ kind = 7;
+ break;
+ case 12:
+ if (curChar == 47)
+ jjstateSet[jjnewStateCnt++] = 6;
+ break;
+ case 13:
+ if (curChar == 48 && kind > 17)
+ kind = 17;
+ break;
+ case 14:
+ if ((0x3fe000000000000L & l) == 0L)
+ break;
+ if (kind > 17)
+ kind = 17;
+ jjCheckNAddTwoStates(15, 16);
+ break;
+ case 15:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 17)
+ kind = 17;
+ jjCheckNAddTwoStates(15, 16);
+ break;
+ case 17:
+ if (curChar == 46)
+ jjCheckNAdd(18);
+ break;
+ case 18:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 18)
+ kind = 18;
+ jjCheckNAddTwoStates(18, 19);
+ break;
+ case 20:
+ if ((0x280000000000L & l) != 0L)
+ jjCheckNAdd(21);
+ break;
+ case 21:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 18)
+ kind = 18;
+ jjCheckNAdd(21);
+ break;
+ case 22:
+ case 23:
+ if (curChar == 39)
+ jjCheckNAddStates(4, 6);
+ break;
+ case 24:
+ if (curChar == 39)
+ jjstateSet[jjnewStateCnt++] = 23;
+ break;
+ case 25:
+ if ((0xffffff7fffffffffL & l) != 0L)
+ jjCheckNAddStates(4, 6);
+ break;
+ case 26:
+ if (curChar == 39 && kind > 20)
+ kind = 20;
+ break;
+ case 27:
+ if (curChar != 36)
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAdd(28);
+ break;
+ case 28:
+ if ((0x3ff001000000000L & l) == 0L)
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAdd(28);
+ break;
+ case 29:
+ if ((0x3ff000000000000L & l) != 0L)
+ jjCheckNAddStates(0, 3);
+ break;
+ case 30:
+ if ((0x3ff000000000000L & l) != 0L)
+ jjCheckNAddTwoStates(30, 31);
+ break;
+ case 31:
+ if (curChar != 46)
+ break;
+ if (kind > 18)
+ kind = 18;
+ jjCheckNAddTwoStates(32, 33);
+ break;
+ case 32:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 18)
+ kind = 18;
+ jjCheckNAddTwoStates(32, 33);
+ break;
+ case 34:
+ if ((0x280000000000L & l) != 0L)
+ jjCheckNAdd(35);
+ break;
+ case 35:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 18)
+ kind = 18;
+ jjCheckNAdd(35);
+ break;
+ case 36:
+ if ((0x3ff000000000000L & l) != 0L)
+ jjCheckNAddTwoStates(36, 37);
+ break;
+ case 38:
+ if ((0x280000000000L & l) != 0L)
+ jjCheckNAdd(39);
+ break;
+ case 39:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 18)
+ kind = 18;
+ jjCheckNAdd(39);
+ break;
+ default:
+ break;
+ }
+ } while (i != startsAt);
+ } else if (curChar < 128) {
+ long l = 1L << (curChar & 077);
+ do {
+ switch (jjstateSet[--i]) {
+ case 5:
+ case 28:
+ if ((0x7fffffe87fffffeL & l) == 0L)
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAdd(28);
+ break;
+ case 1:
+ jjAddStates(7, 9);
+ break;
+ case 7:
+ jjCheckNAddTwoStates(7, 8);
+ break;
+ case 9:
+ case 10:
+ jjCheckNAddTwoStates(10, 8);
+ break;
+ case 16:
+ if ((0x100000001000L & l) != 0L && kind > 17)
+ kind = 17;
+ break;
+ case 19:
+ if ((0x2000000020L & l) != 0L)
+ jjAddStates(13, 14);
+ break;
+ case 25:
+ jjAddStates(4, 6);
+ break;
+ case 33:
+ if ((0x2000000020L & l) != 0L)
+ jjAddStates(15, 16);
+ break;
+ case 37:
+ if ((0x2000000020L & l) != 0L)
+ jjAddStates(17, 18);
+ break;
+ default:
+ break;
+ }
+ } while (i != startsAt);
+ } else {
+ int hiByte = (int) (curChar >> 8);
+ int i1 = hiByte >> 6;
+ long l1 = 1L << (hiByte & 077);
+ int i2 = (curChar & 0xff) >> 6;
+ long l2 = 1L << (curChar & 077);
+ do {
+ switch (jjstateSet[--i]) {
+ case 1:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ jjAddStates(7, 9);
+ break;
+ case 7:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ jjCheckNAddTwoStates(7, 8);
+ break;
+ case 9:
+ case 10:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ jjCheckNAddTwoStates(10, 8);
+ break;
+ case 25:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ jjAddStates(4, 6);
+ break;
+ default:
+ break;
+ }
+ } while (i != startsAt);
+ }
+ if (kind != 0x7fffffff) {
+ jjmatchedKind = kind;
+ jjmatchedPos = curPos;
+ kind = 0x7fffffff;
+ }
+ ++curPos;
+ if ((i = jjnewStateCnt) == (startsAt = 40 - (jjnewStateCnt = startsAt)))
+ break;
+ try {
+ curChar = inputStream.readChar();
+ } catch (java.io.IOException e) {
+ break;
+ }
+ }
+ if (jjmatchedPos > strPos)
+ return curPos;
+
+ int toRet = Math.max(curPos, seenUpto);
+
+ if (curPos < toRet)
+ for (i = toRet - Math.min(curPos, seenUpto); i-- > 0; )
+ try {
+ curChar = inputStream.readChar();
+ } catch (java.io.IOException e) {
+ throw new Error("Internal Error : Please send a bug report.");
+ }
+
+ if (jjmatchedPos < strPos) {
+ jjmatchedKind = strKind;
+ jjmatchedPos = strPos;
+ } else if (jjmatchedPos == strPos && jjmatchedKind > strKind)
+ jjmatchedKind = strKind;
+
+ return toRet;
+ }
+
+ static final int[] JJ_NEXT_STATES = {
+ 30, 31, 36, 37, 24, 25, 26, 1, 2, 4, 8, 9, 11, 20, 21, 34,
+ 35, 38, 39,
+ };
+
+ private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2) {
+ switch (hiByte) {
+ case 0:
+ return (JJ_BIT_VEC_2[i2] & l2) != 0L;
+ default:
+ if ((JJ_BIT_VEC_0[i1] & l1) != 0L)
+ return true;
+ return false;
+ }
+ }
+
+ /**
+ * Token literal values.
+ */
+ public static final String[] JJ_STR_LITERAL_IMAGES = {
+ "", null, null, null, null, null, null, null, null, null, null, null, null,
+ null, null, null, null, null, null, null, null, null, "\75", "\74\76", "\76",
+ "\76\75", "\74", "\74\75", "\50", "\54", "\51", "\53", "\55"};
+
+ /**
+ * Lexer state names.
+ */
+ public static final String[] LEX_STATE_NAMES = {
+ "DEFAULT",
+ };
+ static final long[] JJ_TO_TOKEN = {
+ 0x1fff7ff01L,
+ };
+ static final long[] JJ_TO_SKIP = {
+ 0xfeL,
+ };
+ static final long[] JJ_TO_SPECIAL = {
+ 0x3eL,
+ };
+ protected SimpleCharStream inputStream;
+ private final int[] jjrounds = new int[40];
+ private final int[] jjstateSet = new int[80];
+ protected char curChar;
+
+ /**
+ * Constructor.
+ */
+ public SelectorParserTokenManager(SimpleCharStream stream) {
+ if (SimpleCharStream.STATIC_FLAG)
+ throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.");
+ inputStream = stream;
+ }
+
+ /**
+ * Constructor.
+ */
+ public SelectorParserTokenManager(SimpleCharStream stream, int lexState) {
+ this(stream);
+ SwitchTo(lexState);
+ }
+
+ /**
+ * Reinitialise parser.
+ */
+ public void ReInit(SimpleCharStream stream) {
+ jjmatchedPos = jjnewStateCnt = 0;
+ curLexState = defaultLexState;
+ inputStream = stream;
+ ReInitRounds();
+ }
+
+ private void ReInitRounds() {
+ int i;
+ jjround = 0x80000001;
+ for (i = 40; i-- > 0; )
+ jjrounds[i] = 0x80000000;
+ }
+
+ /**
+ * Reinitialise parser.
+ */
+ public void ReInit(SimpleCharStream stream, int lexState) {
+ ReInit(stream);
+ SwitchTo(lexState);
+ }
+
+ /**
+ * Switch to specified lex state.
+ */
+ public void SwitchTo(int lexState) {
+ if (lexState >= 1 || lexState < 0)
+ throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.",
+ TokenMgrError.INVALID_LEXICAL_STATE);
+ else
+ curLexState = lexState;
+ }
+
+ protected Token jjFillToken() {
+ final Token t;
+ final String curTokenImage;
+ final int beginLine;
+ final int endLine;
+ final int beginColumn;
+ final int endColumn;
+ String im = JJ_STR_LITERAL_IMAGES[jjmatchedKind];
+ curTokenImage = (im == null) ? inputStream.GetImage() : im;
+ beginLine = inputStream.getBeginLine();
+ beginColumn = inputStream.getBeginColumn();
+ endLine = inputStream.getEndLine();
+ endColumn = inputStream.getEndColumn();
+ t = Token.newToken(jjmatchedKind, curTokenImage);
+
+ t.beginLine = beginLine;
+ t.endLine = endLine;
+ t.beginColumn = beginColumn;
+ t.endColumn = endColumn;
+
+ return t;
+ }
+
+ int curLexState = 0;
+ int defaultLexState = 0;
+ int jjnewStateCnt;
+ int jjround;
+ int jjmatchedPos;
+ int jjmatchedKind;
+
+ /**
+ * Get the next Token.
+ */
+ public Token getNextToken() {
+ Token specialToken = null;
+ Token matchedToken;
+ int curPos = 0;
+
+ EOFLoop:
+ for (;;) {
+ try {
+ curChar = inputStream.BeginToken();
+ } catch (java.io.IOException e) {
+ jjmatchedKind = 0;
+ matchedToken = jjFillToken();
+ matchedToken.specialToken = specialToken;
+ return matchedToken;
+ }
+
+ jjmatchedKind = 0x7fffffff;
+ jjmatchedPos = 0;
+ curPos = jjMoveStringLiteralDfa0_0();
+ if (jjmatchedKind != 0x7fffffff) {
+ if (jjmatchedPos + 1 < curPos)
+ inputStream.backup(curPos - jjmatchedPos - 1);
+ if ((JJ_TO_TOKEN[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L) {
+ matchedToken = jjFillToken();
+ matchedToken.specialToken = specialToken;
+ return matchedToken;
+ } else {
+ if ((JJ_TO_SPECIAL[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L) {
+ matchedToken = jjFillToken();
+ if (specialToken == null)
+ specialToken = matchedToken;
+ else {
+ matchedToken.specialToken = specialToken;
+ specialToken = specialToken.next = matchedToken;
+ }
+ }
+ continue EOFLoop;
+ }
+ }
+ int errorLine = inputStream.getEndLine();
+ int errorColumn = inputStream.getEndColumn();
+ String errorAfter = null;
+ boolean eofSeen = false;
+ try {
+ inputStream.readChar();
+ inputStream.backup(1);
+ } catch (java.io.IOException e1) {
+ eofSeen = true;
+ errorAfter = curPos <= 1 ? "" : inputStream.GetImage();
+ if (curChar == '\n' || curChar == '\r') {
+ errorLine++;
+ errorColumn = 0;
+ } else
+ errorColumn++;
+ }
+ if (!eofSeen) {
+ inputStream.backup(1);
+ errorAfter = curPos <= 1 ? "" : inputStream.GetImage();
+ }
+ throw new TokenMgrError(eofSeen, curLexState, errorLine, errorColumn, errorAfter, curChar,
+ TokenMgrError.LEXICAL_ERROR);
+ }
+ }
+
+ private void jjCheckNAdd(int state) {
+ if (jjrounds[state] != jjround) {
+ jjstateSet[jjnewStateCnt++] = state;
+ jjrounds[state] = jjround;
+ }
+ }
+
+ private void jjAddStates(int start, int end) {
+ do {
+ jjstateSet[jjnewStateCnt++] = JJ_NEXT_STATES[start];
+ } while (start++ != end);
+ }
+
+ private void jjCheckNAddTwoStates(int state1, int state2) {
+ jjCheckNAdd(state1);
+ jjCheckNAdd(state2);
+ }
+
+ private void jjCheckNAddStates(int start, int end) {
+ do {
+ jjCheckNAdd(JJ_NEXT_STATES[start]);
+ } while (start++ != end);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java
new file mode 100644
index 0000000..94a54b4
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java
@@ -0,0 +1,502 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 5.0 */
+/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
+package org.apache.rocketmq.filter.parser;
+
+/**
+ * An implementation of interface CharStream, where the stream is assumed to
+ * contain only ASCII characters (without unicode processing).
+ */
+
+public class SimpleCharStream {
+ /**
+ * Whether parser is static.
+ */
+ public static final boolean STATIC_FLAG = false;
+ int bufsize;
+ int available;
+ int tokenBegin;
+ /**
+ * Position in buffer.
+ */
+ public int bufpos = -1;
+ protected int bufline[];
+ protected int bufcolumn[];
+
+ protected int column = 0;
+ protected int line = 1;
+
+ protected boolean prevCharIsCR = false;
+ protected boolean prevCharIsLF = false;
+
+ protected java.io.Reader inputStream;
+
+ protected char[] buffer;
+ protected int maxNextCharInd = 0;
+ protected int inBuf = 0;
+ protected int tabSize = 8;
+
+ protected void setTabSize(int i) {
+ tabSize = i;
+ }
+
+ protected int getTabSize(int i) {
+ return tabSize;
+ }
+
+ protected void ExpandBuff(boolean wrapAround) {
+ char[] newbuffer = new char[bufsize + 2048];
+ int newbufline[] = new int[bufsize + 2048];
+ int newbufcolumn[] = new int[bufsize + 2048];
+
+ try {
+ if (wrapAround) {
+ System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+ System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos);
+ buffer = newbuffer;
+
+ System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+ System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
+ bufline = newbufline;
+
+ System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+ System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);
+ bufcolumn = newbufcolumn;
+
+ maxNextCharInd = bufpos += bufsize - tokenBegin;
+ } else {
+ System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+ buffer = newbuffer;
+
+ System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+ bufline = newbufline;
+
+ System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+ bufcolumn = newbufcolumn;
+
+ maxNextCharInd = bufpos -= tokenBegin;
+ }
+ } catch (Throwable t) {
+ throw new Error(t.getMessage());
+ }
+
+ bufsize += 2048;
+ available = bufsize;
+ tokenBegin = 0;
+ }
+
+ protected void FillBuff() throws java.io.IOException {
+ if (maxNextCharInd == available) {
+ if (available == bufsize) {
+ if (tokenBegin > 2048) {
+ bufpos = maxNextCharInd = 0;
+ available = tokenBegin;
+ } else if (tokenBegin < 0)
+ bufpos = maxNextCharInd = 0;
+ else
+ ExpandBuff(false);
+ } else if (available > tokenBegin)
+ available = bufsize;
+ else if ((tokenBegin - available) < 2048)
+ ExpandBuff(true);
+ else
+ available = tokenBegin;
+ }
+
+ int i;
+ try {
+ if ((i = inputStream.read(buffer, maxNextCharInd, available - maxNextCharInd)) == -1) {
+ inputStream.close();
+ throw new java.io.IOException();
+ } else
+ maxNextCharInd += i;
+ return;
+ } catch (java.io.IOException e) {
+ --bufpos;
+ backup(0);
+ if (tokenBegin == -1)
+ tokenBegin = bufpos;
+ throw e;
+ }
+ }
+
+ /**
+ * Start.
+ */
+ public char BeginToken() throws java.io.IOException {
+ tokenBegin = -1;
+ char c = readChar();
+ tokenBegin = bufpos;
+
+ return c;
+ }
+
+ protected void UpdateLineColumn(char c) {
+ column++;
+
+ if (prevCharIsLF) {
+ prevCharIsLF = false;
+ line += column = 1;
+ } else if (prevCharIsCR) {
+ prevCharIsCR = false;
+ if (c == '\n') {
+ prevCharIsLF = true;
+ } else
+ line += column = 1;
+ }
+
+ switch (c) {
+ case '\r':
+ prevCharIsCR = true;
+ break;
+ case '\n':
+ prevCharIsLF = true;
+ break;
+ case '\t':
+ column--;
+ column += tabSize - (column % tabSize);
+ break;
+ default:
+ break;
+ }
+
+ bufline[bufpos] = line;
+ bufcolumn[bufpos] = column;
+ }
+
+ /**
+ * Read a character.
+ */
+ public char readChar() throws java.io.IOException {
+ if (inBuf > 0) {
+ --inBuf;
+
+ if (++bufpos == bufsize)
+ bufpos = 0;
+
+ return buffer[bufpos];
+ }
+
+ if (++bufpos >= maxNextCharInd)
+ FillBuff();
+
+ char c = buffer[bufpos];
+
+ UpdateLineColumn(c);
+ return c;
+ }
+
+ @Deprecated
+ /**
+ * @deprecated
+ * @see #getEndColumn
+ */
+
+ public int getColumn() {
+ return bufcolumn[bufpos];
+ }
+
+ @Deprecated
+ /**
+ * @deprecated
+ * @see #getEndLine
+ */
+
+ public int getLine() {
+ return bufline[bufpos];
+ }
+
+ /**
+ * Get token end column number.
+ */
+ public int getEndColumn() {
+ return bufcolumn[bufpos];
+ }
+
+ /**
+ * Get token end line number.
+ */
+ public int getEndLine() {
+ return bufline[bufpos];
+ }
+
+ /**
+ * Get token beginning column number.
+ */
+ public int getBeginColumn() {
+ return bufcolumn[tokenBegin];
+ }
+
+ /**
+ * Get token beginning line number.
+ */
+ public int getBeginLine() {
+ return bufline[tokenBegin];
+ }
+
+ /**
+ * Backup a number of characters.
+ */
+ public void backup(int amount) {
+
+ inBuf += amount;
+ if ((bufpos -= amount) < 0)
+ bufpos += bufsize;
+ }
+
+ /**
+ * Constructor.
+ */
+ public SimpleCharStream(java.io.Reader dstream, int startline,
+ int startcolumn, int buffersize) {
+ inputStream = dstream;
+ line = startline;
+ column = startcolumn - 1;
+
+ available = bufsize = buffersize;
+ buffer = new char[buffersize];
+ bufline = new int[buffersize];
+ bufcolumn = new int[buffersize];
+ }
+
+ /**
+ * Constructor.
+ */
+ public SimpleCharStream(java.io.Reader dstream, int startline,
+ int startcolumn) {
+ this(dstream, startline, startcolumn, 4096);
+ }
+
+ /**
+ * Constructor.
+ */
+ public SimpleCharStream(java.io.Reader dstream) {
+ this(dstream, 1, 1, 4096);
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.Reader dstream, int startline,
+ int startcolumn, int buffersize) {
+ inputStream = dstream;
+ line = startline;
+ column = startcolumn - 1;
+
+ if (buffer == null || buffersize != buffer.length) {
+ available = bufsize = buffersize;
+ buffer = new char[buffersize];
+ bufline = new int[buffersize];
+ bufcolumn = new int[buffersize];
+ }
+ prevCharIsLF = prevCharIsCR = false;
+ tokenBegin = inBuf = maxNextCharInd = 0;
+ bufpos = -1;
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.Reader dstream, int startline,
+ int startcolumn) {
+ ReInit(dstream, startline, startcolumn, 4096);
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.Reader dstream) {
+ ReInit(dstream, 1, 1, 4096);
+ }
+
+ /**
+ * Constructor.
+ */
+ public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException {
+ this(encoding == null ?
+ new java.io.InputStreamReader(dstream) :
+ new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ }
+
+ /**
+ * Constructor.
+ */
+ public SimpleCharStream(java.io.InputStream dstream, int startline,
+ int startcolumn, int buffersize) {
+ this(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+ }
+
+ /**
+ * Constructor.
+ */
+ public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn) throws java.io.UnsupportedEncodingException {
+ this(dstream, encoding, startline, startcolumn, 4096);
+ }
+
+ /**
+ * Constructor.
+ */
+ public SimpleCharStream(java.io.InputStream dstream, int startline,
+ int startcolumn) {
+ this(dstream, startline, startcolumn, 4096);
+ }
+
+ /**
+ * Constructor.
+ */
+ public SimpleCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException {
+ this(dstream, encoding, 1, 1, 4096);
+ }
+
+ /**
+ * Constructor.
+ */
+ public SimpleCharStream(java.io.InputStream dstream) {
+ this(dstream, 1, 1, 4096);
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException {
+ ReInit(encoding == null ?
+ new java.io.InputStreamReader(dstream) :
+ new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.InputStream dstream, int startline,
+ int startcolumn, int buffersize) {
+ ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException {
+ ReInit(dstream, encoding, 1, 1, 4096);
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.InputStream dstream) {
+ ReInit(dstream, 1, 1, 4096);
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn) throws java.io.UnsupportedEncodingException {
+ ReInit(dstream, encoding, startline, startcolumn, 4096);
+ }
+
+ /**
+ * Reinitialise.
+ */
+ public void ReInit(java.io.InputStream dstream, int startline,
+ int startcolumn) {
+ ReInit(dstream, startline, startcolumn, 4096);
+ }
+
+ /**
+ * Get token literal value.
+ */
+ public String GetImage() {
+ if (bufpos >= tokenBegin)
+ return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
+ else
+ return new String(buffer, tokenBegin, bufsize - tokenBegin) +
+ new String(buffer, 0, bufpos + 1);
+ }
+
+ /**
+ * Get the suffix.
+ */
+ public char[] GetSuffix(int len) {
+ char[] ret = new char[len];
+
+ if ((bufpos + 1) >= len)
+ System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);
+ else {
+ System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,
+ len - bufpos - 1);
+ System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);
+ }
+
+ return ret;
+ }
+
+ /**
+ * Reset buffer when finished.
+ */
+ public void Done() {
+ buffer = null;
+ bufline = null;
+ bufcolumn = null;
+ }
+
+ /**
+ * Method to adjust line and column numbers for the start of a token.
+ */
+ public void adjustBeginLineColumn(int newLine, int newCol) {
+ int start = tokenBegin;
+ int len;
+
+ if (bufpos >= tokenBegin) {
+ len = bufpos - tokenBegin + inBuf + 1;
+ } else {
+ len = bufsize - tokenBegin + bufpos + 1 + inBuf;
+ }
+
+ int i = 0, j = 0, k = 0;
+ int nextColDiff = 0, columnDiff = 0;
+
+ while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) {
+ bufline[j] = newLine;
+ nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];
+ bufcolumn[j] = newCol + columnDiff;
+ columnDiff = nextColDiff;
+ i++;
+ }
+
+ if (i < len) {
+ bufline[j] = newLine++;
+ bufcolumn[j] = newCol + columnDiff;
+
+ while (i++ < len) {
+ if (bufline[j = start % bufsize] != bufline[++start % bufsize])
+ bufline[j] = newLine++;
+ else
+ bufline[j] = newLine;
+ }
+ }
+
+ line = bufline[j];
+ column = bufcolumn[j];
+ }
+
+}
+/* JavaCC - OriginalChecksum=af79bfe4b18b4b4ea9720ffeb7e52fc5 (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/parser/Token.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/Token.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/Token.java
new file mode 100644
index 0000000..8e6a48a
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/Token.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */
+/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
+package org.apache.rocketmq.filter.parser;
+
+/**
+ * Describes the input token stream.
+ */
+
+public class Token implements java.io.Serializable {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * An integer that describes the kind of this token. This numbering
+ * system is determined by JavaCCParser, and a table of these numbers is
+ * stored in the file ...Constants.java.
+ */
+ public int kind;
+
+ /**
+ * The line number of the first character of this Token.
+ */
+ public int beginLine;
+ /**
+ * The column number of the first character of this Token.
+ */
+ public int beginColumn;
+ /**
+ * The line number of the last character of this Token.
+ */
+ public int endLine;
+ /**
+ * The column number of the last character of this Token.
+ */
+ public int endColumn;
+
+ /**
+ * The string image of the token.
+ */
+ public String image;
+
+ /**
+ * A reference to the next regular (non-special) token from the input
+ * stream. If this is the last token from the input stream, or if the
+ * token manager has not read tokens beyond this one, this field is
+ * set to null. This is true only if this token is also a regular
+ * token. Otherwise, see below for a description of the contents of
+ * this field.
+ */
+ public Token next;
+
+ /**
+ * This field is used to access special tokens that occur prior to this
+ * token, but after the immediately preceding regular (non-special) token.
+ * If there are no such special tokens, this field is set to null.
+ * When there are more than one such special token, this field refers
+ * to the last of these special tokens, which in turn refers to the next
+ * previous special token through its specialToken field, and so on
+ * until the first special token (whose specialToken field is null).
+ * The next fields of special tokens refer to other special tokens that
+ * immediately follow it (without an intervening regular token). If there
+ * is no such token, this field is null.
+ */
+ public Token specialToken;
+
+ /**
+ * An optional attribute value of the Token.
+ * Tokens which are not used as syntactic sugar will often contain
+ * meaningful values that will be used later on by the compiler or
+ * interpreter. This attribute value is often different from the image.
+ * Any subclass of Token that actually wants to return a non-null value can
+ * override this method as appropriate.
+ */
+ public Object getValue() {
+ return null;
+ }
+
+ /**
+ * No-argument constructor
+ */
+ public Token() {
+ }
+
+ /**
+ * Constructs a new token for the specified Image.
+ */
+ public Token(int kind) {
+ this(kind, null);
+ }
+
+ /**
+ * Constructs a new token for the specified Image and Kind.
+ */
+ public Token(int kind, String image) {
+ this.kind = kind;
+ this.image = image;
+ }
+
+ /**
+ * Returns the image.
+ */
+ public String toString() {
+ return image;
+ }
+
+ /**
+ * Returns a new Token object, by default. However, if you want, you
+ * can create and return subclass objects based on the value of ofKind.
+ * Simply add the cases to the switch for all those special cases.
+ * For example, if you have a subclass of Token called IDToken that
+ * you want to create if ofKind is ID, simply add something like :
+ * <p/>
+ * case MyParserConstants.ID : return new IDToken(ofKind, image);
+ * <p/>
+ * to the following switch statement. Then you can cast matchedToken
+ * variable to the appropriate type and use sit in your lexical actions.
+ */
+ public static Token newToken(int ofKind, String image) {
+ switch (ofKind) {
+ default:
+ return new Token(ofKind, image);
+ }
+ }
+
+ public static Token newToken(int ofKind) {
+ return newToken(ofKind, null);
+ }
+
+}
+/* JavaCC - OriginalChecksum=6b0af88eb45a551d929d3cdd9582f827 (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/parser/TokenMgrError.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/TokenMgrError.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/TokenMgrError.java
new file mode 100644
index 0000000..75d83e5
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/TokenMgrError.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 */
+/* JavaCCOptions: */
+package org.apache.rocketmq.filter.parser;
+
+/**
+ * Token Manager Error.
+ */
+public class TokenMgrError extends Error {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /*
+ * Ordinals for various reasons why an Error of this type can be thrown.
+ */
+
+ /**
+ * Lexical error occurred.
+ */
+ static final int LEXICAL_ERROR = 0;
+
+ /**
+ * An attempt was made to create a second instance of a static token manager.
+ */
+ static final int STATIC_LEXER_ERROR = 1;
+
+ /**
+ * Tried to change to an invalid lexical state.
+ */
+ static final int INVALID_LEXICAL_STATE = 2;
+
+ /**
+ * Detected (and bailed out of) an infinite loop in the token manager.
+ */
+ static final int LOOP_DETECTED = 3;
+
+ /**
+ * Indicates the reason why the exception is thrown. It will have
+ * one of the above 4 values.
+ */
+ int errorCode;
+
+ /**
+ * Replaces unprintable characters by their escaped (or unicode escaped)
+ * equivalents in the given string
+ */
+ protected static final String addEscapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for (int i = 0; i < str.length(); i++) {
+ switch (str.charAt(i)) {
+ case 0:
+ continue;
+ case '\b':
+ retval.append("\\b");
+ continue;
+ case '\t':
+ retval.append("\\t");
+ continue;
+ case '\n':
+ retval.append("\\n");
+ continue;
+ case '\f':
+ retval.append("\\f");
+ continue;
+ case '\r':
+ retval.append("\\r");
+ continue;
+ case '\"':
+ retval.append("\\\"");
+ continue;
+ case '\'':
+ retval.append("\\\'");
+ continue;
+ case '\\':
+ retval.append("\\\\");
+ continue;
+ default:
+ if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ } else {
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+
+ /**
+ * Returns a detailed message for the Error when it is thrown by the
+ * token manager to indicate a lexical error.
+ * Parameters :
+ * eofSeen : indicates if EOF caused the lexical error
+ * curLexState : lexical state in which this error occurred
+ * errorLine : line number when the error occurred
+ * errorColumn : column number when the error occurred
+ * errorAfter : prefix that was seen before this error occurred
+ * curchar : the offending character
+ * Note: You can customize the lexical error message by modifying this method.
+ */
+ protected static String LexicalError(boolean eofSeen, int lexState, int errorLine, int errorColumn,
+ String errorAfter, char curChar) {
+ return "Lexical error at line " +
+ errorLine + ", column " +
+ errorColumn + ". Encountered: " +
+ (eofSeen ?
+ "<EOF> " :
+ ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int) curChar + "), ") +
+ "after : \"" + addEscapes(errorAfter) + "\"";
+ }
+
+ /**
+ * You can also modify the body of this method to customize your error messages.
+ * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+ * of end-users concern, so you can return something like :
+ * <p/>
+ * "Internal Error : Please file a bug report .... "
+ * <p/>
+ * from this method for such cases in the release version of your parser.
+ */
+ public String getMessage() {
+ return super.getMessage();
+ }
+
+ /*
+ * Constructors of various flavors follow.
+ */
+
+ /**
+ * No arg constructor.
+ */
+ public TokenMgrError() {
+ }
+
+ /**
+ * Constructor with message and reason.
+ */
+ public TokenMgrError(String message, int reason) {
+ super(message);
+ errorCode = reason;
+ }
+
+ /**
+ * Full Constructor.
+ */
+ public TokenMgrError(boolean eofSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar,
+ int reason) {
+ this(LexicalError(eofSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
+ }
+}
+/* JavaCC - OriginalChecksum=e960778c8dcd73e167ed5bfddd59f288 (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/util/BitsArray.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/util/BitsArray.java b/filter/src/main/java/org/apache/rocketmq/filter/util/BitsArray.java
new file mode 100644
index 0000000..9866854
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/util/BitsArray.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.util;
+
+/**
+ * Wrapper of bytes array, in order to operate single bit easily.
+ */
+public class BitsArray implements Cloneable {
+
+ private byte[] bytes;
+ private int bitLength;
+
+ public static BitsArray create(int bitLength) {
+ return new BitsArray(bitLength);
+ }
+
+ public static BitsArray create(byte[] bytes, int bitLength) {
+ return new BitsArray(bytes, bitLength);
+ }
+
+ public static BitsArray create(byte[] bytes) {
+ return new BitsArray(bytes);
+ }
+
+ private BitsArray(int bitLength) {
+ this.bitLength = bitLength;
+ // init bytes
+ int temp = bitLength / Byte.SIZE;
+ if (bitLength % Byte.SIZE > 0) {
+ temp++;
+ }
+ bytes = new byte[temp];
+ for (int i = 0; i < bytes.length; i++) {
+ bytes[i] = (byte) 0x00;
+ }
+ }
+
+ private BitsArray(byte[] bytes, int bitLength) {
+ if (bytes == null || bytes.length < 1) {
+ throw new IllegalArgumentException("Bytes is empty!");
+ }
+
+ if (bitLength < 1) {
+ throw new IllegalArgumentException("Bit is less than 1.");
+ }
+
+ if (bitLength < bytes.length * Byte.SIZE) {
+ throw new IllegalArgumentException("BitLength is less than bytes.length() * " + Byte.SIZE);
+ }
+
+ this.bytes = new byte[bytes.length];
+ System.arraycopy(bytes, 0, this.bytes, 0, this.bytes.length);
+ this.bitLength = bitLength;
+ }
+
+ private BitsArray(byte[] bytes) {
+ if (bytes == null || bytes.length < 1) {
+ throw new IllegalArgumentException("Bytes is empty!");
+ }
+
+ this.bitLength = bytes.length * Byte.SIZE;
+ this.bytes = new byte[bytes.length];
+ System.arraycopy(bytes, 0, this.bytes, 0, this.bytes.length);
+ }
+
+ public int bitLength() {
+ return this.bitLength;
+ }
+
+ public int byteLength() {
+ return this.bytes.length;
+ }
+
+ public byte[] bytes() {
+ return this.bytes;
+ }
+
+ public void xor(final BitsArray other) {
+ checkInitialized(this);
+ checkInitialized(other);
+
+ int minByteLength = Math.min(this.byteLength(), other.byteLength());
+
+ for (int i = 0; i < minByteLength; i++) {
+ this.bytes[i] = (byte) (this.bytes[i] ^ other.getByte(i));
+ }
+ }
+
+ public void xor(int bitPos, boolean set) {
+ checkBitPosition(bitPos, this);
+
+ boolean value = getBit(bitPos);
+ if (value ^ set) {
+ setBit(bitPos, true);
+ } else {
+ setBit(bitPos, false);
+ }
+ }
+
+ public void or(final BitsArray other) {
+ checkInitialized(this);
+ checkInitialized(other);
+
+ int minByteLength = Math.min(this.byteLength(), other.byteLength());
+
+ for (int i = 0; i < minByteLength; i++) {
+ this.bytes[i] = (byte) (this.bytes[i] | other.getByte(i));
+ }
+ }
+
+ public void or(int bitPos, boolean set) {
+ checkBitPosition(bitPos, this);
+
+ if (set) {
+ setBit(bitPos, true);
+ }
+ }
+
+ public void and(final BitsArray other) {
+ checkInitialized(this);
+ checkInitialized(other);
+
+ int minByteLength = Math.min(this.byteLength(), other.byteLength());
+
+ for (int i = 0; i < minByteLength; i++) {
+ this.bytes[i] = (byte) (this.bytes[i] & other.getByte(i));
+ }
+ }
+
+ public void and(int bitPos, boolean set) {
+ checkBitPosition(bitPos, this);
+
+ if (!set) {
+ setBit(bitPos, false);
+ }
+ }
+
+ public void not(int bitPos) {
+ checkBitPosition(bitPos, this);
+
+ setBit(bitPos, !getBit(bitPos));
+ }
+
+ public void setBit(int bitPos, boolean set) {
+ checkBitPosition(bitPos, this);
+ int sub = subscript(bitPos);
+ int pos = position(bitPos);
+ if (set) {
+ this.bytes[sub] = (byte) (this.bytes[sub] | pos);
+ } else {
+ this.bytes[sub] = (byte) (this.bytes[sub] & ~pos);
+ }
+ }
+
+ public void setByte(int bytePos, byte set) {
+ checkBytePosition(bytePos, this);
+
+ this.bytes[bytePos] = set;
+ }
+
+ public boolean getBit(int bitPos) {
+ checkBitPosition(bitPos, this);
+
+ return (this.bytes[subscript(bitPos)] & position(bitPos)) != 0;
+ }
+
+ public byte getByte(int bytePos) {
+ checkBytePosition(bytePos, this);
+
+ return this.bytes[bytePos];
+ }
+
+ protected int subscript(int bitPos) {
+ return bitPos / Byte.SIZE;
+ }
+
+ protected int position(int bitPos) {
+ return 1 << bitPos % Byte.SIZE;
+ }
+
+ protected void checkBytePosition(int bytePos, BitsArray bitsArray) {
+ checkInitialized(bitsArray);
+ if (bytePos > bitsArray.byteLength()) {
+ throw new IllegalArgumentException("BytePos is greater than " + bytes.length);
+ }
+ if (bytePos < 0) {
+ throw new IllegalArgumentException("BytePos is less than 0");
+ }
+ }
+
+ protected void checkBitPosition(int bitPos, BitsArray bitsArray) {
+ checkInitialized(bitsArray);
+ if (bitPos > bitsArray.bitLength()) {
+ throw new IllegalArgumentException("BitPos is greater than " + bitLength);
+ }
+ if (bitPos < 0) {
+ throw new IllegalArgumentException("BitPos is less than 0");
+ }
+ }
+
+ protected void checkInitialized(BitsArray bitsArray) {
+ if (bitsArray.bytes() == null) {
+ throw new RuntimeException("Not initialized!");
+ }
+ }
+
+ public BitsArray clone() {
+ byte[] clone = new byte[this.byteLength()];
+
+ System.arraycopy(this.bytes, 0, clone, 0, this.byteLength());
+
+ return create(clone, bitLength());
+ }
+
+ @Override
+ public String toString() {
+ if (this.bytes == null) {
+ return "null";
+ }
+ StringBuilder stringBuilder = new StringBuilder(this.bytes.length * Byte.SIZE);
+ for (int i = this.bytes.length - 1; i >= 0; i--) {
+
+ int j = Byte.SIZE - 1;
+ if (i == this.bytes.length - 1 && this.bitLength % Byte.SIZE > 0) {
+ // not full byte
+ j = this.bitLength % Byte.SIZE;
+ }
+
+ for (; j >= 0; j--) {
+
+ byte mask = (byte) (1 << j);
+ if ((this.bytes[i] & mask) == mask) {
+ stringBuilder.append("1");
+ } else {
+ stringBuilder.append("0");
+ }
+ }
+ if (i % 8 == 0) {
+ stringBuilder.append("\n");
+ }
+ }
+
+ return stringBuilder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilter.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilter.java b/filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilter.java
new file mode 100644
index 0000000..f610906
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilter.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.util;
+
+import com.google.common.hash.Hashing;
+
+import java.nio.charset.Charset;
+
+/**
+ * Simple implement of bloom filter.
+ */
+public class BloomFilter {
+
+ public static final Charset UTF_8 = Charset.forName("UTF-8");
+
+ // as error rate, 10/100 = 0.1
+ private int f = 10;
+ private int n = 128;
+
+ // hash function num, by calculation.
+ private int k;
+ // bit count, by calculation.
+ private int m;
+
+ /**
+ * Create bloom filter by error rate and mapping num.
+ *
+ * @param f error rate
+ * @param n num will mapping to bit
+ * @return
+ */
+ public static BloomFilter createByFn(int f, int n) {
+ return new BloomFilter(f, n);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param f error rate
+ * @param n num will mapping to bit
+ */
+ private BloomFilter(int f, int n) {
+ if (f < 1 || f >= 100) {
+ throw new IllegalArgumentException("f must be greater or equal than 1 and less than 100");
+ }
+ if (n < 1) {
+ throw new IllegalArgumentException("n must be greater than 0");
+ }
+
+ this.f = f;
+ this.n = n;
+
+ // set p = e^(-kn/m)
+ // f = (1 - p)^k = e^(kln(1-p))
+ // when p = 0.5, k = ln2 * (m/n), f = (1/2)^k = (0.618)^(m/n)
+ double errorRate = f / 100.0;
+ this.k = (int) Math.ceil(logMN(0.5, errorRate));
+
+ if (this.k < 1) {
+ throw new IllegalArgumentException("Hash function num is less than 1, maybe you should change the value of error rate or bit num!");
+ }
+
+ // m >= n*log2(1/f)*log2(e)
+ this.m = (int) Math.ceil(this.n * logMN(2, 1 / errorRate) * logMN(2, Math.E));
+ // m%8 = 0
+ this.m = (int) (Byte.SIZE * Math.ceil(this.m / (Byte.SIZE * 1.0)));
+ }
+
+ /**
+ * Calculate bit positions of {@code str}.
+ * <p>
+ * See "Less Hashing, Same Performance: Building a Better Bloom Filter" by Adam Kirsch and Michael
+ * Mitzenmacher.
+ * </p>
+ *
+ * @param str
+ * @return
+ */
+ public int[] calcBitPositions(String str) {
+ int[] bitPositions = new int[this.k];
+
+ long hash64 = Hashing.murmur3_128().hashString(str, UTF_8).asLong();
+
+ int hash1 = (int) hash64;
+ int hash2 = (int) (hash64 >>> 32);
+
+ for (int i = 1; i <= this.k; i++) {
+ int combinedHash = hash1 + (i * hash2);
+ // Flip all the bits if it's negative (guaranteed positive number)
+ if (combinedHash < 0) {
+ combinedHash = ~combinedHash;
+ }
+ bitPositions[i - 1] = combinedHash % this.m;
+ }
+
+ return bitPositions;
+ }
+
+ /**
+ * Calculate bit positions of {@code str} to construct {@code BloomFilterData}
+ *
+ * @param str
+ * @return
+ */
+ public BloomFilterData generate(String str) {
+ int[] bitPositions = calcBitPositions(str);
+
+ return new BloomFilterData(bitPositions, this.m);
+ }
+
+ /**
+ * Calculate bit positions of {@code str}, then set the related {@code bits} positions to 1.
+ *
+ * @param str
+ * @param bits
+ */
+ public void hashTo(String str, BitsArray bits) {
+ hashTo(calcBitPositions(str), bits);
+ }
+
+ /**
+ * Set the related {@code bits} positions to 1.
+ *
+ * @param bitPositions
+ * @param bits
+ */
+ public void hashTo(int[] bitPositions, BitsArray bits) {
+ check(bits);
+
+ for (int i : bitPositions) {
+ bits.setBit(i, true);
+ }
+ }
+
+ /**
+ * Extra check:
+ * <li>1. check {@code filterData} belong to this bloom filter.</li>
+ * <p>
+ * Then set the related {@code bits} positions to 1.
+ * </p>
+ *
+ * @param filterData
+ * @param bits
+ */
+ public void hashTo(BloomFilterData filterData, BitsArray bits) {
+ if (!isValid(filterData)) {
+ throw new IllegalArgumentException(
+ String.format("Bloom filter data may not belong to this filter! %s, %s",
+ filterData, this.toString())
+ );
+ }
+ hashTo(filterData.getBitPos(), bits);
+ }
+
+ /**
+ * Calculate bit positions of {@code str}, then check all the related {@code bits} positions is 1.
+ *
+ * @param str
+ * @param bits
+ * @return true: all the related {@code bits} positions is 1
+ */
+ public boolean isHit(String str, BitsArray bits) {
+ return isHit(calcBitPositions(str), bits);
+ }
+
+ /**
+ * Check all the related {@code bits} positions is 1.
+ *
+ * @param bitPositions
+ * @param bits
+ * @return true: all the related {@code bits} positions is 1
+ */
+ public boolean isHit(int[] bitPositions, BitsArray bits) {
+ check(bits);
+ boolean ret = bits.getBit(bitPositions[0]);
+ for (int i = 1; i < bitPositions.length; i++) {
+ ret &= bits.getBit(bitPositions[i]);
+ }
+ return ret;
+ }
+
+ /**
+ * Check all the related {@code bits} positions is 1.
+ *
+ * @param filterData
+ * @param bits
+ * @return true: all the related {@code bits} positions is 1
+ */
+ public boolean isHit(BloomFilterData filterData, BitsArray bits) {
+ if (!isValid(filterData)) {
+ throw new IllegalArgumentException(
+ String.format("Bloom filter data may not belong to this filter! %s, %s",
+ filterData, this.toString())
+ );
+ }
+ return isHit(filterData.getBitPos(), bits);
+ }
+
+ /**
+ * Check whether one of {@code bitPositions} has been occupied.
+ *
+ * @param bitPositions
+ * @param bits
+ * @return true: if all positions have been occupied.
+ */
+ public boolean checkFalseHit(int[] bitPositions, BitsArray bits) {
+ for (int j = 0; j < bitPositions.length; j++) {
+ int pos = bitPositions[j];
+
+ // check position of bits has been set.
+ // that mean no one occupy the position.
+ if (!bits.getBit(pos)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ protected void check(BitsArray bits) {
+ if (bits.bitLength() != this.m) {
+ throw new IllegalArgumentException(
+ String.format("Length(%d) of bits in BitsArray is not equal to %d!", bits.bitLength(), this.m)
+ );
+ }
+ }
+
+ /**
+ * Check {@code BloomFilterData} is valid, and belong to this bloom filter.
+ * <li>1. not null</li>
+ * <li>2. {@link org.apache.rocketmq.filter.util.BloomFilterData#getBitNum} must be equal to {@code m} </li>
+ * <li>3. {@link org.apache.rocketmq.filter.util.BloomFilterData#getBitPos} is not null</li>
+ * <li>4. {@link org.apache.rocketmq.filter.util.BloomFilterData#getBitPos}'s length is equal to {@code k}</li>
+ *
+ * @param filterData
+ * @return
+ */
+ public boolean isValid(BloomFilterData filterData) {
+ if (filterData == null
+ || filterData.getBitNum() != this.m
+ || filterData.getBitPos() == null
+ || filterData.getBitPos().length != this.k) {
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * error rate.
+ *
+ * @return
+ */
+ public int getF() {
+ return f;
+ }
+
+ /**
+ * expect mapping num.
+ *
+ * @return
+ */
+ public int getN() {
+ return n;
+ }
+
+ /**
+ * hash function num.
+ *
+ * @return
+ */
+ public int getK() {
+ return k;
+ }
+
+ /**
+ * total bit num.
+ *
+ * @return
+ */
+ public int getM() {
+ return m;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (!(o instanceof BloomFilter))
+ return false;
+
+ BloomFilter that = (BloomFilter) o;
+
+ if (f != that.f)
+ return false;
+ if (k != that.k)
+ return false;
+ if (m != that.m)
+ return false;
+ if (n != that.n)
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = f;
+ result = 31 * result + n;
+ result = 31 * result + k;
+ result = 31 * result + m;
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("f: %d, n: %d, k: %d, m: %d", f, n, k, m);
+ }
+
+ protected double logMN(double m, double n) {
+ return Math.log(n) / Math.log(m);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilterData.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilterData.java b/filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilterData.java
new file mode 100644
index 0000000..de02d92
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilterData.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.util;
+
+import java.util.Arrays;
+
+/**
+ * Data generated by bloom filter, include:
+ * <li>1. Bit positions allocated to requester;</li>
+ * <li>2. Total bit num when allocating;</li>
+ */
+public class BloomFilterData {
+
+ private int[] bitPos;
+ private int bitNum;
+
+ public BloomFilterData() {
+ }
+
+ public BloomFilterData(int[] bitPos, int bitNum) {
+ this.bitPos = bitPos;
+ this.bitNum = bitNum;
+ }
+
+ public int[] getBitPos() {
+ return bitPos;
+ }
+
+ public int getBitNum() {
+ return bitNum;
+ }
+
+ public void setBitPos(final int[] bitPos) {
+ this.bitPos = bitPos;
+ }
+
+ public void setBitNum(final int bitNum) {
+ this.bitNum = bitNum;
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (!(o instanceof BloomFilterData)) return false;
+
+ final BloomFilterData that = (BloomFilterData) o;
+
+ if (bitNum != that.bitNum) return false;
+ if (!Arrays.equals(bitPos, that.bitPos)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = bitPos != null ? Arrays.hashCode(bitPos) : 0;
+ result = 31 * result + bitNum;
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "BloomFilterData{" +
+ "bitPos=" + Arrays.toString(bitPos) +
+ ", bitNum=" + bitNum +
+ '}';
+ }
+}
[25/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-39] avoid
duplicated codes closes apache/incubator-rocketmq#34
Posted by do...@apache.org.
[ROCKETMQ-39] avoid duplicated codes closes apache/incubator-rocketmq#34
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/c0fe02e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/c0fe02e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/c0fe02e1
Branch: refs/heads/release-4.1.0-incubating
Commit: c0fe02e1c6a0afd516c1323c8b5808fd7ddad72e
Parents: 4d12d11
Author: wu-sheng <wu...@foxmail.com>
Authored: Thu May 25 19:43:21 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Thu May 25 19:43:21 2017 +0800
----------------------------------------------------------------------
.../rocketmq/filtersrv/FiltersrvStartup.java | 24 ++-----
.../apache/rocketmq/namesrv/NamesrvStartup.java | 31 +++------
.../rocketmq/srvutil/ShutdownHookThread.java | 69 ++++++++++++++++++++
3 files changed, 85 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/c0fe02e1/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java
----------------------------------------------------------------------
diff --git a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java
index 5dd15dd..41169c9 100644
--- a/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java
+++ b/filtersrv/src/main/java/org/apache/rocketmq/filtersrv/FiltersrvStartup.java
@@ -22,7 +22,7 @@ import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.Callable;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
@@ -34,6 +34,7 @@ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.netty.NettySystemConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.srvutil.ShutdownHookThread;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -135,24 +136,13 @@ public class FiltersrvStartup {
System.exit(-3);
}
- Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
- private volatile boolean hasShutdown = false;
- private AtomicInteger shutdownTimes = new AtomicInteger(0);
-
+ Runtime.getRuntime().addShutdownHook(new ShutdownHookThread(log, new Callable<Void>() {
@Override
- public void run() {
- synchronized (this) {
- log.info("shutdown hook was invoked, " + this.shutdownTimes.incrementAndGet());
- if (!this.hasShutdown) {
- this.hasShutdown = true;
- long begineTime = System.currentTimeMillis();
- controller.shutdown();
- long consumingTimeTotal = System.currentTimeMillis() - begineTime;
- log.info("shutdown hook over, consuming time total(ms): " + consumingTimeTotal);
- }
- }
+ public Void call() throws Exception {
+ controller.shutdown();
+ return null;
}
- }, "ShutdownHook"));
+ }));
return controller;
} catch (Throwable e) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/c0fe02e1/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java
----------------------------------------------------------------------
diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java
index 4fa97ad..f49d2b3 100644
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java
@@ -22,7 +22,7 @@ import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.Callable;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
@@ -35,6 +35,7 @@ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.netty.NettySystemConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.srvutil.ShutdownHookThread;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,9 +62,7 @@ public class NamesrvStartup {
//PackageConflictDetect.detectFastjson();
Options options = ServerUtil.buildCommandlineOptions(new Options());
- commandLine =
- ServerUtil.parseCmdLine("mqnamesrv", args, buildCommandlineOptions(options),
- new PosixParser());
+ commandLine = ServerUtil.parseCmdLine("mqnamesrv", args, buildCommandlineOptions(options), new PosixParser());
if (null == commandLine) {
System.exit(-1);
return null;
@@ -97,8 +96,7 @@ public class NamesrvStartup {
MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), namesrvConfig);
if (null == namesrvConfig.getRocketmqHome()) {
- System.out.printf("Please set the " + MixAll.ROCKETMQ_HOME_ENV
- + " variable in your environment to match the location of the RocketMQ installation%n");
+ System.out.printf("Please set the " + MixAll.ROCKETMQ_HOME_ENV + " variable in your environment to match the location of the RocketMQ installation%n");
System.exit(-2);
}
@@ -123,24 +121,13 @@ public class NamesrvStartup {
System.exit(-3);
}
- Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
- private volatile boolean hasShutdown = false;
- private AtomicInteger shutdownTimes = new AtomicInteger(0);
-
+ Runtime.getRuntime().addShutdownHook(new ShutdownHookThread(log, new Callable<Void>() {
@Override
- public void run() {
- synchronized (this) {
- log.info("shutdown hook was invoked, " + this.shutdownTimes.incrementAndGet());
- if (!this.hasShutdown) {
- this.hasShutdown = true;
- long begineTime = System.currentTimeMillis();
- controller.shutdown();
- long consumingTimeTotal = System.currentTimeMillis() - begineTime;
- log.info("shutdown hook over, consuming time total(ms): " + consumingTimeTotal);
- }
- }
+ public Void call() throws Exception {
+ controller.shutdown();
+ return null;
}
- }, "ShutdownHook"));
+ }));
controller.start();
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/c0fe02e1/srvutil/src/main/java/org/apache/rocketmq/srvutil/ShutdownHookThread.java
----------------------------------------------------------------------
diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ShutdownHookThread.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ShutdownHookThread.java
new file mode 100644
index 0000000..11f9b2c
--- /dev/null
+++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ShutdownHookThread.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.srvutil;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.slf4j.Logger;
+
+/**
+ * {@link ShutdownHookThread} is the standard hook for filtersrv and namesrv modules.
+ * Through {@link Callable} interface, this hook can customization operations in anywhere.
+ */
+public class ShutdownHookThread extends Thread {
+ private volatile boolean hasShutdown = false;
+ private AtomicInteger shutdownTimes = new AtomicInteger(0);
+ private final Logger log;
+ private final Callable callback;
+
+ /**
+ * Create the standard hook thread, with a call back, by using {@link Callable} interface.
+ *
+ * @param log The log instance is used in hook thread.
+ * @param callback The call back function.
+ */
+ public ShutdownHookThread(Logger log, Callable callback) {
+ super("ShutdownHook");
+ this.log = log;
+ this.callback = callback;
+ }
+
+ /**
+ * Thread run method.
+ * Invoke when the jvm shutdown.
+ * 1. count the invocation times.
+ * 2. execute the {@link ShutdownHookThread#callback}, and time it.
+ */
+ @Override
+ public void run() {
+ synchronized (this) {
+ log.info("shutdown hook was invoked, " + this.shutdownTimes.incrementAndGet() + " times.");
+ if (!this.hasShutdown) {
+ this.hasShutdown = true;
+ long beginTime = System.currentTimeMillis();
+ try {
+ this.callback.call();
+ } catch (Exception e) {
+ log.error("shutdown hook callback invoked failure.", e);
+ }
+ long consumingTimeTotal = System.currentTimeMillis() - beginTime;
+ log.info("shutdown hook done, consuming time total(ms): " + consumingTimeTotal);
+ }
+ }
+ }
+}
[49/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-218] Polish
README.md, closes apache/incubator-rocketmq#113
Posted by do...@apache.org.
[ROCKETMQ-218] Polish README.md, closes apache/incubator-rocketmq#113
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/c4a3e0c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/c4a3e0c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/c4a3e0c1
Branch: refs/heads/release-4.1.0-incubating
Commit: c4a3e0c1325e67aec76aa94e9ade3a6ea1b682d2
Parents: f45a1bc
Author: zhoudiqiu <zh...@wustl.edu>
Authored: Thu Jun 8 11:29:33 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Thu Jun 8 11:29:33 2017 +0800
----------------------------------------------------------------------
README.md | 32 +++++++++++++++++++-------------
1 file changed, 19 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/c4a3e0c1/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 7a9abb1..ab0a865 100644
--- a/README.md
+++ b/README.md
@@ -3,18 +3,25 @@
[](https://github.org/apache/rocketmqreleases)
[](https://www.apache.org/licenses/LICENSE-2.0.html)
-**[Apache RocketMQ](https://rocketmq.incubator.apache.org) is a low latency, reliable, scalable, easy to use message oriented middleware born from alibaba massive messaging business.**
+**[Apache RocketMQ](https://rocketmq.incubator.apache.org) is a distributed messaging and streaming platform with low latency, high performance and reliability, trillion-level capacity and flexible scalability.**
+
+It offers a variety of features:
+
+* Pub/Sub messaging model
+* Scheduled message delivery
+* Message retroactivity by time or offset
+* Log hub for streaming
+* Big data integration
+* Reliable FIFO and strict ordered messaging in the same queue
+* Efficient pull&push consumption model
+* Million-level message accumulation capacity in a single queue
+* Multiple messaging protocols like JMS and OpenMessaging
+* Flexible distributed scale-out deployment architecture
+* Lightning-fast batch message exchange system
+* Various message filter mechanics such as SQL and Tag
+* Docker images for isolated testing and cloud isolated clusters
+* Feature-rich administrative dashboard for configuration, metrics and monitoring
-It offers a variety of features as follows:
-
-* Pub/Sub and P2P messaging model
-* Reliable FIFO and strict sequential messaging in the same queue
-* Long pull queue model,also support push consumption style
-* Million message accumulation ability in single queue
-* Over a variety of messaging protocols.such as JMS,MQTT etc.
-* Distributed high available deploy architecture, meets at least once message delivery semantics
-* Docker images for isolated testing and cloud Isolated clusters
-* Feature-rich administrative dashboard for configuration,metrics and monitoring
----------
@@ -29,13 +36,12 @@ It offers a variety of features as follows:
----------
## Apache RocketMQ Community
-* [RocketMQ Community Incubator Projects](https://github.com/rocketmq)
* [RocketMQ Community Projects](https://github.com/apache/incubator-rocketmq-externals)
----------
## Contributing
-We are always very happy to have contributions, whether for trivial cleanups,big new features or other material rewards. more details see [here](CONTRIBUTING.md)
+We always welcome new contributions, whether for trivial cleanups, big new features or other material rewards. more details see [here](CONTRIBUTING.md)
----------
## License
[20/50] [abbrv] incubator-rocketmq git commit: This is to close issue
ROCKETMQ-198: Add language code for Go and PHP Thanks to StyleTang
Posted by do...@apache.org.
This is to close issue ROCKETMQ-198: Add language code for Go and PHP
Thanks to StyleTang
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/1630f277
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/1630f277
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/1630f277
Branch: refs/heads/release-4.1.0-incubating
Commit: 1630f277b9c9f8e85c9bdb09a323d39e13e797e4
Parents: 80aac13
Author: Li Zhanhui <li...@apache.org>
Authored: Tue May 16 14:34:56 2017 +0800
Committer: Li Zhanhui <li...@apache.org>
Committed: Tue May 16 14:34:56 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/rocketmq/remoting/protocol/LanguageCode.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1630f277/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java
----------------------------------------------------------------------
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java
index 669a383..17ce919 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java
@@ -26,7 +26,9 @@ public enum LanguageCode {
ERLANG((byte) 5),
RUBY((byte) 6),
OTHER((byte) 7),
- HTTP((byte) 8);
+ HTTP((byte) 8),
+ GO((byte) 9),
+ PHP((byte) 10);
private byte code;
[24/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-161] Update
runbroker.sh and runserver.sh to support user defined jvm mem flag closes
apache/incubator-rocketmq#87
Posted by do...@apache.org.
[ROCKETMQ-161] Update runbroker.sh and runserver.sh to support user defined jvm mem flag closes apache/incubator-rocketmq#87
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/4d12d112
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/4d12d112
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/4d12d112
Branch: refs/heads/release-4.1.0-incubating
Commit: 4d12d11249950468d2830cae31c0f8909ad14395
Parents: 1d966b5
Author: dongeforever <zh...@yeah.net>
Authored: Thu May 25 14:26:37 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Thu May 25 14:26:37 2017 +0800
----------------------------------------------------------------------
distribution/bin/runbroker.sh | 1 +
distribution/bin/runserver.sh | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/4d12d112/distribution/bin/runbroker.sh
----------------------------------------------------------------------
diff --git a/distribution/bin/runbroker.sh b/distribution/bin/runbroker.sh
index 3405d39..b0c490e 100644
--- a/distribution/bin/runbroker.sh
+++ b/distribution/bin/runbroker.sh
@@ -46,6 +46,7 @@ JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=15g"
JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking"
JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${BASE_DIR}/lib"
#JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n"
+JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}"
JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}"
numactl --interleave=all pwd > /dev/null 2>&1
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/4d12d112/distribution/bin/runserver.sh
----------------------------------------------------------------------
diff --git a/distribution/bin/runserver.sh b/distribution/bin/runserver.sh
index e85991c..7c41b0c 100644
--- a/distribution/bin/runserver.sh
+++ b/distribution/bin/runserver.sh
@@ -43,6 +43,7 @@ JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow"
JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages"
JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${BASE_DIR}/lib"
#JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n"
+JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}"
JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}"
$JAVA ${JAVA_OPT} $@
[50/50] [abbrv] incubator-rocketmq git commit: Add license for
OpenMessaging
Posted by do...@apache.org.
Add license for OpenMessaging
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/10933cc0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/10933cc0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/10933cc0
Branch: refs/heads/release-4.1.0-incubating
Commit: 10933cc0a9be43c418b5b9c0b78cd9e07674b099
Parents: c4a3e0c
Author: dongeforever <do...@apache.org>
Authored: Thu Jun 8 15:47:40 2017 +0800
Committer: dongeforever <do...@apache.org>
Committed: Thu Jun 8 15:47:40 2017 +0800
----------------------------------------------------------------------
distribution/LICENSE-BIN | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/10933cc0/distribution/LICENSE-BIN
----------------------------------------------------------------------
diff --git a/distribution/LICENSE-BIN b/distribution/LICENSE-BIN
index 32d0208..3726172 100644
--- a/distribution/LICENSE-BIN
+++ b/distribution/LICENSE-BIN
@@ -314,4 +314,21 @@ The source code of guava can be found at https://github.com/google/guava.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+------
+This product has a bundle OpenMessaging, which is available under the ASL2 License.
+The source code of OpenMessaging can be found at https://github.com/openmessaging/openmessaging.
+
+ Copyright (C) 2017 The OpenMessaging authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
[45/50] [abbrv] incubator-rocketmq git commit: Include guava
copyright announcement
Posted by do...@apache.org.
Include guava copyright announcement
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/2c28baad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/2c28baad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/2c28baad
Branch: refs/heads/release-4.1.0-incubating
Commit: 2c28baad8f6cb5f8273e8ec03ac45e453dd1f25a
Parents: 3d1ec32
Author: dongeforever <zh...@yeah.net>
Authored: Tue Jun 6 15:13:43 2017 +0800
Committer: dongeforever <do...@apache.org>
Committed: Tue Jun 6 16:28:15 2017 +0800
----------------------------------------------------------------------
distribution/LICENSE-BIN | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/2c28baad/distribution/LICENSE-BIN
----------------------------------------------------------------------
diff --git a/distribution/LICENSE-BIN b/distribution/LICENSE-BIN
index 9796b3c..32d0208 100644
--- a/distribution/LICENSE-BIN
+++ b/distribution/LICENSE-BIN
@@ -296,4 +296,22 @@ The source code of jna can be found at https://github.com/java-native-access/jna
A copy is also included in the downloadable source code package
containing JNA, in file "AL2.0", under the same directory
- as this file.
\ No newline at end of file
+ as this file.
+------
+This product has a bundle guava, which is available under the ASL2 License.
+The source code of guava can be found at https://github.com/google/guava.
+
+ Copyright (C) 2007 The Guava authors
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
[47/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-220] Add IT
test for Filter By SQL 92, closes apache/incubator-rocketmq#114
Posted by do...@apache.org.
[ROCKETMQ-220] Add IT test for Filter By SQL 92, closes
apache/incubator-rocketmq#114
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/703ac00b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/703ac00b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/703ac00b
Branch: refs/heads/release-4.1.0-incubating
Commit: 703ac00bd271583c0290daad14e45f97f9991f49
Parents: 7374914
Author: dongeforever <zh...@yeah.net>
Authored: Thu Jun 8 11:12:26 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Thu Jun 8 11:20:15 2017 +0800
----------------------------------------------------------------------
.../rocketmq/broker/BrokerController.java | 2 +-
.../rocketmq/example/filter/SqlConsumer.java | 1 -
.../test/client/rmq/RMQSqlConsumer.java | 42 +++++++++++
.../rocketmq/test/factory/ConsumerFactory.java | 12 ++++
.../test/listener/AbstractListener.java | 22 ++++++
.../rocketmq/test/base/IntegrationTestBase.java | 1 +
.../client/consumer/filter/SqlFilterIT.java | 74 ++++++++++++++++++++
7 files changed, 152 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/703ac00b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
index bacd25c..1416ebf 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
@@ -421,7 +421,7 @@ public class BrokerController {
this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.clientManageExecutor);
this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor);
- this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor);
+ this.fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor);
/**
* ConsumerManageProcessor
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/703ac00b/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java b/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java
index 9a3b813..52c2474 100644
--- a/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java
@@ -31,7 +31,6 @@ public class SqlConsumer {
public static void main(String[] args) {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_4");
-
try {
consumer.subscribe("TopicTest",
MessageSelector.bySql("(TAGS is not null and TAGS in ('TagA', 'TagB'))" +
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/703ac00b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQSqlConsumer.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQSqlConsumer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQSqlConsumer.java
new file mode 100644
index 0000000..cb0210f
--- /dev/null
+++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQSqlConsumer.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.test.client.rmq;
+
+import org.apache.log4j.Logger;
+import org.apache.rocketmq.client.consumer.MessageSelector;
+import org.apache.rocketmq.test.listener.AbstractListener;
+
+public class RMQSqlConsumer extends RMQNormalConsumer {
+ private static Logger logger = Logger.getLogger(RMQSqlConsumer.class);
+ private MessageSelector selector;
+ public RMQSqlConsumer(String nsAddr, String topic, MessageSelector selector,
+ String consumerGroup, AbstractListener listener) {
+ super(nsAddr, topic, "*", consumerGroup, listener);
+ this.selector = selector;
+ }
+
+ @Override
+ public void create() {
+ super.create();
+ try {
+ consumer.subscribe(topic, selector);
+ } catch (Exception e) {
+ logger.error("Subscribe Sql Errored", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/703ac00b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java
index b5b3fdd..7dd747f 100644
--- a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java
+++ b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java
@@ -17,8 +17,10 @@
package org.apache.rocketmq.test.factory;
+import org.apache.rocketmq.client.consumer.MessageSelector;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;
+import org.apache.rocketmq.test.client.rmq.RMQSqlConsumer;
import org.apache.rocketmq.test.listener.AbstractListener;
public class ConsumerFactory {
@@ -42,4 +44,14 @@ public class ConsumerFactory {
consumer.start();
return consumer;
}
+
+ public static RMQSqlConsumer getRMQSqlConsumer(String nsAddr, String consumerGroup,
+ String topic, MessageSelector selector,
+ AbstractListener listner) {
+ RMQSqlConsumer consumer = new RMQSqlConsumer(nsAddr, topic, selector,
+ consumerGroup, listner);
+ consumer.create();
+ consumer.start();
+ return consumer;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/703ac00b/test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java b/test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
index 974434a..14da397 100644
--- a/test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
+++ b/test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
@@ -95,6 +95,28 @@ public class AbstractListener extends MQCollector implements MessageListener {
return sendMsgs;
}
+ public long waitForMessageConsume(int size,
+ int timeoutMills) {
+
+ long curTime = System.currentTimeMillis();
+ while (true) {
+ if (msgBodys.getDataSize() >= size) {
+ break;
+ }
+ if (System.currentTimeMillis() - curTime >= timeoutMills) {
+ logger.error(String.format("timeout but [%s] not recv all send messages!",
+ listnerName));
+ break;
+ } else {
+ logger.info(String.format("[%s] still [%s] msg not recv!", listnerName,
+ size - msgBodys.getDataSize()));
+ TestUtil.waitForMonment(500);
+ }
+ }
+
+ return msgBodys.getDataSize();
+ }
+
public void waitForMessageConsume(Map<Object, Object> sendMsgIndex, int timeoutMills) {
Collection<Object> notRecvMsgs = waitForMessageConsume(sendMsgIndex.keySet(), timeoutMills);
for (Object object : notRecvMsgs) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/703ac00b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
index 9805eba..07af7aa 100644
--- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
+++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
@@ -127,6 +127,7 @@ public class IntegrationTestBase {
brokerConfig.setBrokerName(BROKER_NAME_PREFIX + BROKER_INDEX.getAndIncrement());
brokerConfig.setBrokerIP1("127.0.0.1");
brokerConfig.setNamesrvAddr(nsAddr);
+ brokerConfig.setEnablePropertyFilter(true);
storeConfig.setStorePathRootDir(baseDir);
storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog");
storeConfig.setHaListenPort(8000 + random.nextInt(1000));
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/703ac00b/test/src/test/java/org/apache/rocketmq/test/client/consumer/filter/SqlFilterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/filter/SqlFilterIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/filter/SqlFilterIT.java
new file mode 100644
index 0000000..7eef2ab
--- /dev/null
+++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/filter/SqlFilterIT.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.test.client.consumer.filter;
+
+import org.apache.log4j.Logger;
+import org.apache.rocketmq.client.consumer.MessageSelector;
+import org.apache.rocketmq.test.base.BaseConf;
+import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
+import org.apache.rocketmq.test.client.consumer.broadcast.normal.NormalMsgTwoSameGroupConsumerIT;
+import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
+import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
+import org.apache.rocketmq.test.client.rmq.RMQSqlConsumer;
+import org.apache.rocketmq.test.factory.ConsumerFactory;
+import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListner;
+import org.apache.rocketmq.test.util.VerifyUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static com.google.common.truth.Truth.assertThat;
+
+public class SqlFilterIT extends BaseConf {
+ private static Logger logger = Logger.getLogger(SqlFilterIT.class);
+ private RMQNormalProducer producer = null;
+ private String topic = null;
+
+ @Before
+ public void setUp() {
+ topic = initTopic();
+ logger.info(String.format("use topic: %s;", topic));
+ producer = getProducer(nsAddr, topic);
+ }
+
+ @After
+ public void tearDown() {
+ super.shutDown();
+ }
+
+ @Test
+ public void testFilterConsumer() throws Exception {
+ int msgSize = 16;
+
+ String group = initConsumerGroup();
+ MessageSelector selector = MessageSelector.bySql("(TAGS is not null and TAGS in ('TagA', 'TagB'))");
+ RMQSqlConsumer consumer = ConsumerFactory.getRMQSqlConsumer(nsAddr, group, topic, selector, new RMQNormalListner(group + "_1"));
+ Thread.sleep(3000);
+ producer.send("TagA", msgSize);
+ producer.send("TagB", msgSize);
+ producer.send("TagC", msgSize);
+ Assert.assertEquals("Not all sent succeeded", msgSize * 3, producer.getAllUndupMsgBody().size());
+ consumer.getListner().waitForMessageConsume(msgSize * 2, consumeTime);
+ assertThat(producer.getAllMsgBody())
+ .containsAllIn(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),
+ consumer.getListner().getAllMsgBody()));
+
+ assertThat(consumer.getListner().getAllMsgBody().size()).isEqualTo(msgSize * 2);
+ }
+}
[26/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-178] Fix -p
-m options closes apache/incubator-rocketmq#93
Posted by do...@apache.org.
[ROCKETMQ-178] Fix -p -m options closes apache/incubator-rocketmq#93
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/42826c41
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/42826c41
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/42826c41
Branch: refs/heads/release-4.1.0-incubating
Commit: 42826c4115d91ee48512df288b6fe3822af73f06
Parents: c0fe02e
Author: Li Zhanhui <li...@apache.org>
Authored: Fri May 26 14:42:23 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Fri May 26 14:42:23 2017 +0800
----------------------------------------------------------------------
.../apache/rocketmq/broker/BrokerStartup.java | 32 +++++++++++---------
.../rocketmq/common/constant/LoggerName.java | 1 +
distribution/conf/logback_broker.xml | 5 +++
3 files changed, 23 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/42826c41/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java
index 98ff136..85d2e3a 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java
@@ -103,20 +103,6 @@ public class BrokerStartup {
messageStoreConfig.setAccessMessageInMemoryMaxRatio(ratio);
}
- if (commandLine.hasOption('p')) {
- MixAll.printObjectProperties(null, brokerConfig);
- MixAll.printObjectProperties(null, nettyServerConfig);
- MixAll.printObjectProperties(null, nettyClientConfig);
- MixAll.printObjectProperties(null, messageStoreConfig);
- System.exit(0);
- } else if (commandLine.hasOption('m')) {
- MixAll.printObjectProperties(null, brokerConfig, true);
- MixAll.printObjectProperties(null, nettyServerConfig, true);
- MixAll.printObjectProperties(null, nettyClientConfig, true);
- MixAll.printObjectProperties(null, messageStoreConfig, true);
- System.exit(0);
- }
-
if (commandLine.hasOption('c')) {
String file = commandLine.getOptionValue('c');
if (file != null) {
@@ -181,8 +167,24 @@ public class BrokerStartup {
configurator.setContext(lc);
lc.reset();
configurator.doConfigure(brokerConfig.getRocketmqHome() + "/conf/logback_broker.xml");
- log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+ if (commandLine.hasOption('p')) {
+ Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME);
+ MixAll.printObjectProperties(console, brokerConfig);
+ MixAll.printObjectProperties(console, nettyServerConfig);
+ MixAll.printObjectProperties(console, nettyClientConfig);
+ MixAll.printObjectProperties(console, messageStoreConfig);
+ System.exit(0);
+ } else if (commandLine.hasOption('m')) {
+ Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME);
+ MixAll.printObjectProperties(console, brokerConfig, true);
+ MixAll.printObjectProperties(console, nettyServerConfig, true);
+ MixAll.printObjectProperties(console, nettyClientConfig, true);
+ MixAll.printObjectProperties(console, messageStoreConfig, true);
+ System.exit(0);
+ }
+
+ log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
MixAll.printObjectProperties(log, brokerConfig);
MixAll.printObjectProperties(log, nettyServerConfig);
MixAll.printObjectProperties(log, nettyClientConfig);
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/42826c41/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
index 385c121..12070dd 100644
--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
+++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
@@ -20,6 +20,7 @@ public class LoggerName {
public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv";
public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv";
public static final String BROKER_LOGGER_NAME = "RocketmqBroker";
+ public static final String BROKER_CONSOLE_NAME = "RocketmqConsole";
public static final String CLIENT_LOGGER_NAME = "RocketmqClient";
public static final String TOOLS_LOGGER_NAME = "RocketmqTools";
public static final String COMMON_LOGGER_NAME = "RocketmqCommon";
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/42826c41/distribution/conf/logback_broker.xml
----------------------------------------------------------------------
diff --git a/distribution/conf/logback_broker.xml b/distribution/conf/logback_broker.xml
index dd5c63f..3945fac 100644
--- a/distribution/conf/logback_broker.xml
+++ b/distribution/conf/logback_broker.xml
@@ -349,6 +349,11 @@
<appender-ref ref="RocketmqFilterAppender"/>
</logger>
+ <logger name="RocketmqConsole" additivity="false">
+ <level value="INFO" />
+ <appender-ref ref="STDOUT" />
+ </logger>
+
<root>
<level value="INFO"/>
<appender-ref ref="DefaultAppender"/>
[42/50] [abbrv] incubator-rocketmq git commit: Fix error tests,
producer should wait a while for consumer to be ready
Posted by do...@apache.org.
Fix error tests, producer should wait a while for consumer to be ready
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/3d1ec328
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/3d1ec328
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/3d1ec328
Branch: refs/heads/release-4.1.0-incubating
Commit: 3d1ec3289d96bc8464e507b6a446506ca26cf7e0
Parents: 0fe9471
Author: dongeforever <zh...@yeah.net>
Authored: Tue Jun 6 10:44:55 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Tue Jun 6 10:44:55 2017 +0800
----------------------------------------------------------------------
.../consumer/broadcast/normal/BroadCastNormalMsgNotRecvIT.java | 4 ++--
.../producer/async/AsyncSendWithMessageQueueSelectorIT.java | 1 +
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/3d1ec328/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgNotRecvIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgNotRecvIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgNotRecvIT.java
index 65762fa..32b13fd 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgNotRecvIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgNotRecvIT.java
@@ -50,13 +50,13 @@ public class BroadCastNormalMsgNotRecvIT extends BaseBroadCastIT {
}
@Test
- public void testNotConsumeAfterConsume() {
+ public void testNotConsumeAfterConsume() throws Exception {
int msgSize = 16;
String group = initConsumerGroup();
RMQBroadCastConsumer consumer1 = getBroadCastConsumer(nsAddr, group, topic, "*",
new RMQNormalListner(group + "_1"));
-
+ Thread.sleep(3000);
producer.send(msgSize);
Assert.assertEquals("Not all sent succeeded", msgSize, producer.getAllUndupMsgBody().size());
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/3d1ec328/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java
index 843441d..82012ea 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java
@@ -80,6 +80,7 @@ public class AsyncSendWithMessageQueueSelectorIT extends BaseConf {
producer.clearMsg();
consumer.clearMsg();
+ producer.getSuccessSendResult().clear();
producer.asyncSend(msgSize, new MessageQueueSelector() {
@Override
[11/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-121]Support
message filtering based on SQL92 closes apache/incubator-rocketmq#82
Posted by do...@apache.org.
[ROCKETMQ-121]Support message filtering based on SQL92 closes apache/incubator-rocketmq#82
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/58f1574b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/58f1574b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/58f1574b
Branch: refs/heads/release-4.1.0-incubating
Commit: 58f1574b28bf8bf18a795036545c7a700437ed0b
Parents: 42f78c2
Author: vsair <li...@gmail.com>
Authored: Fri Apr 21 18:17:58 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Fri Apr 21 18:17:58 2017 +0800
----------------------------------------------------------------------
broker/pom.xml | 4 +
.../rocketmq/broker/BrokerController.java | 27 +
.../rocketmq/broker/BrokerPathConfigHelper.java | 3 +
.../broker/client/ConsumerGroupEvent.java | 33 +
.../client/ConsumerIdsChangeListener.java | 6 +-
.../rocketmq/broker/client/ConsumerManager.java | 11 +-
.../DefaultConsumerIdsChangeListener.java | 37 +-
.../filter/CommitLogDispatcherCalcBitMap.java | 110 ++
.../broker/filter/ConsumerFilterData.java | 151 ++
.../broker/filter/ConsumerFilterManager.java | 471 ++++++
.../filter/ExpressionForRetryMessageFilter.java | 97 ++
.../broker/filter/ExpressionMessageFilter.java | 162 +++
.../broker/filter/MessageEvaluationContext.java | 58 +
.../NotifyMessageArrivingListener.java | 8 +-
.../broker/longpolling/PullRequest.java | 10 +-
.../longpolling/PullRequestHoldService.java | 19 +-
.../rocketmq/broker/out/BrokerOuterAPI.java | 2 +-
.../plugin/AbstractPluginMessageStore.java | 18 +-
.../broker/processor/AdminBrokerProcessor.java | 91 ++
.../broker/processor/ClientManageProcessor.java | 44 +
.../broker/processor/PullMessageProcessor.java | 59 +-
.../CommitLogDispatcherCalcBitMapTest.java | 192 +++
.../filter/ConsumerFilterManagerTest.java | 291 ++++
.../filter/MessageStoreWithFilterTest.java | 392 +++++
.../processor/PullMessageProcessorTest.java | 9 +-
.../client/consumer/DefaultMQPushConsumer.java | 15 +
.../client/consumer/MQPushConsumer.java | 21 +
.../client/consumer/MessageSelector.java | 77 +
.../rocketmq/client/impl/FindBrokerResult.java | 12 +
.../rocketmq/client/impl/MQClientAPIImpl.java | 57 +-
.../consumer/DefaultMQPushConsumerImpl.java | 40 +-
.../client/impl/consumer/PullAPIWrapper.java | 40 +
.../client/impl/factory/MQClientInstance.java | 60 +-
.../apache/rocketmq/common/BrokerConfig.java | 67 +
.../java/org/apache/rocketmq/common/MixAll.java | 14 +-
.../rocketmq/common/constant/LoggerName.java | 1 +
.../rocketmq/common/filter/ExpressionType.java | 67 +
.../rocketmq/common/filter/FilterAPI.java | 18 +
.../apache/rocketmq/common/message/Message.java | 6 +
.../rocketmq/common/message/MessageDecoder.java | 39 +
.../rocketmq/common/namesrv/TopAddressing.java | 2 +-
.../rocketmq/common/protocol/RequestCode.java | 4 +
.../rocketmq/common/protocol/ResponseCode.java | 4 +
.../protocol/body/CheckClientRequestBody.java | 52 +
.../common/protocol/body/ConsumeQueueData.java | 98 ++
.../body/QueryConsumeQueueResponseBody.java | 72 +
.../header/PullMessageRequestHeader.java | 9 +
.../header/QueryConsumeQueueRequestHeader.java | 75 +
.../protocol/heartbeat/SubscriptionData.java | 17 +-
.../rocketmq/common/filter/FilterAPITest.java | 49 +
.../common/message/MessageDecoderTest.java | 80 ++
distribution/conf/logback_broker.xml | 28 +
distribution/release.xml | 1 +
.../rocketmq/example/benchmark/Consumer.java | 31 +-
.../rocketmq/example/benchmark/Producer.java | 34 +-
.../rocketmq/example/filter/SqlConsumer.java | 62 +
.../rocketmq/example/filter/SqlProducer.java | 67 +
filter/pom.xml | 43 +
.../apache/rocketmq/filter/FilterFactory.java | 72 +
.../org/apache/rocketmq/filter/FilterSpi.java | 43 +
.../org/apache/rocketmq/filter/SqlFilter.java | 43 +
.../rocketmq/filter/constant/UnaryType.java | 26 +
.../filter/expression/BinaryExpression.java | 91 ++
.../filter/expression/BooleanExpression.java | 39 +
.../filter/expression/ComparisonExpression.java | 413 ++++++
.../filter/expression/ConstantExpression.java | 156 ++
.../expression/EmptyEvaluationContext.java | 35 +
.../filter/expression/EvaluationContext.java | 43 +
.../rocketmq/filter/expression/Expression.java | 38 +
.../filter/expression/LogicExpression.java | 94 ++
.../filter/expression/MQFilterException.java | 46 +
.../filter/expression/NowExpression.java | 36 +
.../filter/expression/PropertyExpression.java | 70 +
.../filter/expression/UnaryExpression.java | 267 ++++
.../filter/expression/UnaryInExpression.java | 61 +
.../rocketmq/filter/parser/ParseException.java | 204 +++
.../rocketmq/filter/parser/SelectorParser.java | 1354 ++++++++++++++++++
.../rocketmq/filter/parser/SelectorParser.jj | 524 +++++++
.../filter/parser/SelectorParserConstants.java | 140 ++
.../parser/SelectorParserTokenManager.java | 919 ++++++++++++
.../filter/parser/SimpleCharStream.java | 502 +++++++
.../apache/rocketmq/filter/parser/Token.java | 152 ++
.../rocketmq/filter/parser/TokenMgrError.java | 174 +++
.../apache/rocketmq/filter/util/BitsArray.java | 260 ++++
.../rocketmq/filter/util/BloomFilter.java | 338 +++++
.../rocketmq/filter/util/BloomFilterData.java | 83 ++
.../apache/rocketmq/filter/BitsArrayTest.java | 123 ++
.../apache/rocketmq/filter/BloomFilterTest.java | 172 +++
.../apache/rocketmq/filter/ExpressionTest.java | 594 ++++++++
.../apache/rocketmq/filter/FilterSpiTest.java | 84 ++
.../org/apache/rocketmq/filter/ParserTest.java | 129 ++
pom.xml | 11 +
srvutil/pom.xml | 4 +
.../org/apache/rocketmq/store/CommitLog.java | 8 +-
.../rocketmq/store/CommitLogDispatcher.java | 26 +
.../org/apache/rocketmq/store/ConsumeQueue.java | 122 +-
.../apache/rocketmq/store/ConsumeQueueExt.java | 638 +++++++++
.../rocketmq/store/DefaultMessageFilter.java | 29 +-
.../rocketmq/store/DefaultMessageStore.java | 132 +-
.../apache/rocketmq/store/DispatchRequest.java | 21 +-
.../org/apache/rocketmq/store/MappedFile.java | 25 +
.../apache/rocketmq/store/MappedFileQueue.java | 2 +-
.../rocketmq/store/MessageArrivingListener.java | 5 +-
.../apache/rocketmq/store/MessageFilter.java | 26 +-
.../org/apache/rocketmq/store/MessageStore.java | 8 +-
.../store/config/MessageStoreConfig.java | 31 +
.../store/config/StorePathConfigHelper.java | 4 +
.../store/schedule/ScheduleMessageService.java | 14 +
.../rocketmq/store/ConsumeQueueExtTest.java | 251 ++++
.../apache/rocketmq/store/ConsumeQueueTest.java | 226 +++
.../rocketmq/store/DefaultMessageStoreTest.java | 4 +-
.../rocketmq/tools/admin/DefaultMQAdminExt.java | 9 +
.../tools/admin/DefaultMQAdminExtImpl.java | 8 +
.../apache/rocketmq/tools/admin/MQAdminExt.java | 22 +
.../rocketmq/tools/command/MQAdminStartup.java | 3 +
.../command/queue/QueryConsumeQueueCommand.java | 159 ++
116 files changed, 12552 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/pom.xml
----------------------------------------------------------------------
diff --git a/broker/pom.xml b/broker/pom.xml
index 8cdafea..0f8ad0a 100644
--- a/broker/pom.xml
+++ b/broker/pom.xml
@@ -49,6 +49,10 @@
<artifactId>rocketmq-srvutil</artifactId>
</dependency>
<dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-filter</artifactId>
+ </dependency>
+ <dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
index 6acd40c..bacd25c 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
@@ -37,6 +37,8 @@ import org.apache.rocketmq.broker.client.DefaultConsumerIdsChangeListener;
import org.apache.rocketmq.broker.client.ProducerManager;
import org.apache.rocketmq.broker.client.net.Broker2Client;
import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager;
+import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap;
+import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
import org.apache.rocketmq.broker.filtersrv.FilterServerManager;
import org.apache.rocketmq.broker.latency.BrokerFastFailure;
import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor;
@@ -96,6 +98,7 @@ public class BrokerController {
private final MessageStoreConfig messageStoreConfig;
private final ConsumerOffsetManager consumerOffsetManager;
private final ConsumerManager consumerManager;
+ private final ConsumerFilterManager consumerFilterManager;
private final ProducerManager producerManager;
private final ClientHousekeepingService clientHousekeepingService;
private final PullMessageProcessor pullMessageProcessor;
@@ -149,6 +152,7 @@ public class BrokerController {
this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService);
this.consumerIdsChangeListener = new DefaultConsumerIdsChangeListener(this);
this.consumerManager = new ConsumerManager(this.consumerIdsChangeListener);
+ this.consumerFilterManager = new ConsumerFilterManager(this);
this.producerManager = new ProducerManager();
this.clientHousekeepingService = new ClientHousekeepingService(this);
this.broker2Client = new Broker2Client(this);
@@ -192,6 +196,7 @@ public class BrokerController {
result = result && this.consumerOffsetManager.load();
result = result && this.subscriptionGroupManager.load();
+ result = result && this.consumerFilterManager.load();
if (result) {
try {
@@ -202,6 +207,7 @@ public class BrokerController {
//load plugin
MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig);
this.messageStore = MessageStoreFactory.build(context, this.messageStore);
+ this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager));
} catch (IOException e) {
result = false;
e.printStackTrace();
@@ -278,6 +284,17 @@ public class BrokerController {
@Override
public void run() {
try {
+ BrokerController.this.consumerFilterManager.persist();
+ } catch (Throwable e) {
+ log.error("schedule persist consumer filter error.", e);
+ }
+ }
+ }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);
+
+ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
BrokerController.this.protectBroker();
} catch (Exception e) {
log.error("protectBroker error.", e);
@@ -400,9 +417,11 @@ public class BrokerController {
ClientManageProcessor clientProcessor = new ClientManageProcessor(this);
this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.clientManageExecutor);
this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor);
this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.clientManageExecutor);
this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor);
+ this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor);
/**
* ConsumerManageProcessor
@@ -504,6 +523,10 @@ public class BrokerController {
return consumerManager;
}
+ public ConsumerFilterManager getConsumerFilterManager() {
+ return consumerFilterManager;
+ }
+
public ConsumerOffsetManager getConsumerOffsetManager() {
return consumerOffsetManager;
}
@@ -590,6 +613,10 @@ public class BrokerController {
if (this.brokerFastFailure != null) {
this.brokerFastFailure.shutdown();
}
+
+ if (this.consumerFilterManager != null) {
+ this.consumerFilterManager.persist();
+ }
}
private void unregisterBrokerAll() {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
index 24876df..0a323ee 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
@@ -44,4 +44,7 @@ public class BrokerPathConfigHelper {
return rootDir + File.separator + "config" + File.separator + "subscriptionGroup.json";
}
+ public static String getConsumerFilterPath(final String rootDir) {
+ return rootDir + File.separator + "config" + File.separator + "consumerFilter.json";
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java
new file mode 100644
index 0000000..717fb70
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.client;
+
+public enum ConsumerGroupEvent {
+
+ /**
+ * Some consumers in the group are changed.
+ */
+ CHANGE,
+ /**
+ * The group of consumer is unregistered.
+ */
+ UNREGISTER,
+ /**
+ * The group of consumer is registered.
+ */
+ REGISTER
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerIdsChangeListener.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerIdsChangeListener.java
index 07d28dc..831e293 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerIdsChangeListener.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerIdsChangeListener.java
@@ -16,9 +16,7 @@
*/
package org.apache.rocketmq.broker.client;
-import io.netty.channel.Channel;
-import java.util.List;
-
public interface ConsumerIdsChangeListener {
- void consumerIdsChanged(final String group, final List<Channel> channels);
+
+ void handle(ConsumerGroupEvent event, String group, Object... args);
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
index a2d88d5..a5ddec8 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
@@ -85,10 +85,11 @@ public class ConsumerManager {
if (remove != null) {
log.info("unregister consumer ok, no any connection, and remove consumer group, {}",
next.getKey());
+ this.consumerIdsChangeListener.handle(ConsumerGroupEvent.UNREGISTER, next.getKey());
}
}
- this.consumerIdsChangeListener.consumerIdsChanged(next.getKey(), info.getAllChannel());
+ this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel());
}
}
}
@@ -111,10 +112,12 @@ public class ConsumerManager {
if (r1 || r2) {
if (isNotifyConsumerIdsChangedEnable) {
- this.consumerIdsChangeListener.consumerIdsChanged(group, consumerGroupInfo.getAllChannel());
+ this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
}
}
+ this.consumerIdsChangeListener.handle(ConsumerGroupEvent.REGISTER, group, subList);
+
return r1 || r2;
}
@@ -126,10 +129,12 @@ public class ConsumerManager {
ConsumerGroupInfo remove = this.consumerTable.remove(group);
if (remove != null) {
log.info("unregister consumer ok, no any connection, and remove consumer group, {}", group);
+
+ this.consumerIdsChangeListener.handle(ConsumerGroupEvent.UNREGISTER, group);
}
}
if (isNotifyConsumerIdsChangedEnable) {
- this.consumerIdsChangeListener.consumerIdsChanged(group, consumerGroupInfo.getAllChannel());
+ this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
index a1b2d8a..d716a33 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
@@ -17,8 +17,12 @@
package org.apache.rocketmq.broker.client;
import io.netty.channel.Channel;
+
+import java.util.Collection;
import java.util.List;
+
import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListener {
private final BrokerController brokerController;
@@ -28,11 +32,34 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen
}
@Override
- public void consumerIdsChanged(String group, List<Channel> channels) {
- if (channels != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) {
- for (Channel chl : channels) {
- this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group);
- }
+ public void handle(ConsumerGroupEvent event, String group, Object... args) {
+ if (event == null) {
+ return;
+ }
+ switch (event) {
+ case CHANGE:
+ if (args == null || args.length < 1) {
+ return;
+ }
+ List<Channel> channels = (List<Channel>) args[0];
+ if (channels != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) {
+ for (Channel chl : channels) {
+ this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group);
+ }
+ }
+ break;
+ case UNREGISTER:
+ this.brokerController.getConsumerFilterManager().unRegister(group);
+ break;
+ case REGISTER:
+ if (args == null || args.length < 1) {
+ return;
+ }
+ Collection<SubscriptionData> subscriptionDataList = (Collection<SubscriptionData>) args[0];
+ this.brokerController.getConsumerFilterManager().register(group, subscriptionDataList);
+ break;
+ default:
+ throw new RuntimeException("Unknown event " + event);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMap.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMap.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMap.java
new file mode 100644
index 0000000..85415d6
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMap.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.filter;
+
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.filter.util.BitsArray;
+import org.apache.rocketmq.store.CommitLogDispatcher;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Iterator;
+
+/**
+ * Calculate bit map of filter.
+ */
+public class CommitLogDispatcherCalcBitMap implements CommitLogDispatcher {
+
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME);
+
+ protected final BrokerConfig brokerConfig;
+ protected final ConsumerFilterManager consumerFilterManager;
+
+ public CommitLogDispatcherCalcBitMap(BrokerConfig brokerConfig, ConsumerFilterManager consumerFilterManager) {
+ this.brokerConfig = brokerConfig;
+ this.consumerFilterManager = consumerFilterManager;
+ }
+
+ @Override
+ public void dispatch(DispatchRequest request) {
+ if (!this.brokerConfig.isEnableCalcFilterBitMap()) {
+ return;
+ }
+
+ try {
+
+ Collection<ConsumerFilterData> filterDatas = consumerFilterManager.get(request.getTopic());
+
+ if (filterDatas == null || filterDatas.isEmpty()) {
+ return;
+ }
+
+ Iterator<ConsumerFilterData> iterator = filterDatas.iterator();
+ BitsArray filterBitMap = BitsArray.create(
+ this.consumerFilterManager.getBloomFilter().getM()
+ );
+
+ long startTime = System.currentTimeMillis();
+ while (iterator.hasNext()) {
+ ConsumerFilterData filterData = iterator.next();
+
+ if (filterData.getCompiledExpression() == null) {
+ log.error("[BUG] Consumer in filter manager has no compiled expression! {}", filterData);
+ continue;
+ }
+
+ if (filterData.getBloomFilterData() == null) {
+ log.error("[BUG] Consumer in filter manager has no bloom data! {}", filterData);
+ continue;
+ }
+
+ Object ret = null;
+ try {
+ MessageEvaluationContext context = new MessageEvaluationContext(request.getPropertiesMap());
+
+ ret = filterData.getCompiledExpression().evaluate(context);
+ } catch (Throwable e) {
+ log.error("Calc filter bit map error!commitLogOffset={}, consumer={}, {}", request.getCommitLogOffset(), filterData, e);
+ }
+
+ log.debug("Result of Calc bit map:ret={}, data={}, props={}, offset={}", ret, filterData, request.getPropertiesMap(), request.getCommitLogOffset());
+
+ // eval true
+ if (ret != null && ret instanceof Boolean && (Boolean) ret) {
+ consumerFilterManager.getBloomFilter().hashTo(
+ filterData.getBloomFilterData(),
+ filterBitMap
+ );
+ }
+ }
+
+ request.setBitMap(filterBitMap.bytes());
+
+ long eclipseTime = System.currentTimeMillis() - startTime;
+ // 1ms
+ if (eclipseTime >= 1) {
+ log.warn("Spend {} ms to calc bit map, consumerNum={}, topic={}", eclipseTime, filterDatas.size(), request.getTopic());
+ }
+ } catch (Throwable e) {
+ log.error("Calc bit map error! topic={}, offset={}, queueId={}, {}", request.getTopic(), request.getCommitLogOffset(), request.getQueueId(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterData.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterData.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterData.java
new file mode 100644
index 0000000..4db02e2
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterData.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.filter;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+import org.apache.rocketmq.filter.expression.Expression;
+import org.apache.rocketmq.filter.util.BloomFilterData;
+
+import java.util.Collections;
+
+/**
+ * Filter data of consumer.
+ */
+public class ConsumerFilterData {
+
+ private String consumerGroup;
+ private String topic;
+ private String expression;
+ private String expressionType;
+ private transient Expression compiledExpression;
+ private long bornTime;
+ private long deadTime = 0;
+ private BloomFilterData bloomFilterData;
+ private long clientVersion;
+
+ public boolean isDead() {
+ return this.deadTime >= this.bornTime;
+ }
+
+ public long howLongAfterDeath() {
+ if (isDead()) {
+ return System.currentTimeMillis() - getDeadTime();
+ }
+ return -1;
+ }
+
+ /**
+ * Check this filter data has been used to calculate bit map when msg was stored in server.
+ *
+ * @param msgStoreTime
+ * @return
+ */
+ public boolean isMsgInLive(long msgStoreTime) {
+ return msgStoreTime > getBornTime();
+ }
+
+ public String getConsumerGroup() {
+ return consumerGroup;
+ }
+
+ public void setConsumerGroup(final String consumerGroup) {
+ this.consumerGroup = consumerGroup;
+ }
+
+ public String getTopic() {
+ return topic;
+ }
+
+ public void setTopic(final String topic) {
+ this.topic = topic;
+ }
+
+ public String getExpression() {
+ return expression;
+ }
+
+ public void setExpression(final String expression) {
+ this.expression = expression;
+ }
+
+ public String getExpressionType() {
+ return expressionType;
+ }
+
+ public void setExpressionType(final String expressionType) {
+ this.expressionType = expressionType;
+ }
+
+ public Expression getCompiledExpression() {
+ return compiledExpression;
+ }
+
+ public void setCompiledExpression(final Expression compiledExpression) {
+ this.compiledExpression = compiledExpression;
+ }
+
+ public long getBornTime() {
+ return bornTime;
+ }
+
+ public void setBornTime(final long bornTime) {
+ this.bornTime = bornTime;
+ }
+
+ public long getDeadTime() {
+ return deadTime;
+ }
+
+ public void setDeadTime(final long deadTime) {
+ this.deadTime = deadTime;
+ }
+
+ public BloomFilterData getBloomFilterData() {
+ return bloomFilterData;
+ }
+
+ public void setBloomFilterData(final BloomFilterData bloomFilterData) {
+ this.bloomFilterData = bloomFilterData;
+ }
+
+ public long getClientVersion() {
+ return clientVersion;
+ }
+
+ public void setClientVersion(long clientVersion) {
+ this.clientVersion = clientVersion;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return EqualsBuilder.reflectionEquals(this, o, Collections.<String>emptyList());
+ }
+
+ @Override
+ public int hashCode() {
+ return HashCodeBuilder.reflectionHashCode(this, Collections.<String>emptyList());
+ }
+
+ @Override
+ public String toString() {
+ return ReflectionToStringBuilder.toString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java
new file mode 100644
index 0000000..7f790af
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java
@@ -0,0 +1,471 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.filter;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.BrokerPathConfigHelper;
+import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.filter.FilterFactory;
+import org.apache.rocketmq.common.filter.ExpressionType;
+import org.apache.rocketmq.filter.util.BloomFilter;
+import org.apache.rocketmq.filter.util.BloomFilterData;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Consumer filter data manager.Just manage the consumers use expression filter.
+ */
+public class ConsumerFilterManager extends ConfigManager {
+
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME);
+
+ private static final long MS_24_HOUR = 24 * 3600 * 1000;
+
+ private ConcurrentHashMap<String/*Topic*/, FilterDataMapByTopic>
+ filterDataByTopic = new ConcurrentHashMap<String/*consumer group*/, FilterDataMapByTopic>(256);
+
+ private transient BrokerController brokerController;
+ private transient BloomFilter bloomFilter;
+
+ public ConsumerFilterManager() {
+ // just for test
+ this.bloomFilter = BloomFilter.createByFn(20, 64);
+ }
+
+ public ConsumerFilterManager(BrokerController brokerController) {
+ this.brokerController = brokerController;
+ this.bloomFilter = BloomFilter.createByFn(
+ brokerController.getBrokerConfig().getMaxErrorRateOfBloomFilter(),
+ brokerController.getBrokerConfig().getExpectConsumerNumUseFilter()
+ );
+ // then set bit map length of store config.
+ brokerController.getMessageStoreConfig().setBitMapLengthConsumeQueueExt(
+ this.bloomFilter.getM()
+ );
+ }
+
+ /**
+ * Build consumer filter data.Be care, bloom filter data is not included.
+ *
+ * @param topic
+ * @param consumerGroup
+ * @param expression
+ * @param type
+ * @param clientVersion
+ * @return maybe null
+ */
+ public static ConsumerFilterData build(final String topic, final String consumerGroup,
+ final String expression, final String type,
+ final long clientVersion) {
+ if (ExpressionType.isTagType(type)) {
+ return null;
+ }
+
+ ConsumerFilterData consumerFilterData = new ConsumerFilterData();
+ consumerFilterData.setTopic(topic);
+ consumerFilterData.setConsumerGroup(consumerGroup);
+ consumerFilterData.setBornTime(System.currentTimeMillis());
+ consumerFilterData.setDeadTime(0);
+ consumerFilterData.setExpression(expression);
+ consumerFilterData.setExpressionType(type);
+ consumerFilterData.setClientVersion(clientVersion);
+ try {
+ consumerFilterData.setCompiledExpression(
+ FilterFactory.INSTANCE.get(type).compile(expression)
+ );
+ } catch (Throwable e) {
+ log.error("parse error: expr={}, topic={}, group={}, error={}", expression, topic, consumerGroup, e.getMessage());
+ return null;
+ }
+
+ return consumerFilterData;
+ }
+
+ public void register(final String consumerGroup, final Collection<SubscriptionData> subList) {
+ for (SubscriptionData subscriptionData : subList) {
+ register(
+ subscriptionData.getTopic(),
+ consumerGroup,
+ subscriptionData.getSubString(),
+ subscriptionData.getExpressionType(),
+ subscriptionData.getSubVersion()
+ );
+ }
+
+ // make illegal topic dead.
+ Collection<ConsumerFilterData> groupFilterData = getByGroup(consumerGroup);
+
+ Iterator<ConsumerFilterData> iterator = groupFilterData.iterator();
+ while (iterator.hasNext()) {
+ ConsumerFilterData filterData = iterator.next();
+
+ boolean exist = false;
+ for (SubscriptionData subscriptionData : subList) {
+ if (subscriptionData.getTopic().equals(filterData.getTopic())) {
+ exist = true;
+ break;
+ }
+ }
+
+ if (!exist && !filterData.isDead()) {
+ filterData.setDeadTime(System.currentTimeMillis());
+ log.info("Consumer filter changed: {}, make illegal topic dead:{}", consumerGroup, filterData);
+ }
+ }
+ }
+
+ public boolean register(final String topic, final String consumerGroup, final String expression,
+ final String type, final long clientVersion) {
+ if (ExpressionType.isTagType(type)) {
+ return false;
+ }
+
+ if (expression == null || expression.length() == 0) {
+ return false;
+ }
+
+ FilterDataMapByTopic filterDataMapByTopic = this.filterDataByTopic.get(topic);
+
+ if (filterDataMapByTopic == null) {
+ FilterDataMapByTopic temp = new FilterDataMapByTopic(topic);
+ FilterDataMapByTopic prev = this.filterDataByTopic.putIfAbsent(topic, temp);
+ filterDataMapByTopic = prev != null ? prev : temp;
+ }
+
+ BloomFilterData bloomFilterData = bloomFilter.generate(consumerGroup + "#" + topic);
+
+ return filterDataMapByTopic.register(consumerGroup, expression, type, bloomFilterData, clientVersion);
+ }
+
+ public void unRegister(final String consumerGroup) {
+ for (String topic : filterDataByTopic.keySet()) {
+ this.filterDataByTopic.get(topic).unRegister(consumerGroup);
+ }
+ }
+
+ public ConsumerFilterData get(final String topic, final String consumerGroup) {
+ if (!this.filterDataByTopic.containsKey(topic)) {
+ return null;
+ }
+ if (this.filterDataByTopic.get(topic).getGroupFilterData().isEmpty()) {
+ return null;
+ }
+
+ return this.filterDataByTopic.get(topic).getGroupFilterData().get(consumerGroup);
+ }
+
+ public Collection<ConsumerFilterData> getByGroup(final String consumerGroup) {
+ Collection<ConsumerFilterData> ret = new HashSet<ConsumerFilterData>();
+
+ Iterator<FilterDataMapByTopic> topicIterator = this.filterDataByTopic.values().iterator();
+ while (topicIterator.hasNext()) {
+ FilterDataMapByTopic filterDataMapByTopic = topicIterator.next();
+
+ Iterator<ConsumerFilterData> filterDataIterator = filterDataMapByTopic.getGroupFilterData().values().iterator();
+
+ while (filterDataIterator.hasNext()) {
+ ConsumerFilterData filterData = filterDataIterator.next();
+
+ if (filterData.getConsumerGroup().equals(consumerGroup)) {
+ ret.add(filterData);
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ public final Collection<ConsumerFilterData> get(final String topic) {
+ if (!this.filterDataByTopic.containsKey(topic)) {
+ return null;
+ }
+ if (this.filterDataByTopic.get(topic).getGroupFilterData().isEmpty()) {
+ return null;
+ }
+
+ return this.filterDataByTopic.get(topic).getGroupFilterData().values();
+ }
+
+ public BloomFilter getBloomFilter() {
+ return bloomFilter;
+ }
+
+ @Override
+ public String encode() {
+ return encode(false);
+ }
+
+ @Override
+ public String configFilePath() {
+ if (this.brokerController != null) {
+ return BrokerPathConfigHelper.getConsumerFilterPath(
+ this.brokerController.getMessageStoreConfig().getStorePathRootDir()
+ );
+ }
+ return BrokerPathConfigHelper.getConsumerFilterPath("./unit_test");
+ }
+
+ @Override
+ public void decode(final String jsonString) {
+ ConsumerFilterManager load = RemotingSerializable.fromJson(jsonString, ConsumerFilterManager.class);
+ if (load != null && load.filterDataByTopic != null) {
+ boolean bloomChanged = false;
+ for (String topic : load.filterDataByTopic.keySet()) {
+ FilterDataMapByTopic dataMapByTopic = load.filterDataByTopic.get(topic);
+ if (dataMapByTopic == null) {
+ continue;
+ }
+
+ for (String group : dataMapByTopic.getGroupFilterData().keySet()) {
+
+ ConsumerFilterData filterData = dataMapByTopic.getGroupFilterData().get(group);
+
+ if (filterData == null) {
+ continue;
+ }
+
+ try {
+ filterData.setCompiledExpression(
+ FilterFactory.INSTANCE.get(filterData.getExpressionType()).compile(filterData.getExpression())
+ );
+ } catch (Exception e) {
+ log.error("load filter data error, " + filterData, e);
+ }
+
+ // check whether bloom filter is changed
+ // if changed, ignore the bit map calculated before.
+ if (!this.bloomFilter.isValid(filterData.getBloomFilterData())) {
+ bloomChanged = true;
+ log.info("Bloom filter is changed!So ignore all filter data persisted! {}, {}", this.bloomFilter, filterData.getBloomFilterData());
+ break;
+ }
+
+ log.info("load exist consumer filter data: {}", filterData);
+
+ if (filterData.getDeadTime() == 0) {
+ // we think all consumers are dead when load
+ long deadTime = System.currentTimeMillis() - 30 * 1000;
+ filterData.setDeadTime(
+ deadTime <= filterData.getBornTime() ? filterData.getBornTime() : deadTime
+ );
+ }
+ }
+ }
+
+ if (!bloomChanged) {
+ this.filterDataByTopic = load.filterDataByTopic;
+ }
+ }
+ }
+
+ @Override
+ public String encode(final boolean prettyFormat) {
+ // clean
+ {
+ clean();
+ }
+ return RemotingSerializable.toJson(this, prettyFormat);
+ }
+
+ public void clean() {
+ Iterator<Map.Entry<String, FilterDataMapByTopic>> topicIterator = this.filterDataByTopic.entrySet().iterator();
+ while (topicIterator.hasNext()) {
+ Map.Entry<String, FilterDataMapByTopic> filterDataMapByTopic = topicIterator.next();
+
+ Iterator<Map.Entry<String, ConsumerFilterData>> filterDataIterator
+ = filterDataMapByTopic.getValue().getGroupFilterData().entrySet().iterator();
+
+ while (filterDataIterator.hasNext()) {
+ Map.Entry<String, ConsumerFilterData> filterDataByGroup = filterDataIterator.next();
+
+ ConsumerFilterData filterData = filterDataByGroup.getValue();
+ if (filterData.howLongAfterDeath() >= (this.brokerController == null ? MS_24_HOUR : this.brokerController.getBrokerConfig().getFilterDataCleanTimeSpan())) {
+ log.info("Remove filter consumer {}, died too long!", filterDataByGroup.getValue());
+ filterDataIterator.remove();
+ }
+ }
+
+ if (filterDataMapByTopic.getValue().getGroupFilterData().isEmpty()) {
+ log.info("Topic has no consumer, remove it! {}", filterDataMapByTopic.getKey());
+ topicIterator.remove();
+ }
+ }
+ }
+
+ public ConcurrentHashMap<String, FilterDataMapByTopic> getFilterDataByTopic() {
+ return filterDataByTopic;
+ }
+
+ public void setFilterDataByTopic(final ConcurrentHashMap<String, FilterDataMapByTopic> filterDataByTopic) {
+ this.filterDataByTopic = filterDataByTopic;
+ }
+
+ public static class FilterDataMapByTopic {
+
+ private ConcurrentHashMap<String/*consumer group*/, ConsumerFilterData>
+ groupFilterData = new ConcurrentHashMap<String, ConsumerFilterData>();
+
+ private String topic;
+
+ public FilterDataMapByTopic() {
+ }
+
+ public FilterDataMapByTopic(String topic) {
+ this.topic = topic;
+ }
+
+ public void unRegister(String consumerGroup) {
+ if (!this.groupFilterData.containsKey(consumerGroup)) {
+ return;
+ }
+
+ ConsumerFilterData data = this.groupFilterData.get(consumerGroup);
+
+ if (data == null || data.isDead()) {
+ return;
+ }
+
+ long now = System.currentTimeMillis();
+
+ log.info("Unregister consumer filter: {}, deadTime: {}", data, now);
+
+ data.setDeadTime(now);
+ }
+
+ public boolean register(String consumerGroup, String expression, String type, BloomFilterData bloomFilterData, long clientVersion) {
+ ConsumerFilterData old = this.groupFilterData.get(consumerGroup);
+
+ if (old == null) {
+ ConsumerFilterData consumerFilterData = build(topic, consumerGroup, expression, type, clientVersion);
+ if (consumerFilterData == null) {
+ return false;
+ }
+ consumerFilterData.setBloomFilterData(bloomFilterData);
+
+ old = this.groupFilterData.putIfAbsent(consumerGroup, consumerFilterData);
+ if (old == null) {
+ log.info("New consumer filter registered: {}", consumerFilterData);
+ return true;
+ } else {
+ if (clientVersion <= old.getClientVersion()) {
+ if (!type.equals(old.getExpressionType()) || !expression.equals(old.getExpression())) {
+ log.warn("Ignore consumer({} : {}) filter(concurrent), because of version {} <= {}, but maybe info changed!old={}:{}, ignored={}:{}",
+ consumerGroup, topic,
+ clientVersion, old.getClientVersion(),
+ old.getExpressionType(), old.getExpression(),
+ type, expression);
+ }
+ if (clientVersion == old.getClientVersion() && old.isDead()) {
+ reAlive(old);
+ return true;
+ }
+
+ return false;
+ } else {
+ this.groupFilterData.put(consumerGroup, consumerFilterData);
+ log.info("New consumer filter registered(concurrent): {}, old: {}", consumerFilterData, old);
+ return true;
+ }
+ }
+ } else {
+ if (clientVersion <= old.getClientVersion()) {
+ if (!type.equals(old.getExpressionType()) || !expression.equals(old.getExpression())) {
+ log.info("Ignore consumer({}:{}) filter, because of version {} <= {}, but maybe info changed!old={}:{}, ignored={}:{}",
+ consumerGroup, topic,
+ clientVersion, old.getClientVersion(),
+ old.getExpressionType(), old.getExpression(),
+ type, expression);
+ }
+ if (clientVersion == old.getClientVersion() && old.isDead()) {
+ reAlive(old);
+ return true;
+ }
+
+ return false;
+ }
+
+ boolean change = !old.getExpression().equals(expression) || !old.getExpressionType().equals(type);
+ if (old.getBloomFilterData() == null && bloomFilterData != null) {
+ change = true;
+ }
+ if (old.getBloomFilterData() != null && !old.getBloomFilterData().equals(bloomFilterData)) {
+ change = true;
+ }
+
+ // if subscribe data is changed, or consumer is died too long.
+ if (change) {
+ ConsumerFilterData consumerFilterData = build(topic, consumerGroup, expression, type, clientVersion);
+ if (consumerFilterData == null) {
+ // new expression compile error, remove old, let client report error.
+ this.groupFilterData.remove(consumerGroup);
+ return false;
+ }
+ consumerFilterData.setBloomFilterData(bloomFilterData);
+
+ this.groupFilterData.put(consumerGroup, consumerFilterData);
+
+ log.info("Consumer filter info change, old: {}, new: {}, change: {}",
+ old, consumerFilterData, change);
+
+ return true;
+ } else {
+ old.setClientVersion(clientVersion);
+ if (old.isDead()) {
+ reAlive(old);
+ }
+ return true;
+ }
+ }
+ }
+
+ protected void reAlive(ConsumerFilterData filterData) {
+ long oldDeadTime = filterData.getDeadTime();
+ filterData.setDeadTime(0);
+ log.info("Re alive consumer filter: {}, oldDeadTime: {}", filterData, oldDeadTime);
+ }
+
+ public final ConsumerFilterData get(String consumerGroup) {
+ return this.groupFilterData.get(consumerGroup);
+ }
+
+ public final ConcurrentHashMap<String, ConsumerFilterData> getGroupFilterData() {
+ return this.groupFilterData;
+ }
+
+ public void setGroupFilterData(final ConcurrentHashMap<String, ConsumerFilterData> groupFilterData) {
+ this.groupFilterData = groupFilterData;
+ }
+
+ public String getTopic() {
+ return topic;
+ }
+
+ public void setTopic(final String topic) {
+ this.topic = topic;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionForRetryMessageFilter.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionForRetryMessageFilter.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionForRetryMessageFilter.java
new file mode 100644
index 0000000..9518178
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionForRetryMessageFilter.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.filter;
+
+
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.filter.ExpressionType;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+/**
+ * Support filter to retry topic.
+ * <br>It will decode properties first in order to get real topic.
+ */
+public class ExpressionForRetryMessageFilter extends ExpressionMessageFilter {
+ public ExpressionForRetryMessageFilter(SubscriptionData subscriptionData, ConsumerFilterData consumerFilterData, ConsumerFilterManager consumerFilterManager) {
+ super(subscriptionData, consumerFilterData, consumerFilterManager);
+ }
+
+ @Override
+ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {
+ if (subscriptionData == null) {
+ return true;
+ }
+
+ if (subscriptionData.isClassFilterMode()) {
+ return true;
+ }
+
+ boolean isRetryTopic = subscriptionData.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX);
+
+ if (!isRetryTopic && ExpressionType.isTagType(subscriptionData.getExpressionType())) {
+ return true;
+ }
+
+ ConsumerFilterData realFilterData = this.consumerFilterData;
+ Map<String, String> tempProperties = properties;
+ boolean decoded = false;
+ if (isRetryTopic) {
+ // retry topic, use original filter data.
+ // poor performance to support retry filter.
+ if (tempProperties == null && msgBuffer != null) {
+ decoded = true;
+ tempProperties = MessageDecoder.decodeProperties(msgBuffer);
+ }
+ String realTopic = tempProperties.get(MessageConst.PROPERTY_RETRY_TOPIC);
+ String group = subscriptionData.getTopic().substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length());
+ realFilterData = this.consumerFilterManager.get(realTopic, group);
+ }
+
+ // no expression
+ if (realFilterData == null || realFilterData.getExpression() == null
+ || realFilterData.getCompiledExpression() == null) {
+ return true;
+ }
+
+ if (!decoded && tempProperties == null && msgBuffer != null) {
+ tempProperties = MessageDecoder.decodeProperties(msgBuffer);
+ }
+
+ Object ret = null;
+ try {
+ MessageEvaluationContext context = new MessageEvaluationContext(tempProperties);
+
+ ret = realFilterData.getCompiledExpression().evaluate(context);
+ } catch (Throwable e) {
+ log.error("Message Filter error, " + realFilterData + ", " + tempProperties, e);
+ }
+
+ log.debug("Pull eval result: {}, {}, {}", ret, realFilterData, tempProperties);
+
+ if (ret == null || !(ret instanceof Boolean)) {
+ return false;
+ }
+
+ return (Boolean) ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionMessageFilter.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionMessageFilter.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionMessageFilter.java
new file mode 100644
index 0000000..893df0d
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionMessageFilter.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.filter;
+
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.filter.ExpressionType;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.filter.util.BitsArray;
+import org.apache.rocketmq.filter.util.BloomFilter;
+import org.apache.rocketmq.store.ConsumeQueueExt;
+import org.apache.rocketmq.store.MessageFilter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+public class ExpressionMessageFilter implements MessageFilter {
+
+ protected static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME);
+
+ protected final SubscriptionData subscriptionData;
+ protected final ConsumerFilterData consumerFilterData;
+ protected final ConsumerFilterManager consumerFilterManager;
+ protected final boolean bloomDataValid;
+
+ public ExpressionMessageFilter(SubscriptionData subscriptionData, ConsumerFilterData consumerFilterData,
+ ConsumerFilterManager consumerFilterManager) {
+ this.subscriptionData = subscriptionData;
+ this.consumerFilterData = consumerFilterData;
+ this.consumerFilterManager = consumerFilterManager;
+ if (consumerFilterData == null) {
+ bloomDataValid = false;
+ return;
+ }
+ BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
+ if (bloomFilter != null && bloomFilter.isValid(consumerFilterData.getBloomFilterData())) {
+ bloomDataValid = true;
+ } else {
+ bloomDataValid = false;
+ }
+ }
+
+ @Override
+ public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
+ if (null == subscriptionData) {
+ return true;
+ }
+
+ if (subscriptionData.isClassFilterMode()) {
+ return true;
+ }
+
+ // by tags code.
+ if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {
+
+ if (tagsCode == null || tagsCode < 0L) {
+ return true;
+ }
+
+ if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
+ return true;
+ }
+
+ return subscriptionData.getCodeSet().contains(tagsCode.intValue());
+ } else {
+ // no expression or no bloom
+ if (consumerFilterData == null || consumerFilterData.getExpression() == null
+ || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
+ return true;
+ }
+
+ // message is before consumer
+ if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
+ log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
+ return true;
+ }
+
+ byte[] filterBitMap = cqExtUnit.getFilterBitMap();
+ BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
+ if (filterBitMap == null || !this.bloomDataValid
+ || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
+ return true;
+ }
+
+ BitsArray bitsArray = null;
+ try {
+ bitsArray = BitsArray.create(filterBitMap);
+ boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
+ log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
+ return ret;
+ } catch (Throwable e) {
+ log.error("bloom filter error, sub=" + subscriptionData
+ + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {
+ if (subscriptionData == null) {
+ return true;
+ }
+
+ if (subscriptionData.isClassFilterMode()) {
+ return true;
+ }
+
+ if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {
+ return true;
+ }
+
+ ConsumerFilterData realFilterData = this.consumerFilterData;
+ Map<String, String> tempProperties = properties;
+
+ // no expression
+ if (realFilterData == null || realFilterData.getExpression() == null
+ || realFilterData.getCompiledExpression() == null) {
+ return true;
+ }
+
+ if (tempProperties == null && msgBuffer != null) {
+ tempProperties = MessageDecoder.decodeProperties(msgBuffer);
+ }
+
+ Object ret = null;
+ try {
+ MessageEvaluationContext context = new MessageEvaluationContext(tempProperties);
+
+ ret = realFilterData.getCompiledExpression().evaluate(context);
+ } catch (Throwable e) {
+ log.error("Message Filter error, " + realFilterData + ", " + tempProperties, e);
+ }
+
+ log.debug("Pull eval result: {}, {}, {}", ret, realFilterData, tempProperties);
+
+ if (ret == null || !(ret instanceof Boolean)) {
+ return false;
+ }
+
+ return (Boolean) ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/filter/MessageEvaluationContext.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/filter/MessageEvaluationContext.java b/broker/src/main/java/org/apache/rocketmq/broker/filter/MessageEvaluationContext.java
new file mode 100644
index 0000000..879d179
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/filter/MessageEvaluationContext.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.filter;
+
+import org.apache.rocketmq.filter.expression.EvaluationContext;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Evaluation context from message.
+ */
+public class MessageEvaluationContext implements EvaluationContext {
+
+ private Map<String, String> properties;
+
+ public MessageEvaluationContext(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ @Override
+ public Object get(final String name) {
+ if (this.properties == null) {
+ return null;
+ }
+ return this.properties.get(name);
+ }
+
+ @Override
+ public Map<String, Object> keyValues() {
+ if (properties == null) {
+ return null;
+ }
+
+ Map<String, Object> copy = new HashMap<String, Object>(properties.size(), 1);
+
+ for (String key : properties.keySet()) {
+ copy.put(key, properties.get(key));
+ }
+
+ return copy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
index 2dec9f7..fd38c4f 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
@@ -19,6 +19,8 @@ package org.apache.rocketmq.broker.longpolling;
import org.apache.rocketmq.store.MessageArrivingListener;
+import java.util.Map;
+
public class NotifyMessageArrivingListener implements MessageArrivingListener {
private final PullRequestHoldService pullRequestHoldService;
@@ -27,7 +29,9 @@ public class NotifyMessageArrivingListener implements MessageArrivingListener {
}
@Override
- public void arriving(String topic, int queueId, long logicOffset, long tagsCode) {
- this.pullRequestHoldService.notifyMessageArriving(topic, queueId, logicOffset, tagsCode);
+ public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
+ long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
+ this.pullRequestHoldService.notifyMessageArriving(topic, queueId, logicOffset, tagsCode,
+ msgStoreTime, filterBitMap, properties);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequest.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequest.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequest.java
index b66344f..045ab9b 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequest.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequest.java
@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.longpolling;
import io.netty.channel.Channel;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.MessageFilter;
public class PullRequest {
private final RemotingCommand requestCommand;
@@ -27,15 +28,18 @@ public class PullRequest {
private final long suspendTimestamp;
private final long pullFromThisOffset;
private final SubscriptionData subscriptionData;
+ private final MessageFilter messageFilter;
public PullRequest(RemotingCommand requestCommand, Channel clientChannel, long timeoutMillis, long suspendTimestamp,
- long pullFromThisOffset, SubscriptionData subscriptionData) {
+ long pullFromThisOffset, SubscriptionData subscriptionData,
+ MessageFilter messageFilter) {
this.requestCommand = requestCommand;
this.clientChannel = clientChannel;
this.timeoutMillis = timeoutMillis;
this.suspendTimestamp = suspendTimestamp;
this.pullFromThisOffset = pullFromThisOffset;
this.subscriptionData = subscriptionData;
+ this.messageFilter = messageFilter;
}
public RemotingCommand getRequestCommand() {
@@ -61,4 +65,8 @@ public class PullRequest {
public SubscriptionData getSubscriptionData() {
return subscriptionData;
}
+
+ public MessageFilter getMessageFilter() {
+ return messageFilter;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
index fdba50d..1a53db1 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
@@ -18,13 +18,13 @@ package org.apache.rocketmq.broker.longpolling;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.common.SystemClock;
import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.store.DefaultMessageFilter;
-import org.apache.rocketmq.store.MessageFilter;
+import org.apache.rocketmq.store.ConsumeQueueExt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,7 +33,6 @@ public class PullRequestHoldService extends ServiceThread {
private static final String TOPIC_QUEUEID_SEPARATOR = "@";
private final BrokerController brokerController;
private final SystemClock systemClock = new SystemClock();
- private final MessageFilter messageFilter = new DefaultMessageFilter();
private ConcurrentHashMap<String/* topic@queueId */, ManyPullRequest> pullRequestTable =
new ConcurrentHashMap<String, ManyPullRequest>(1024);
@@ -110,10 +109,11 @@ public class PullRequestHoldService extends ServiceThread {
}
public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset) {
- notifyMessageArriving(topic, queueId, maxOffset, null);
+ notifyMessageArriving(topic, queueId, maxOffset, null, 0, null, null);
}
- public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset, final Long tagsCode) {
+ public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset, final Long tagsCode,
+ long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
String key = this.buildKey(topic, queueId);
ManyPullRequest mpr = this.pullRequestTable.get(key);
if (mpr != null) {
@@ -128,7 +128,14 @@ public class PullRequestHoldService extends ServiceThread {
}
if (newestOffset > request.getPullFromThisOffset()) {
- if (this.messageFilter.isMessageMatched(request.getSubscriptionData(), tagsCode)) {
+ boolean match = request.getMessageFilter().isMatchedByConsumeQueue(tagsCode,
+ new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap));
+ // match by bit map, need eval again when properties is not null.
+ if (match && properties != null) {
+ match = request.getMessageFilter().isMatchedByCommitLog(null, properties);
+ }
+
+ if (match) {
try {
this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
request.getRequestCommand());
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
index 039c942..6c2a987 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
@@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory;
public class BrokerOuterAPI {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final RemotingClient remotingClient;
- private final TopAddressing topAddressing = new TopAddressing(MixAll.WS_ADDR);
+ private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr());
private String nameSrvAddr = null;
public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java b/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
index 00257fd..8ded973 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
@@ -18,11 +18,14 @@
package org.apache.rocketmq.broker.plugin;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.Set;
import org.apache.rocketmq.common.message.MessageExt;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.store.CommitLogDispatcher;
+import org.apache.rocketmq.store.ConsumeQueue;
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.store.MessageFilter;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.QueryMessageResult;
@@ -84,8 +87,8 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
@Override
public GetMessageResult getMessage(String group, String topic, int queueId, long offset,
- int maxMsgNums, SubscriptionData subscriptionData) {
- return next.getMessage(group, topic, queueId, offset, maxMsgNums, subscriptionData);
+ int maxMsgNums, final MessageFilter messageFilter) {
+ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);
}
@Override
@@ -234,4 +237,13 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
next.setConfirmOffset(phyOffset);
}
+ @Override
+ public LinkedList<CommitLogDispatcher> getDispatcherList() {
+ return next.getDispatcherList();
+ }
+
+ @Override
+ public ConsumeQueue getConsumeQueue(String topic, int queueId) {
+ return next.getConsumeQueue(topic, queueId);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
----------------------------------------------------------------------
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
index e35316d..daea53c 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
@@ -16,6 +16,7 @@
*/
package org.apache.rocketmq.broker.processor;
+import com.alibaba.fastjson.JSON;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import java.io.UnsupportedEncodingException;
@@ -32,6 +33,8 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.broker.filter.ConsumerFilterData;
+import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
import org.apache.rocketmq.common.MQVersion;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
@@ -49,6 +52,7 @@ import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.body.BrokerStatsData;
import org.apache.rocketmq.common.protocol.body.BrokerStatsItem;
import org.apache.rocketmq.common.protocol.body.Connection;
+import org.apache.rocketmq.common.protocol.body.ConsumeQueueData;
import org.apache.rocketmq.common.protocol.body.ConsumeStatsList;
import org.apache.rocketmq.common.protocol.body.ConsumerConnection;
import org.apache.rocketmq.common.protocol.body.GroupList;
@@ -56,6 +60,7 @@ import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody;
import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueryConsumeTimeSpanBody;
import org.apache.rocketmq.common.protocol.body.QueryCorrectionOffsetBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
@@ -81,6 +86,7 @@ import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader;
import org.apache.rocketmq.common.protocol.header.GetMinOffsetResponseHeader;
import org.apache.rocketmq.common.protocol.header.GetProducerConnectionListRequestHeader;
import org.apache.rocketmq.common.protocol.header.GetTopicStatsInfoRequestHeader;
+import org.apache.rocketmq.common.protocol.header.QueryConsumeQueueRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryConsumeTimeSpanRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryCorrectionOffsetHeader;
import org.apache.rocketmq.common.protocol.header.QueryTopicConsumeByWhoRequestHeader;
@@ -94,6 +100,7 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.stats.StatsItem;
import org.apache.rocketmq.common.stats.StatsSnapshot;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.filter.util.BitsArray;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
@@ -101,7 +108,10 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.LanguageCode;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.store.ConsumeQueue;
+import org.apache.rocketmq.store.ConsumeQueueExt;
import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.MessageFilter;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -187,6 +197,8 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
return ViewBrokerStatsData(ctx, request);
case RequestCode.GET_BROKER_CONSUME_STATS:
return fetchAllConsumeStatsInBroker(ctx, request);
+ case RequestCode.QUERY_CONSUME_QUEUE:
+ return queryConsumeQueue(ctx, request);
default:
break;
}
@@ -1244,4 +1256,83 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
}
}
+ private RemotingCommand queryConsumeQueue(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
+ QueryConsumeQueueRequestHeader requestHeader =
+ (QueryConsumeQueueRequestHeader) request.decodeCommandCustomHeader(QueryConsumeQueueRequestHeader.class);
+
+ RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ ConsumeQueue consumeQueue = this.brokerController.getMessageStore().getConsumeQueue(requestHeader.getTopic(),
+ requestHeader.getQueueId());
+ if (consumeQueue == null) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(String.format("%d@%s is not exist!", requestHeader.getQueueId(), requestHeader.getTopic()));
+ return response;
+ }
+
+ QueryConsumeQueueResponseBody body = new QueryConsumeQueueResponseBody();
+ response.setCode(ResponseCode.SUCCESS);
+ response.setBody(body.encode());
+
+ body.setMaxQueueIndex(consumeQueue.getMaxOffsetInQueue());
+ body.setMinQueueIndex(consumeQueue.getMinOffsetInQueue());
+
+ MessageFilter messageFilter = null;
+ if (requestHeader.getConsumerGroup() != null) {
+ SubscriptionData subscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(
+ requestHeader.getConsumerGroup(), requestHeader.getTopic()
+ );
+ body.setSubscriptionData(subscriptionData);
+ if (subscriptionData == null) {
+ body.setFilterData(String.format("%s@%s is not online!", requestHeader.getConsumerGroup(), requestHeader.getTopic()));
+ } else {
+ ConsumerFilterData filterData = this.brokerController.getConsumerFilterManager()
+ .get(requestHeader.getTopic(), requestHeader.getConsumerGroup());
+ body.setFilterData(JSON.toJSONString(filterData, true));
+
+ messageFilter = new ExpressionMessageFilter(subscriptionData, filterData,
+ this.brokerController.getConsumerFilterManager());
+ }
+ }
+
+ SelectMappedBufferResult result = consumeQueue.getIndexBuffer(requestHeader.getIndex());
+ if (result == null) {
+ response.setRemark(String.format("Index %d of %d@%s is not exist!", requestHeader.getIndex(), requestHeader.getQueueId(), requestHeader.getTopic()));
+ return response;
+ }
+ try {
+ List<ConsumeQueueData> queues = new ArrayList<>();
+ for (int i = 0; i < result.getSize() && i < requestHeader.getCount() * ConsumeQueue.CQ_STORE_UNIT_SIZE; i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
+ ConsumeQueueData one = new ConsumeQueueData();
+ one.setPhysicOffset(result.getByteBuffer().getLong());
+ one.setPhysicSize(result.getByteBuffer().getInt());
+ one.setTagsCode(result.getByteBuffer().getLong());
+
+ if (!consumeQueue.isExtAddr(one.getTagsCode())) {
+ queues.add(one);
+ continue;
+ }
+
+ ConsumeQueueExt.CqExtUnit cqExtUnit = consumeQueue.getExt(one.getTagsCode());
+ if (cqExtUnit != null) {
+ one.setExtendDataJson(JSON.toJSONString(cqExtUnit));
+ if (cqExtUnit.getFilterBitMap() != null) {
+ one.setBitMap(BitsArray.create(cqExtUnit.getFilterBitMap()).toString());
+ }
+ if (messageFilter != null) {
+ one.setEval(messageFilter.isMatchedByConsumeQueue(cqExtUnit.getTagsCode(), cqExtUnit));
+ }
+ } else {
+ one.setMsg("Cq extend not exist!addr: " + one.getTagsCode());
+ }
+
+ queues.add(one);
+ }
+ body.setQueueData(queues);
+ } finally {
+ result.release();
+ }
+
+ return response;
+ }
}
[28/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-189]
Misleading tip on consumeTimestamp and wrong consumeTimestamp exception
message closes apache/incubator-rocketmq#97
Posted by do...@apache.org.
[ROCKETMQ-189] Misleading tip on consumeTimestamp and wrong consumeTimestamp exception message closes apache/incubator-rocketmq#97
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/99ce40a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/99ce40a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/99ce40a7
Branch: refs/heads/release-4.1.0-incubating
Commit: 99ce40a72382dd5b7ecb5eb5717b8daf84a90682
Parents: 8280388
Author: lindzh <li...@163.com>
Authored: Fri May 26 15:28:28 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Fri May 26 15:28:28 2017 +0800
----------------------------------------------------------------------
.../client/impl/consumer/DefaultMQPushConsumerImpl.java | 8 ++++----
.../rocketmq/client/impl/consumer/RebalancePushImpl.java | 2 +-
common/src/main/java/org/apache/rocketmq/common/UtilAll.java | 2 +-
.../org/apache/rocketmq/example/simple/PushConsumer.java | 2 ++
4 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/99ce40a7/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
index 2cafe29..8767964 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
@@ -655,12 +655,12 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
null);
}
- Date dt = UtilAll.parseDate(this.defaultMQPushConsumer.getConsumeTimestamp(), UtilAll.YYYY_MMDD_HHMMSS);
+ Date dt = UtilAll.parseDate(this.defaultMQPushConsumer.getConsumeTimestamp(), UtilAll.YYYYMMDDHHMMSS);
if (null == dt) {
throw new MQClientException(
- "consumeTimestamp is invalid, YYYY_MMDD_HHMMSS"
- + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
- null);
+ "consumeTimestamp is invalid, the valid format is yyyyMMddHHmmss,but received "
+ + this.defaultMQPushConsumer.getConsumeTimestamp()
+ + " " + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null);
}
// allocateMessageQueueStrategy
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/99ce40a7/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
index 509c9a4..112bcee 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
@@ -171,7 +171,7 @@ public class RebalancePushImpl extends RebalanceImpl {
} else {
try {
long timestamp = UtilAll.parseDate(this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeTimestamp(),
- UtilAll.YYYY_MMDD_HHMMSS).getTime();
+ UtilAll.YYYYMMDDHHMMSS).getTime();
result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp);
} catch (MQClientException e) {
result = -1;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/99ce40a7/common/src/main/java/org/apache/rocketmq/common/UtilAll.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java
index 016da0b..e9d926f 100644
--- a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java
+++ b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java
@@ -41,7 +41,7 @@ import org.apache.rocketmq.remoting.common.RemotingHelper;
public class UtilAll {
public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss";
public static final String YYYY_MM_DD_HH_MM_SS_SSS = "yyyy-MM-dd#HH:mm:ss:SSS";
- public static final String YYYY_MMDD_HHMMSS = "yyyyMMddHHmmss";
+ public static final String YYYYMMDDHHMMSS = "yyyyMMddHHmmss";
final static char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray();
public static int getPid() {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/99ce40a7/example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java
index d03c2c5..c8252d0 100644
--- a/example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java
@@ -31,6 +31,8 @@ public class PushConsumer {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1");
consumer.subscribe("Jodie_topic_1023", "*");
consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
+ //wrong time format 2017_0422_221800
+ consumer.setConsumeTimestamp("20170422221800");
consumer.registerMessageListener(new MessageListenerConcurrently() {
/**
[34/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-200]-Cluster
name is always missing when fetch ClusterInfo from name server closes
apache/incubator-rocketmq#105
Posted by do...@apache.org.
[ROCKETMQ-200]-Cluster name is always missing when fetch ClusterInfo from name server closes apache/incubator-rocketmq#105
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/c7961407
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/c7961407
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/c7961407
Branch: refs/heads/release-4.1.0-incubating
Commit: c79614071b1941940f934c066bde5711062bdc7e
Parents: 8c8610f
Author: Jaskey <li...@gmail.com>
Authored: Sat May 27 11:24:02 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 11:24:02 2017 +0800
----------------------------------------------------------------------
.../rocketmq/common/protocol/route/BrokerData.java | 14 +++++++++++---
.../rocketmq/namesrv/routeinfo/RouteInfoManager.java | 10 ++--------
2 files changed, 13 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/c7961407/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
index 612e5b4..9d868ae 100644
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
@@ -15,9 +15,7 @@
* limitations under the License.
*/
-/**
- * $Id: BrokerData.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $
- */
+
package org.apache.rocketmq.common.protocol.route;
import java.util.HashMap;
@@ -29,6 +27,16 @@ public class BrokerData implements Comparable<BrokerData> {
private String brokerName;
private HashMap<Long/* brokerId */, String/* broker address */> brokerAddrs;
+ public BrokerData() {
+
+ }
+
+ public BrokerData(String cluster, String brokerName, HashMap<Long, String> brokerAddrs) {
+ this.cluster = cluster;
+ this.brokerName = brokerName;
+ this.brokerAddrs = brokerAddrs;
+ }
+
public String selectBrokerAddr() {
String value = this.brokerAddrs.get(MixAll.MASTER_ID);
if (null == value) {
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/c7961407/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
----------------------------------------------------------------------
diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
index 16b7847..5a953a9 100644
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
@@ -125,11 +125,7 @@ public class RouteInfoManager {
BrokerData brokerData = this.brokerAddrTable.get(brokerName);
if (null == brokerData) {
registerFirst = true;
- brokerData = new BrokerData();
- brokerData.setBrokerName(brokerName);
- HashMap<Long, String> brokerAddrs = new HashMap<Long, String>();
- brokerData.setBrokerAddrs(brokerAddrs);
-
+ brokerData = new BrokerData(clusterName, brokerName, new HashMap<Long, String>());
this.brokerAddrTable.put(brokerName, brokerData);
}
String oldAddr = brokerData.getBrokerAddrs().put(brokerId, brokerAddr);
@@ -381,9 +377,7 @@ public class RouteInfoManager {
for (String brokerName : brokerNameSet) {
BrokerData brokerData = this.brokerAddrTable.get(brokerName);
if (null != brokerData) {
- BrokerData brokerDataClone = new BrokerData();
- brokerDataClone.setBrokerName(brokerData.getBrokerName());
- brokerDataClone.setBrokerAddrs((HashMap<Long, String>) brokerData
+ BrokerData brokerDataClone = new BrokerData(brokerData.getCluster(), brokerData.getBrokerName(), (HashMap<Long, String>) brokerData
.getBrokerAddrs().clone());
brokerDataList.add(brokerDataClone);
foundBrokerData = true;
[29/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-194] log
appender support closes apache/incubator-rocketmq#101
Posted by do...@apache.org.
[ROCKETMQ-194] log appender support closes apache/incubator-rocketmq#101
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/37fbb7be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/37fbb7be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/37fbb7be
Branch: refs/heads/release-4.1.0-incubating
Commit: 37fbb7be8ae2a932f14006a2c08e3247143f75ea
Parents: 99ce40a
Author: lindzh <li...@163.com>
Authored: Fri May 26 16:12:16 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Fri May 26 16:12:16 2017 +0800
----------------------------------------------------------------------
logappender/pom.xml | 86 +++++++
.../logappender/common/ProducerInstance.java | 93 ++++++++
.../log4j/RocketmqLog4jAppender.java | 194 +++++++++++++++
.../log4j2/RocketmqLog4j2Appender.java | 233 +++++++++++++++++++
.../logback/RocketmqLogbackAppender.java | 183 +++++++++++++++
.../rocketmq/logappender/AbstractTestCase.java | 154 ++++++++++++
.../logappender/Log4jPropertiesTest.java | 32 +++
.../apache/rocketmq/logappender/Log4jTest.java | 43 ++++
.../rocketmq/logappender/Log4jXmlTest.java | 32 +++
.../rocketmq/logappender/LogbackTest.java | 54 +++++
.../apache/rocketmq/logappender/log4j2Test.java | 44 ++++
.../src/test/resources/log4j-example.properties | 38 +++
.../src/test/resources/log4j-example.xml | 62 +++++
.../src/test/resources/log4j2-example.xml | 41 ++++
.../src/test/resources/logback-example.xml | 89 +++++++
pom.xml | 11 +
style/rmq_checkstyle.xml | 6 +
17 files changed, 1395 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/pom.xml
----------------------------------------------------------------------
diff --git a/logappender/pom.xml b/logappender/pom.xml
new file mode 100644
index 0000000..5974c75
--- /dev/null
+++ b/logappender/pom.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-all</artifactId>
+ <version>4.1.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>rocketmq-logappender</artifactId>
+ <packaging>jar</packaging>
+ <name>rocketmq-logappender ${project.version}</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-namesrv</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-broker</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ </dependencies>
+
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/main/java/org/apache/rocketmq/logappender/common/ProducerInstance.java
----------------------------------------------------------------------
diff --git a/logappender/src/main/java/org/apache/rocketmq/logappender/common/ProducerInstance.java b/logappender/src/main/java/org/apache/rocketmq/logappender/common/ProducerInstance.java
new file mode 100644
index 0000000..669e30c
--- /dev/null
+++ b/logappender/src/main/java/org/apache/rocketmq/logappender/common/ProducerInstance.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender.common;
+
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.MQProducer;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Common Producer component
+ */
+public class ProducerInstance {
+
+ public static final String APPENDER_TYPE = "APPENDER_TYPE";
+
+ public static final String LOG4J_APPENDER = "LOG4J_APPENDER";
+
+ public static final String LOG4J2_APPENDER = "LOG4J2_APPENDER";
+
+ public static final String LOGBACK_APPENDER = "LOGBACK_APPENDER";
+
+ public static final String DEFAULT_GROUP = "rocketmq_appender";
+
+ private static ConcurrentHashMap<String, MQProducer> producerMap = new ConcurrentHashMap<String, MQProducer>();
+
+ private static String genKey(String nameServerAddress, String group) {
+ return nameServerAddress + "_" + group;
+ }
+
+
+ public static MQProducer getInstance(String nameServerAddress, String group) throws MQClientException {
+ if (group == null) {
+ group = DEFAULT_GROUP;
+ }
+
+ String genKey = genKey(nameServerAddress, group);
+ MQProducer p = producerMap.get(genKey);
+ if (p != null) {
+ return p;
+ }
+
+ DefaultMQProducer defaultMQProducer = new DefaultMQProducer(group);
+ defaultMQProducer.setNamesrvAddr(nameServerAddress);
+ MQProducer beforeProducer = null;
+ //cas put producer
+ beforeProducer = producerMap.putIfAbsent(genKey, defaultMQProducer);
+ if (beforeProducer != null) {
+ return beforeProducer;
+ }
+ defaultMQProducer.start();
+ return defaultMQProducer;
+ }
+
+
+ public static void removeAndClose(String nameServerAddress, String group) {
+ if (group == null) {
+ group = DEFAULT_GROUP;
+ }
+ String genKey = genKey(nameServerAddress, group);
+ MQProducer producer = producerMap.remove(genKey);
+
+ if (producer != null) {
+ producer.shutdown();
+ }
+ }
+
+ public static void closeAll() {
+ Set<Map.Entry<String, MQProducer>> entries = producerMap.entrySet();
+ for (Map.Entry<String, MQProducer> entry : entries) {
+ producerMap.remove(entry.getKey());
+ entry.getValue().shutdown();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/main/java/org/apache/rocketmq/logappender/log4j/RocketmqLog4jAppender.java
----------------------------------------------------------------------
diff --git a/logappender/src/main/java/org/apache/rocketmq/logappender/log4j/RocketmqLog4jAppender.java b/logappender/src/main/java/org/apache/rocketmq/logappender/log4j/RocketmqLog4jAppender.java
new file mode 100644
index 0000000..b2983b6
--- /dev/null
+++ b/logappender/src/main/java/org/apache/rocketmq/logappender/log4j/RocketmqLog4jAppender.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender.log4j;
+
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.logappender.common.ProducerInstance;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.helpers.LogLog;
+import org.apache.log4j.spi.ErrorCode;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.rocketmq.client.producer.MQProducer;
+
+/**
+ * Log4j Appender Component
+ */
+public class RocketmqLog4jAppender extends AppenderSkeleton {
+
+ /**
+ * Appended message tag define
+ */
+ private String tag;
+
+ /**
+ * Whitch topic to send log messages
+ */
+ private String topic;
+
+ private boolean locationInfo;
+
+ /**
+ * Log producer send instance
+ */
+ private MQProducer producer;
+
+ /**
+ * RocketMQ nameserver address
+ */
+ private String nameServerAddress;
+
+ /**
+ * Log producer group
+ */
+ private String producerGroup;
+
+ public RocketmqLog4jAppender() {
+ }
+
+
+ public void activateOptions() {
+ LogLog.debug("Getting initial context.");
+ if (!checkEntryConditions()) {
+ return;
+ }
+ try {
+ producer = ProducerInstance.getInstance(nameServerAddress, producerGroup);
+ } catch (Exception e) {
+ LogLog.error("activateOptions nameserver:" + nameServerAddress + " group:" + producerGroup + " " + e.getMessage());
+ }
+ }
+
+
+ /**
+ * Info,error,warn,callback method implementation
+ *
+ * @param event
+ */
+ public void append(LoggingEvent event) {
+ if (null == producer) {
+ return;
+ }
+ if (locationInfo) {
+ event.getLocationInformation();
+ }
+ byte[] data = this.layout.format(event).getBytes();
+ try {
+ Message msg = new Message(topic, tag, data);
+ msg.getProperties().put(ProducerInstance.APPENDER_TYPE, ProducerInstance.LOG4J_APPENDER);
+
+ //Send message and do not wait for the ack from the message broker.
+ producer.sendOneway(msg);
+ } catch (Exception e) {
+ String msg = new String(data);
+ errorHandler.error("Could not send message in RocketmqLog4jAppender [" + name + "].Message is :" + msg, e,
+ ErrorCode.GENERIC_FAILURE);
+ }
+ }
+
+ protected boolean checkEntryConditions() {
+ String fail = null;
+
+ if (this.topic == null) {
+ fail = "No topic";
+ } else if (this.tag == null) {
+ fail = "No tag";
+ }
+
+ if (fail != null) {
+ errorHandler.error(fail + " for RocketmqLog4jAppender named [" + name + "].");
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * When system exit,this method will be called to close resources
+ */
+ public synchronized void close() {
+ // The synchronized modifier avoids concurrent append and close operations
+
+ if (this.closed)
+ return;
+
+ LogLog.debug("Closing RocketmqLog4jAppender [" + name + "].");
+ this.closed = true;
+
+ try {
+ ProducerInstance.removeAndClose(this.nameServerAddress, this.producerGroup);
+ } catch (Exception e) {
+ LogLog.error("Closing RocketmqLog4jAppender [" + name + "] nameServerAddress:" + nameServerAddress + " group:" + producerGroup + " " + e.getMessage());
+ }
+ // Help garbage collection
+ producer = null;
+ }
+
+ public boolean requiresLayout() {
+ return true;
+ }
+
+ public String getTopic() {
+ return topic;
+ }
+
+
+ public void setTopic(String topic) {
+ this.topic = topic;
+ }
+
+ public String getTag() {
+ return tag;
+ }
+
+ public void setTag(String tag) {
+ this.tag = tag;
+ }
+
+ /**
+ * Returns value of the <b>LocationInfo</b> property which
+ * determines whether location (stack) info is sent to the remote
+ * subscriber.
+ */
+ public boolean isLocationInfo() {
+ return locationInfo;
+ }
+
+ /**
+ * If true, the information sent to the remote subscriber will
+ * include caller's location information. By default no location
+ * information is sent to the subscriber.
+ */
+ public void setLocationInfo(boolean locationInfo) {
+ this.locationInfo = locationInfo;
+ }
+
+ /**
+ * Returns the message producer,Only valid after
+ * activateOptions() method has been invoked.
+ */
+ protected MQProducer getProducer() {
+ return producer;
+ }
+
+ public void setNameServerAddress(String nameServerAddress) {
+ this.nameServerAddress = nameServerAddress;
+ }
+
+ public void setProducerGroup(String producerGroup) {
+ this.producerGroup = producerGroup;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/main/java/org/apache/rocketmq/logappender/log4j2/RocketmqLog4j2Appender.java
----------------------------------------------------------------------
diff --git a/logappender/src/main/java/org/apache/rocketmq/logappender/log4j2/RocketmqLog4j2Appender.java b/logappender/src/main/java/org/apache/rocketmq/logappender/log4j2/RocketmqLog4j2Appender.java
new file mode 100644
index 0000000..fb8341f
--- /dev/null
+++ b/logappender/src/main/java/org/apache/rocketmq/logappender/log4j2/RocketmqLog4j2Appender.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender.log4j2;
+
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.ErrorHandler;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.config.Node;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory;
+import org.apache.logging.log4j.core.config.plugins.PluginElement;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.logappender.common.ProducerInstance;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.plugins.validation.constraints.Required;
+import org.apache.logging.log4j.core.layout.SerializedLayout;
+import org.apache.rocketmq.client.producer.MQProducer;
+
+import java.io.Serializable;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Log4j2 Appender Component
+ */
+@Plugin(name = "RocketMQ",
+ category = Node.CATEGORY,
+ elementType = Appender.ELEMENT_TYPE,
+ printObject = true)
+public class RocketmqLog4j2Appender extends AbstractAppender {
+
+ /**
+ * RocketMQ nameserver address
+ */
+ private String nameServerAddress;
+
+ /**
+ * Log producer group
+ */
+ private String producerGroup;
+
+ /**
+ * Log producer send instance
+ */
+ private MQProducer producer;
+
+ /**
+ * Appended message tag define
+ */
+ private String tag;
+
+ /**
+ * Whitch topic to send log messages
+ */
+ private String topic;
+
+
+ protected RocketmqLog4j2Appender(String name, Filter filter, Layout<? extends Serializable> layout,
+ boolean ignoreExceptions, String nameServerAddress, String producerGroup,
+ String topic, String tag) {
+ super(name, filter, layout, ignoreExceptions);
+ this.producer = producer;
+ this.topic = topic;
+ this.tag = tag;
+ this.nameServerAddress = nameServerAddress;
+ this.producerGroup = producerGroup;
+ try {
+ this.producer = ProducerInstance.getInstance(this.nameServerAddress, this.producerGroup);
+ } catch (Exception e) {
+ ErrorHandler handler = this.getHandler();
+ if (handler != null) {
+ handler.error("Starting RocketmqLog4j2Appender [" + this.getName()
+ + "] nameServerAddress:" + nameServerAddress + " group:" + producerGroup + " " + e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Info,error,warn,callback method implementation
+ *
+ * @param event
+ */
+ public void append(LogEvent event) {
+ if (null == producer) {
+ return;
+ }
+ byte[] data = this.getLayout().toByteArray(event);
+ try {
+ Message msg = new Message(topic, tag, data);
+ msg.getProperties().put(ProducerInstance.APPENDER_TYPE, ProducerInstance.LOG4J2_APPENDER);
+
+ //Send message and do not wait for the ack from the message broker.
+ producer.sendOneway(msg);
+ } catch (Exception e) {
+ ErrorHandler handler = this.getHandler();
+ if (handler != null) {
+ String msg = new String(data);
+ handler.error("Could not send message in RocketmqLog4j2Appender [" + this.getName() + "].Message is : " + msg, e);
+ }
+
+ }
+ }
+
+ /**
+ * When system exit,this method will be called to close resources
+ *
+ * @param timeout
+ * @param timeUnit
+ * @return
+ */
+ public boolean stop(long timeout, TimeUnit timeUnit) {
+ this.setStopping();
+ try {
+ ProducerInstance.removeAndClose(this.nameServerAddress, this.producerGroup);
+ } catch (Exception e) {
+ ErrorHandler handler = this.getHandler();
+ if (handler != null) {
+ handler.error("Closeing RocketmqLog4j2Appender [" + this.getName()
+ + "] nameServerAddress:" + nameServerAddress + " group:" + producerGroup + " " + e.getMessage());
+ }
+ }
+
+ boolean stopped = super.stop(timeout, timeUnit, false);
+ this.setStopped();
+ return stopped;
+ }
+
+ /**
+ * Log4j2 builder creator
+ */
+ @PluginBuilderFactory
+ public static RocketmqLog4j2Appender.Builder newBuilder() {
+ return new RocketmqLog4j2Appender.Builder();
+ }
+
+ /**
+ * Log4j2 xml builder define
+ */
+ public static class Builder implements org.apache.logging.log4j.core.util.Builder<RocketmqLog4j2Appender> {
+
+ @PluginBuilderAttribute
+ @Required(message = "A name for the RocketmqLog4j2Appender must be specified")
+ private String name;
+
+ @PluginElement("Layout")
+ private Layout<? extends Serializable> layout;
+
+ @PluginElement("Filter")
+ private Filter filter;
+
+ @PluginBuilderAttribute
+ private boolean ignoreExceptions;
+
+ @PluginBuilderAttribute
+ private String tag;
+
+ @PluginBuilderAttribute
+ private String nameServerAddress;
+
+ @PluginBuilderAttribute
+ private String producerGroup;
+
+ @PluginBuilderAttribute
+ @Required(message = "A topic name must be specified")
+ private String topic;
+
+ private Builder() {
+ this.layout = SerializedLayout.createLayout();
+ this.ignoreExceptions = true;
+ }
+
+ public RocketmqLog4j2Appender.Builder setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public RocketmqLog4j2Appender.Builder setLayout(Layout<? extends Serializable> layout) {
+ this.layout = layout;
+ return this;
+ }
+
+ public RocketmqLog4j2Appender.Builder setFilter(Filter filter) {
+ this.filter = filter;
+ return this;
+ }
+
+ public RocketmqLog4j2Appender.Builder setIgnoreExceptions(boolean ignoreExceptions) {
+ this.ignoreExceptions = ignoreExceptions;
+ return this;
+ }
+
+ public RocketmqLog4j2Appender.Builder setTag(final String tag) {
+ this.tag = tag;
+ return this;
+ }
+
+ public RocketmqLog4j2Appender.Builder setTopic(final String topic) {
+ this.topic = topic;
+ return this;
+ }
+
+ public RocketmqLog4j2Appender.Builder setNameServerAddress(String nameServerAddress) {
+ this.nameServerAddress = nameServerAddress;
+ return this;
+ }
+
+ public RocketmqLog4j2Appender.Builder setProducerGroup(String producerGroup) {
+ this.producerGroup = producerGroup;
+ return this;
+ }
+
+ public RocketmqLog4j2Appender build() {
+ return new RocketmqLog4j2Appender(name, filter, layout, ignoreExceptions,
+ nameServerAddress, producerGroup, topic, tag);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/main/java/org/apache/rocketmq/logappender/logback/RocketmqLogbackAppender.java
----------------------------------------------------------------------
diff --git a/logappender/src/main/java/org/apache/rocketmq/logappender/logback/RocketmqLogbackAppender.java b/logappender/src/main/java/org/apache/rocketmq/logappender/logback/RocketmqLogbackAppender.java
new file mode 100644
index 0000000..cb45522
--- /dev/null
+++ b/logappender/src/main/java/org/apache/rocketmq/logappender/logback/RocketmqLogbackAppender.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender.logback;
+
+import ch.qos.logback.classic.net.LoggingEventPreSerializationTransformer;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.AppenderBase;
+import ch.qos.logback.core.Layout;
+import ch.qos.logback.core.spi.PreSerializationTransformer;
+import ch.qos.logback.core.status.ErrorStatus;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.logappender.common.ProducerInstance;
+import org.apache.rocketmq.client.producer.MQProducer;
+
+/**
+ * Logback Appender Component
+ */
+public class RocketmqLogbackAppender extends AppenderBase<ILoggingEvent> {
+
+ /**
+ * Message tag define
+ */
+ private String tag;
+
+ /**
+ * Whitch topic to send log messages
+ */
+ private String topic;
+
+ /**
+ * RocketMQ nameserver address
+ */
+ private String nameServerAddress;
+
+ /**
+ * Log producer group
+ */
+ private String producerGroup;
+
+ /**
+ * Log producer send instance
+ */
+ private MQProducer producer;
+
+ private Layout layout;
+
+ private PreSerializationTransformer<ILoggingEvent> pst = new LoggingEventPreSerializationTransformer();
+
+ /**
+ * Info,error,warn,callback method implementation
+ *
+ * @param event
+ */
+ @Override
+ protected void append(ILoggingEvent event) {
+ if (!isStarted()) {
+ return;
+ }
+ String logStr = this.layout.doLayout(event);
+ try {
+ Message msg = new Message(topic, tag, logStr.getBytes());
+ msg.getProperties().put(ProducerInstance.APPENDER_TYPE, ProducerInstance.LOGBACK_APPENDER);
+
+ //Send message and do not wait for the ack from the message broker.
+ producer.sendOneway(msg);
+ } catch (Exception e) {
+ addError("Could not send message in RocketmqLogbackAppender [" + name + "]. Message is : " + logStr, e);
+ }
+ }
+
+ /**
+ * Options are activated and become effective only after calling this method.
+ */
+ public void start() {
+ int errors = 0;
+
+ if (this.layout == null) {
+ addStatus(new ErrorStatus("No layout set for the RocketmqLogbackAppender named \"" + name + "\".", this));
+ errors++;
+ }
+
+ if (errors > 0 || !checkEntryConditions()) {
+ return;
+ }
+ try {
+ producer = ProducerInstance.getInstance(nameServerAddress, producerGroup);
+ } catch (Exception e) {
+ addError("Starting RocketmqLogbackAppender [" + this.getName()
+ + "] nameServerAddress:" + nameServerAddress + " group:" + producerGroup + " " + e.getMessage());
+ }
+ if (producer != null) {
+ super.start();
+ }
+ }
+
+ /**
+ * When system exit,this method will be called to close resources
+ */
+ public synchronized void stop() {
+ // The synchronized modifier avoids concurrent append and close operations
+ if (!this.started) {
+ return;
+ }
+
+ this.started = false;
+
+ try {
+ ProducerInstance.removeAndClose(this.nameServerAddress, this.producerGroup);
+ } catch (Exception e) {
+ addError("Closeing RocketmqLogbackAppender [" + this.getName()
+ + "] nameServerAddress:" + nameServerAddress + " group:" + producerGroup + " " + e.getMessage());
+ }
+
+ // Help garbage collection
+ producer = null;
+ }
+
+ protected boolean checkEntryConditions() {
+ String fail = null;
+
+ if (this.topic == null) {
+ fail = "No topic";
+ }
+
+ if (fail != null) {
+ addError(fail + " for RocketmqLogbackAppender named [" + name + "].");
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+
+ public Layout getLayout() {
+ return this.layout;
+ }
+
+ /**
+ * Set the pattern layout to format the log.
+ */
+ public void setLayout(Layout layout) {
+ this.layout = layout;
+ }
+
+ public String getTag() {
+ return tag;
+ }
+
+
+ public void setTag(String tag) {
+ this.tag = tag;
+ }
+
+ public String getTopic() {
+ return topic;
+ }
+
+ public void setTopic(String topic) {
+ this.topic = topic;
+ }
+
+ public void setNameServerAddress(String nameServerAddress) {
+ this.nameServerAddress = nameServerAddress;
+ }
+
+ public void setProducerGroup(String producerGroup) {
+ this.producerGroup = producerGroup;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/java/org/apache/rocketmq/logappender/AbstractTestCase.java
----------------------------------------------------------------------
diff --git a/logappender/src/test/java/org/apache/rocketmq/logappender/AbstractTestCase.java b/logappender/src/test/java/org/apache/rocketmq/logappender/AbstractTestCase.java
new file mode 100644
index 0000000..d3e2f8a
--- /dev/null
+++ b/logappender/src/test/java/org/apache/rocketmq/logappender/AbstractTestCase.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MQVersion;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.namesrv.NamesrvConfig;
+import org.apache.rocketmq.logappender.common.ProducerInstance;
+import org.apache.rocketmq.namesrv.NamesrvController;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Basic test rocketmq broker and name server init
+ */
+public class AbstractTestCase {
+
+ private static String nameServer = "localhost:9876";
+
+ private static NamesrvController namesrvController;
+
+ private static BrokerController brokerController;
+
+ private static String topic = "TopicTest";
+
+ @BeforeClass
+ public static void startRocketmqService() throws Exception {
+
+ startNamesrv();
+
+ startBroker();
+ }
+
+ /**
+ * Start rocketmq name server
+ * @throws Exception
+ */
+ private static void startNamesrv() throws Exception {
+
+ NamesrvConfig namesrvConfig = new NamesrvConfig();
+ NettyServerConfig nettyServerConfig = new NettyServerConfig();
+ nettyServerConfig.setListenPort(9876);
+
+ namesrvController = new NamesrvController(namesrvConfig, nettyServerConfig);
+ boolean initResult = namesrvController.initialize();
+ if (!initResult) {
+ namesrvController.shutdown();
+ throw new Exception();
+ }
+ namesrvController.start();
+ }
+
+ /**
+ * Start rocketmq broker service
+ * @throws Exception
+ */
+ private static void startBroker() throws Exception {
+
+ System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));
+
+ BrokerConfig brokerConfig = new BrokerConfig();
+ brokerConfig.setNamesrvAddr(nameServer);
+ brokerConfig.setBrokerId(MixAll.MASTER_ID);
+ NettyServerConfig nettyServerConfig = new NettyServerConfig();
+ nettyServerConfig.setListenPort(10911);
+ NettyClientConfig nettyClientConfig = new NettyClientConfig();
+ MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+
+ brokerController = new BrokerController(brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig);
+ boolean initResult = brokerController.initialize();
+ if (!initResult) {
+ brokerController.shutdown();
+ throw new Exception();
+ }
+ brokerController.start();
+ }
+
+ @AfterClass
+ public static void stop() {
+ ProducerInstance.closeAll();
+ if (brokerController != null) {
+ brokerController.shutdown();
+ }
+
+ if (namesrvController != null) {
+ namesrvController.shutdown();
+ }
+ }
+
+ protected int consumeMessages(int count,final String key,int timeout) throws MQClientException, InterruptedException {
+
+ final AtomicInteger cc = new AtomicInteger(0);
+ final CountDownLatch countDownLatch = new CountDownLatch(count);
+
+ DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("hello");
+ consumer.setNamesrvAddr(nameServer);
+ consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
+ consumer.subscribe(topic, "*");
+
+ consumer.registerMessageListener(new MessageListenerConcurrently() {
+
+ @Override
+ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
+ ConsumeConcurrentlyContext context) {
+ for (MessageExt msg : msgs) {
+ String body = new String(msg.getBody());
+ if(key==null||body.contains(key)){
+ countDownLatch.countDown();
+ cc.incrementAndGet();
+ continue;
+ }
+ }
+ return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+ }
+ });
+ consumer.start();
+ countDownLatch.await(timeout, TimeUnit.SECONDS);
+ consumer.shutdown();
+ return cc.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jPropertiesTest.java
----------------------------------------------------------------------
diff --git a/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jPropertiesTest.java b/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jPropertiesTest.java
new file mode 100644
index 0000000..8675230
--- /dev/null
+++ b/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jPropertiesTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender;
+
+import org.apache.log4j.PropertyConfigurator;
+
+public class Log4jPropertiesTest extends Log4jTest {
+
+ @Override
+ public void init() {
+ PropertyConfigurator.configure("src/test/resources/log4j-example.properties");
+ }
+
+ @Override
+ public String getType() {
+ return "properties";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jTest.java
----------------------------------------------------------------------
diff --git a/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jTest.java b/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jTest.java
new file mode 100644
index 0000000..75f9bf2
--- /dev/null
+++ b/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender;
+
+import org.apache.log4j.Logger;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public abstract class Log4jTest extends AbstractTestCase{
+
+ @Before
+ public abstract void init();
+
+ public abstract String getType();
+
+ @Test
+ public void testLog4j() throws InterruptedException, MQClientException {
+ Logger logger = Logger.getLogger("testLogger");
+ for (int i = 0; i < 50; i++) {
+ logger.info("log4j " + this.getType() + " simple test message " + i);
+ }
+ int received = consumeMessages(30, "log4j",30);
+ Assert.assertTrue(received>20);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jXmlTest.java
----------------------------------------------------------------------
diff --git a/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jXmlTest.java b/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jXmlTest.java
new file mode 100644
index 0000000..6743f7c
--- /dev/null
+++ b/logappender/src/test/java/org/apache/rocketmq/logappender/Log4jXmlTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender;
+
+import org.apache.log4j.xml.DOMConfigurator;
+
+public class Log4jXmlTest extends Log4jTest {
+
+ @Override
+ public void init() {
+ DOMConfigurator.configure("src/test/resources/log4j-example.xml");
+ }
+
+ @Override
+ public String getType() {
+ return "xml";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/java/org/apache/rocketmq/logappender/LogbackTest.java
----------------------------------------------------------------------
diff --git a/logappender/src/test/java/org/apache/rocketmq/logappender/LogbackTest.java b/logappender/src/test/java/org/apache/rocketmq/logappender/LogbackTest.java
new file mode 100644
index 0000000..15a21a3
--- /dev/null
+++ b/logappender/src/test/java/org/apache/rocketmq/logappender/LogbackTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.core.joran.spi.JoranException;
+import ch.qos.logback.core.util.StatusPrinter;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+public class LogbackTest extends AbstractTestCase{
+
+ @Before
+ public void init() throws JoranException {
+ LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
+ JoranConfigurator configurator = new JoranConfigurator();
+ configurator.setContext(lc);
+ lc.reset();
+ configurator.doConfigure(new File("src/test/resources/logback-example.xml"));
+ StatusPrinter.printInCaseOfErrorsOrWarnings(lc);
+ }
+
+
+ @Test
+ public void testLogback() throws InterruptedException, MQClientException {
+ Logger logger = LoggerFactory.getLogger("testLogger");
+ for (int i = 0; i < 50; i++) {
+ logger.info("logback test message " + i);
+ }
+ int received = consumeMessages(30, "logback",30);
+ Assert.assertTrue(received>20);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/java/org/apache/rocketmq/logappender/log4j2Test.java
----------------------------------------------------------------------
diff --git a/logappender/src/test/java/org/apache/rocketmq/logappender/log4j2Test.java b/logappender/src/test/java/org/apache/rocketmq/logappender/log4j2Test.java
new file mode 100644
index 0000000..75ba523
--- /dev/null
+++ b/logappender/src/test/java/org/apache/rocketmq/logappender/log4j2Test.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.logappender;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class log4j2Test extends AbstractTestCase{
+
+ @Before
+ public void init() {
+ Configurator.initialize("log4j2", "src/test/resources/log4j2-example.xml");
+ }
+
+
+ @Test
+ public void testLog4j2() throws InterruptedException, MQClientException {
+ Logger logger = LogManager.getLogger("test");
+ for (int i = 0; i < 50; i++) {
+ logger.info("log4j2 log message " + i);
+ }
+ int received = consumeMessages(30, "log4j2",30);
+ Assert.assertTrue(received>20);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/resources/log4j-example.properties
----------------------------------------------------------------------
diff --git a/logappender/src/test/resources/log4j-example.properties b/logappender/src/test/resources/log4j-example.properties
new file mode 100644
index 0000000..b4e8114
--- /dev/null
+++ b/logappender/src/test/resources/log4j-example.properties
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO,stdout
+
+log4j.logger.testLogger=INFO,mq
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d %-4r [%t] (%F:%L) %-5p - %m%n
+
+log4j.appender.store=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.store.File=${user.home}/logs/rocketmqlogs/appender.log
+log4j.appender.store.Append=true
+log4j.appender.store.DatePattern ='_'yyyy-MM-dd'.log'
+log4j.appender.store.layout=org.apache.log4j.PatternLayout
+log4j.appender.store.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-4r [%t] (%F:%L) %-5p - %m%n
+
+log4j.appender.mq=org.apache.rocketmq.logappender.log4j.RocketmqLog4jAppender
+log4j.appender.mq.Tag=log
+log4j.appender.mq.Topic=TopicTest
+log4j.appender.mq.ProducerGroup=log4jp
+log4j.appender.mq.NameServerAddress=127.0.0.1:9876
+log4j.appender.mq.layout=org.apache.log4j.PatternLayout
+log4j.appender.mq.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-4r [%t] (%F:%L) %-5p - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/resources/log4j-example.xml
----------------------------------------------------------------------
diff --git a/logappender/src/test/resources/log4j-example.xml b/logappender/src/test/resources/log4j-example.xml
new file mode 100644
index 0000000..e58bcb0
--- /dev/null
+++ b/logappender/src/test/resources/log4j-example.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="consoleAppender" class="org.apache.log4j.ConsoleAppender">
+ <param name="Encoding" value="UTF-8" />
+ <param name="Target" value="System.out" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss},%d %-4r [%t] (%F:%L) %-5p - %m%n" />
+ </layout>
+ </appender>
+
+ <appender name="mqAppender1" class="org.apache.rocketmq.logappender.log4j.RocketmqLog4jAppender">
+ <param name="Tag" value="log1" />
+ <param name="Topic" value="TopicTest" />
+ <param name="ProducerGroup" value="log4jxml" />
+ <param name="NameServerAddress" value="127.0.0.1:9876"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss}-%p %t %c - %m%n" />
+ </layout>
+ </appender>
+
+ <appender name="mqAsyncAppender1" class="org.apache.log4j.AsyncAppender">
+ <param name="BufferSize" value="1024" />
+ <param name="Blocking" value="false" />
+ <appender-ref ref="mqAppender1"/>
+ </appender>
+
+ <logger name="testLogger" additivity="false">
+ <level value="INFO" />
+ <appender-ref ref="mqAsyncAppender1" />
+ <appender-ref ref="consoleAppender" />
+ </logger>
+
+ <logger name="consoleLogger" additivity="false">
+ <level value="INFO" />
+ <appender-ref ref="consoleAppender" />
+ </logger>
+
+
+ <root>
+ <level value="INFO" />
+ <appender-ref ref="consoleAppender"/>
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/resources/log4j2-example.xml
----------------------------------------------------------------------
diff --git a/logappender/src/test/resources/log4j2-example.xml b/logappender/src/test/resources/log4j2-example.xml
new file mode 100644
index 0000000..358d40e
--- /dev/null
+++ b/logappender/src/test/resources/log4j2-example.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<Configuration status="warn" name="Rocketmq">
+<Appenders>
+ <RocketMQ name="rocketmqAppender" producerGroup="log4j2" nameServerAddress="127.0.0.1:9876"
+ topic="TopicTest" tag="log">
+ <PatternLayout pattern="%d [%p] hahahah %c %m%n"/>
+ </RocketMQ>
+
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+ </Console>
+</Appenders>
+<Loggers>
+
+ <Logger name="rocketmqLogger" level="info">
+ <AppenderRef ref="rocketmqAppender"/>
+ </Logger>
+
+ <Root level="debug">
+ <AppenderRef ref="Console"/>
+ <AppenderRef ref="rocketmqAppender"/>
+ </Root>
+</Loggers>
+</Configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/logappender/src/test/resources/logback-example.xml
----------------------------------------------------------------------
diff --git a/logappender/src/test/resources/logback-example.xml b/logappender/src/test/resources/logback-example.xml
new file mode 100644
index 0000000..21b5434
--- /dev/null
+++ b/logappender/src/test/resources/logback-example.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<configuration>
+
+ <appender name="system" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${user.home}/logs/simple/system.log</file>
+ <append>true</append>
+ <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${user.home}/logs/simple/system.%i.log
+ </fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>30</maxIndex>
+ </rollingPolicy>
+ <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>100MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <pattern>%date %p %t - %m%n</pattern>
+ <charset class="java.nio.charset.Charset">UTF-8</charset>
+ </encoder>
+ </appender>
+
+ <appender name="consoleAppender" class="ch.qos.logback.core.ConsoleAppender">
+ <target>System.out</target>
+ <encoder>
+ <pattern>%date %p %t - %m%n</pattern>
+ <charset class="java.nio.charset.Charset">UTF-8</charset>
+ </encoder>
+ </appender>
+
+ <appender name="dailyAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${user.home}/logs/simple/daily.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${user.home}/logs/simple/daily.log.%d{yyyy-MM-dd_HH}</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <encoder>
+ <pattern>%date %p %t - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="mqAppender1" class="org.apache.rocketmq.logappender.logback.RocketmqLogbackAppender">
+ <tag>log1</tag>
+ <topic>TopicTest</topic>
+ <producerGroup>logback</producerGroup>
+ <nameServerAddress>127.0.0.1:9876</nameServerAddress>
+ <layout>
+ <pattern>%date %p %t - %m%n</pattern>
+ </layout>
+ </appender>
+
+ <appender name="mqAsyncAppender1" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>1024</queueSize>
+ <discardingThreshold>80</discardingThreshold>
+ <maxFlushTime>2000</maxFlushTime>
+ <neverBlock>true</neverBlock>
+ <appender-ref ref="mqAppender1"/>
+ </appender>
+
+ <root>
+ <level value="debug"/>
+ <appender-ref ref="consoleAppender"/>
+ </root>
+
+ <logger name="systemLogger" level="debug" additivity="false">
+ <appender-ref ref="system"/>
+ </logger>
+
+ <logger name="testLogger" level="debug" additivity="false">
+ <appender-ref ref="mqAsyncAppender1"/>
+ <appender-ref ref="consoleAppender"/>
+ </logger>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 25e4c84..c60c93c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -175,6 +175,7 @@
<module>store</module>
<module>namesrv</module>
<module>remoting</module>
+ <module>logappender</module>
<module>example</module>
<module>filtersrv</module>
<module>srvutil</module>
@@ -623,6 +624,16 @@
<artifactId>openmessaging-api</artifactId>
<version>0.1.0-alpha</version>
</dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>1.2.17</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.7</version>
+ </dependency>
</dependencies>
</dependencyManagement>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/37fbb7be/style/rmq_checkstyle.xml
----------------------------------------------------------------------
diff --git a/style/rmq_checkstyle.xml b/style/rmq_checkstyle.xml
index 2872eb7..6ec2ad0 100644
--- a/style/rmq_checkstyle.xml
+++ b/style/rmq_checkstyle.xml
@@ -30,6 +30,12 @@
<!-- header -->
<module name="RegexpHeader">
<property name="header" value="/\*\nLicensed to the Apache Software Foundation*"/>
+ <property name="fileExtensions" value="java"/>
+ </module>
+
+ <module name="RegexpHeader">
+ <property name="header" value="#[\s]*Licensed to the Apache Software Foundation*"/>
+ <property name="fileExtensions" value="properties"/>
</module>
<module name="RegexpSingleline">
[09/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-121]Support
message filtering based on SQL92 closes apache/incubator-rocketmq#82
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/message/Message.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/message/Message.java b/common/src/main/java/org/apache/rocketmq/common/message/Message.java
index f3bff83..2c81f5c 100644
--- a/common/src/main/java/org/apache/rocketmq/common/message/Message.java
+++ b/common/src/main/java/org/apache/rocketmq/common/message/Message.java
@@ -81,6 +81,12 @@ public class Message implements Serializable {
throw new RuntimeException(String.format(
"The Property<%s> is used by system, input another please", name));
}
+ if (value == null || value == "" || value.trim() == ""
+ || name == null || name == "" || name.trim() == "") {
+ throw new IllegalArgumentException(
+ "The name or value of property can not be null or blank string!"
+ );
+ }
this.putProperty(name, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
index 90b837a..e41ec9d 100644
--- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
+++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
@@ -41,6 +41,20 @@ public class MessageDecoder {
public final static int MESSAGE_MAGIC_CODE = 0xAABBCCDD ^ 1880681586 + 8;
public static final char NAME_VALUE_SEPARATOR = 1;
public static final char PROPERTY_SEPARATOR = 2;
+ public static final int BODY_SIZE_POSITION = 4 // 1 TOTALSIZE
+ + 4 // 2 MAGICCODE
+ + 4 // 3 BODYCRC
+ + 4 // 4 QUEUEID
+ + 4 // 5 FLAG
+ + 8 // 6 QUEUEOFFSET
+ + 8 // 7 PHYSICALOFFSET
+ + 4 // 8 SYSFLAG
+ + 8 // 9 BORNTIMESTAMP
+ + 8 // 10 BORNHOST
+ + 8 // 11 STORETIMESTAMP
+ + 8 // 12 STOREHOSTADDRESS
+ + 4 // 13 RECONSUMETIMES
+ + 8; // 14 Prepared Transaction Offset
public static String createMessageId(final ByteBuffer input, final ByteBuffer addr, final long offset) {
input.flip();
@@ -80,6 +94,31 @@ public class MessageDecoder {
return new MessageId(address, offset);
}
+ /**
+ * Just decode properties from msg buffer.
+ *
+ * @param byteBuffer msg commit log buffer.
+ * @return
+ */
+ public static Map<String, String> decodeProperties(java.nio.ByteBuffer byteBuffer) {
+ int topicLengthPosition = BODY_SIZE_POSITION + 4 + byteBuffer.getInt(BODY_SIZE_POSITION);
+
+ byte topicLength = byteBuffer.get(topicLengthPosition);
+
+ short propertiesLength = byteBuffer.getShort(topicLengthPosition + 1 + topicLength);
+
+ byteBuffer.position(topicLengthPosition + 1 + topicLength + 2);
+
+ if (propertiesLength > 0) {
+ byte[] properties = new byte[propertiesLength];
+ byteBuffer.get(properties);
+ String propertiesString = new String(properties, CHARSET_UTF8);
+ Map<String, String> map = string2messageProperties(propertiesString);
+ return map;
+ }
+ return null;
+ }
+
public static MessageExt decode(java.nio.ByteBuffer byteBuffer) {
return decode(byteBuffer, true, true, false);
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java b/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java
index 74fd965..990e748 100644
--- a/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java
+++ b/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java
@@ -88,7 +88,7 @@ public class TopAddressing {
if (verbose) {
String errorMsg =
- "connect to " + url + " failed, maybe the domain name " + MixAll.WS_DOMAIN_NAME + " not bind in /etc/hosts";
+ "connect to " + url + " failed, maybe the domain name " + MixAll.getWSAddr() + " not bind in /etc/hosts";
errorMsg += FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL);
log.warn(errorMsg);
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
index c6b0925..6f132f7 100644
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
@@ -68,6 +68,8 @@ public class RequestCode {
public static final int GET_ALL_DELAY_OFFSET = 45;
+ public static final int CHECK_CLIENT_CONFIG = 46;
+
public static final int PUT_KV_CONFIG = 100;
public static final int GET_KV_CONFIG = 101;
@@ -162,4 +164,6 @@ public class RequestCode {
public static final int SEND_BATCH_MESSAGE = 320;
+
+ public static final int QUERY_CONSUME_QUEUE = 321;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java
index 90b182b..f62c4ea 100644
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java
@@ -53,6 +53,10 @@ public class ResponseCode extends RemotingSysResponseCode {
public static final int SUBSCRIPTION_GROUP_NOT_EXIST = 26;
+ public static final int FILTER_DATA_NOT_EXIST = 27;
+
+ public static final int FILTER_DATA_NOT_LATEST = 28;
+
public static final int TRANSACTION_SHOULD_COMMIT = 200;
public static final int TRANSACTION_SHOULD_ROLLBACK = 201;
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/protocol/body/CheckClientRequestBody.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/CheckClientRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/CheckClientRequestBody.java
new file mode 100644
index 0000000..a78ce55
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/CheckClientRequestBody.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.protocol.body;
+
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+
+public class CheckClientRequestBody extends RemotingSerializable {
+
+ private String clientId;
+ private String group;
+ private SubscriptionData subscriptionData;
+
+ public String getClientId() {
+ return clientId;
+ }
+
+ public void setClientId(String clientId) {
+ this.clientId = clientId;
+ }
+
+ public String getGroup() {
+ return group;
+ }
+
+ public void setGroup(String group) {
+ this.group = group;
+ }
+
+ public SubscriptionData getSubscriptionData() {
+ return subscriptionData;
+ }
+
+ public void setSubscriptionData(SubscriptionData subscriptionData) {
+ this.subscriptionData = subscriptionData;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumeQueueData.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumeQueueData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumeQueueData.java
new file mode 100644
index 0000000..7268dcd
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumeQueueData.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.protocol.body;
+
+public class ConsumeQueueData {
+
+ private long physicOffset;
+ private int physicSize;
+ private long tagsCode;
+ private String extendDataJson;
+ private String bitMap;
+ private boolean eval;
+ private String msg;
+
+ public long getPhysicOffset() {
+ return physicOffset;
+ }
+
+ public void setPhysicOffset(long physicOffset) {
+ this.physicOffset = physicOffset;
+ }
+
+ public int getPhysicSize() {
+ return physicSize;
+ }
+
+ public void setPhysicSize(int physicSize) {
+ this.physicSize = physicSize;
+ }
+
+ public long getTagsCode() {
+ return tagsCode;
+ }
+
+ public void setTagsCode(long tagsCode) {
+ this.tagsCode = tagsCode;
+ }
+
+ public String getExtendDataJson() {
+ return extendDataJson;
+ }
+
+ public void setExtendDataJson(String extendDataJson) {
+ this.extendDataJson = extendDataJson;
+ }
+
+ public String getBitMap() {
+ return bitMap;
+ }
+
+ public void setBitMap(String bitMap) {
+ this.bitMap = bitMap;
+ }
+
+ public boolean isEval() {
+ return eval;
+ }
+
+ public void setEval(boolean eval) {
+ this.eval = eval;
+ }
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public void setMsg(String msg) {
+ this.msg = msg;
+ }
+
+ @Override
+ public String toString() {
+ return "ConsumeQueueData{" +
+ "physicOffset=" + physicOffset +
+ ", physicSize=" + physicSize +
+ ", tagsCode=" + tagsCode +
+ ", extendDataJson='" + extendDataJson + '\'' +
+ ", bitMap='" + bitMap + '\'' +
+ ", eval=" + eval +
+ ", msg='" + msg + '\'' +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/protocol/body/QueryConsumeQueueResponseBody.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/QueryConsumeQueueResponseBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/QueryConsumeQueueResponseBody.java
new file mode 100644
index 0000000..be93da9
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/QueryConsumeQueueResponseBody.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.protocol.body;
+
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+
+import java.util.List;
+
+public class QueryConsumeQueueResponseBody extends RemotingSerializable {
+
+ private SubscriptionData subscriptionData;
+ private String filterData;
+ private List<ConsumeQueueData> queueData;
+ private long maxQueueIndex;
+ private long minQueueIndex;
+
+ public SubscriptionData getSubscriptionData() {
+ return subscriptionData;
+ }
+
+ public void setSubscriptionData(SubscriptionData subscriptionData) {
+ this.subscriptionData = subscriptionData;
+ }
+
+ public String getFilterData() {
+ return filterData;
+ }
+
+ public void setFilterData(String filterData) {
+ this.filterData = filterData;
+ }
+
+ public List<ConsumeQueueData> getQueueData() {
+ return queueData;
+ }
+
+ public void setQueueData(List<ConsumeQueueData> queueData) {
+ this.queueData = queueData;
+ }
+
+ public long getMaxQueueIndex() {
+ return maxQueueIndex;
+ }
+
+ public void setMaxQueueIndex(long maxQueueIndex) {
+ this.maxQueueIndex = maxQueueIndex;
+ }
+
+ public long getMinQueueIndex() {
+ return minQueueIndex;
+ }
+
+ public void setMinQueueIndex(long minQueueIndex) {
+ this.minQueueIndex = minQueueIndex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
index 8a59213..106e89e 100644
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
@@ -46,6 +46,7 @@ public class PullMessageRequestHeader implements CommandCustomHeader {
private String subscription;
@CFNotNull
private Long subVersion;
+ private String expressionType;
@Override
public void checkFields() throws RemotingCommandException {
@@ -130,4 +131,12 @@ public class PullMessageRequestHeader implements CommandCustomHeader {
public void setSubVersion(Long subVersion) {
this.subVersion = subVersion;
}
+
+ public String getExpressionType() {
+ return expressionType;
+ }
+
+ public void setExpressionType(String expressionType) {
+ this.expressionType = expressionType;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryConsumeQueueRequestHeader.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryConsumeQueueRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryConsumeQueueRequestHeader.java
new file mode 100644
index 0000000..642fe17
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryConsumeQueueRequestHeader.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.protocol.header;
+
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+
+public class QueryConsumeQueueRequestHeader implements CommandCustomHeader {
+
+ private String topic;
+ private int queueId;
+ private long index;
+ private int count;
+ private String consumerGroup;
+
+ public String getTopic() {
+ return topic;
+ }
+
+ public void setTopic(String topic) {
+ this.topic = topic;
+ }
+
+ public int getQueueId() {
+ return queueId;
+ }
+
+ public void setQueueId(int queueId) {
+ this.queueId = queueId;
+ }
+
+ public long getIndex() {
+ return index;
+ }
+
+ public void setIndex(long index) {
+ this.index = index;
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ public void setCount(int count) {
+ this.count = count;
+ }
+
+ public String getConsumerGroup() {
+ return consumerGroup;
+ }
+
+ public void setConsumerGroup(String consumerGroup) {
+ this.consumerGroup = consumerGroup;
+ }
+
+ @Override
+ public void checkFields() throws RemotingCommandException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/main/java/org/apache/rocketmq/common/protocol/heartbeat/SubscriptionData.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/heartbeat/SubscriptionData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/heartbeat/SubscriptionData.java
index 81f5954..e456b7e 100644
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/heartbeat/SubscriptionData.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/heartbeat/SubscriptionData.java
@@ -32,6 +32,7 @@ public class SubscriptionData implements Comparable<SubscriptionData> {
private Set<String> tagsSet = new HashSet<String>();
private Set<Integer> codeSet = new HashSet<Integer>();
private long subVersion = System.currentTimeMillis();
+ private String expressionType;
@JSONField(serialize = false)
private String filterClassSource;
@@ -102,6 +103,14 @@ public class SubscriptionData implements Comparable<SubscriptionData> {
this.classFilterMode = classFilterMode;
}
+ public String getExpressionType() {
+ return expressionType;
+ }
+
+ public void setExpressionType(String expressionType) {
+ this.expressionType = expressionType;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
@@ -111,6 +120,7 @@ public class SubscriptionData implements Comparable<SubscriptionData> {
result = prime * result + ((subString == null) ? 0 : subString.hashCode());
result = prime * result + ((tagsSet == null) ? 0 : tagsSet.hashCode());
result = prime * result + ((topic == null) ? 0 : topic.hashCode());
+ result = prime * result + ((expressionType == null) ? 0 : expressionType.hashCode());
return result;
}
@@ -147,6 +157,11 @@ public class SubscriptionData implements Comparable<SubscriptionData> {
return false;
} else if (!topic.equals(other.topic))
return false;
+ if (expressionType == null) {
+ if (other.expressionType != null)
+ return false;
+ } else if (!expressionType.equals(other.expressionType))
+ return false;
return true;
}
@@ -154,7 +169,7 @@ public class SubscriptionData implements Comparable<SubscriptionData> {
public String toString() {
return "SubscriptionData [classFilterMode=" + classFilterMode + ", topic=" + topic + ", subString="
+ subString + ", tagsSet=" + tagsSet + ", codeSet=" + codeSet + ", subVersion=" + subVersion
- + "]";
+ + ", expressionType=" + expressionType + "]";
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/test/java/org/apache/rocketmq/common/filter/FilterAPITest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/rocketmq/common/filter/FilterAPITest.java b/common/src/test/java/org/apache/rocketmq/common/filter/FilterAPITest.java
index 5137f32..c5f8460 100644
--- a/common/src/test/java/org/apache/rocketmq/common/filter/FilterAPITest.java
+++ b/common/src/test/java/org/apache/rocketmq/common/filter/FilterAPITest.java
@@ -42,4 +42,53 @@ public class FilterAPITest {
}
assertThat(subscriptionData.getTagsSet()).isEqualTo(tagSet);
}
+
+ @Test
+ public void testBuildTagSome() {
+ try {
+ SubscriptionData subscriptionData = FilterAPI.build(
+ "TOPIC", "A || B", ExpressionType.TAG
+ );
+
+ assertThat(subscriptionData).isNotNull();
+ assertThat(subscriptionData.getTopic()).isEqualTo("TOPIC");
+ assertThat(subscriptionData.getSubString()).isEqualTo("A || B");
+ assertThat(ExpressionType.isTagType(subscriptionData.getExpressionType())).isTrue();
+
+ assertThat(subscriptionData.getTagsSet()).isNotNull();
+ assertThat(subscriptionData.getTagsSet()).containsExactly("A", "B");
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+ }
+
+ @Test
+ public void testBuildSQL() {
+ try {
+ SubscriptionData subscriptionData = FilterAPI.build(
+ "TOPIC", "a is not null", ExpressionType.SQL92
+ );
+
+ assertThat(subscriptionData).isNotNull();
+ assertThat(subscriptionData.getTopic()).isEqualTo("TOPIC");
+ assertThat(subscriptionData.getExpressionType()).isEqualTo(ExpressionType.SQL92);
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+ }
+
+ @Test
+ public void testBuildSQLWithNullSubString() {
+ try {
+ FilterAPI.build(
+ "TOPIC", null, ExpressionType.SQL92
+ );
+
+ assertThat(Boolean.FALSE).isTrue();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java b/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java
new file mode 100644
index 0000000..d14d6b0
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.message;
+
+import org.junit.Test;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class MessageDecoderTest {
+
+ @Test
+ public void testDecodeProperties() {
+ MessageExt messageExt = new MessageExt();
+
+ messageExt.setMsgId("645100FA00002A9F000000489A3AA09E");
+ messageExt.setTopic("abc");
+ messageExt.setBody("hello!q!".getBytes());
+ try {
+ messageExt.setBornHost(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0));
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+ messageExt.setBornTimestamp(System.currentTimeMillis());
+ messageExt.setCommitLogOffset(123456);
+ messageExt.setPreparedTransactionOffset(0);
+ messageExt.setQueueId(0);
+ messageExt.setQueueOffset(123);
+ messageExt.setReconsumeTimes(0);
+ try {
+ messageExt.setStoreHost(new InetSocketAddress(InetAddress.getLocalHost(), 0));
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+
+ messageExt.putUserProperty("a", "123");
+ messageExt.putUserProperty("b", "hello");
+ messageExt.putUserProperty("c", "3.14");
+
+ byte[] msgBytes = new byte[0];
+ try {
+ msgBytes = MessageDecoder.encode(messageExt, false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertThat(Boolean.FALSE).isTrue();
+ }
+
+ ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length);
+ byteBuffer.put(msgBytes);
+
+ Map<String, String> properties = MessageDecoder.decodeProperties(byteBuffer);
+
+ assertThat(properties).isNotNull();
+ assertThat("123").isEqualTo(properties.get("a"));
+ assertThat("hello").isEqualTo(properties.get("b"));
+ assertThat("3.14").isEqualTo(properties.get("c"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/distribution/conf/logback_broker.xml
----------------------------------------------------------------------
diff --git a/distribution/conf/logback_broker.xml b/distribution/conf/logback_broker.xml
index 05c0ee4..dd5c63f 100644
--- a/distribution/conf/logback_broker.xml
+++ b/distribution/conf/logback_broker.xml
@@ -222,6 +222,29 @@
<appender-ref ref="RocketmqRebalanceLockAppender_inner"/>
</appender>
+ <appender name="RocketmqFilterAppender_inner"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${user.home}/logs/rocketmqlogs/filter.log</file>
+ <append>true</append>
+ <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/filter.%i.log
+ </fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>10</maxIndex>
+ </rollingPolicy>
+ <triggeringPolicy
+ class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>100MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+ <charset class="java.nio.charset.Charset">UTF-8</charset>
+ </encoder>
+ </appender>
+ <appender name="RocketmqFilterAppender" class="ch.qos.logback.classic.AsyncAppender">
+ <appender-ref ref="RocketmqFilterAppender_inner"/>
+ </appender>
+
<appender name="RocketmqStatsAppender"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${user.home}/logs/rocketmqlogs/stats.log</file>
@@ -321,6 +344,11 @@
<appender-ref ref="RocketmqCommercialAppender"/>
</logger>
+ <logger name="RocketmqFilter" additivity="false">
+ <level value="INFO"/>
+ <appender-ref ref="RocketmqFilterAppender"/>
+ </logger>
+
<root>
<level value="INFO"/>
<appender-ref ref="DefaultAppender"/>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/distribution/release.xml
----------------------------------------------------------------------
diff --git a/distribution/release.xml b/distribution/release.xml
index 2d3ec1e..9e4ef2a 100644
--- a/distribution/release.xml
+++ b/distribution/release.xml
@@ -67,6 +67,7 @@
<include>org.apache.rocketmq:rocketmq-namesrv</include>
<include>org.apache.rocketmq:rocketmq-filtersrv</include>
<include>org.apache.rocketmq:rocketmq-example</include>
+ <include>org.apache.rocketmq:rocketmq-filter</include>
</includes>
<binaries>
<outputDirectory>lib/</outputDirectory>
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
index 473e4c7..3e1b79b 100644
--- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
@@ -27,10 +27,13 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.MessageSelector;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.srvutil.ServerUtil;
@@ -46,12 +49,14 @@ public class Consumer {
final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "BenchmarkTest";
final String groupPrefix = commandLine.hasOption('g') ? commandLine.getOptionValue('g').trim() : "benchmark_consumer";
final String isPrefixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "true";
+ final String filterType = commandLine.hasOption('f') ? commandLine.getOptionValue('f').trim() : null;
+ final String expression = commandLine.hasOption('e') ? commandLine.getOptionValue('e').trim() : null;
String group = groupPrefix;
if (Boolean.parseBoolean(isPrefixEnable)) {
group = groupPrefix + "_" + Long.toString(System.currentTimeMillis() % 100);
}
- System.out.printf("topic %s group %s prefix %s%n", topic, group, isPrefixEnable);
+ System.out.printf("topic: %s, group: %s, prefix: %s, filterType: %s, expression: %s%n", topic, group, isPrefixEnable, filterType, expression);
final StatsBenchmarkConsumer statsBenchmarkConsumer = new StatsBenchmarkConsumer();
@@ -99,7 +104,21 @@ public class Consumer {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group);
consumer.setInstanceName(Long.toString(System.currentTimeMillis()));
- consumer.subscribe(topic, "*");
+ if (filterType == null || expression == null) {
+ consumer.subscribe(topic, "*");
+ } else {
+ if (ExpressionType.TAG.equals(filterType)) {
+ String expr = MixAll.file2String(expression);
+ System.out.printf("Expression: %s%n", expr);
+ consumer.subscribe(topic, MessageSelector.byTag(expr));
+ } else if (ExpressionType.SQL92.equals(filterType)) {
+ String expr = MixAll.file2String(expression);
+ System.out.printf("Expression: %s%n", expr);
+ consumer.subscribe(topic, MessageSelector.bySql(expr));
+ } else {
+ throw new IllegalArgumentException("Not support filter type! " + filterType);
+ }
+ }
consumer.registerMessageListener(new MessageListenerConcurrently() {
@Override
@@ -142,6 +161,14 @@ public class Consumer {
opt.setRequired(false);
options.addOption(opt);
+ opt = new Option("f", "filterType", true, "TAG, SQL92");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("e", "expression", true, "filter expression content file path.ie: ./test/expr");
+ opt.setRequired(false);
+ options.addOption(opt);
+
return options;
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
index 50d750d..2d8d0f6 100644
--- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
@@ -18,11 +18,13 @@ package org.apache.rocketmq.example.benchmark;
import java.io.UnsupportedEncodingException;
import java.util.LinkedList;
+import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
@@ -50,13 +52,12 @@ public class Producer {
final int threadCount = commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 64;
final int messageSize = commandLine.hasOption('s') ? Integer.parseInt(commandLine.getOptionValue('s')) : 128;
final boolean keyEnable = commandLine.hasOption('k') && Boolean.parseBoolean(commandLine.getOptionValue('k'));
+ final int propertySize = commandLine.hasOption('p') ? Integer.parseInt(commandLine.getOptionValue('p')) : 0;
System.out.printf("topic %s threadCount %d messageSize %d keyEnable %s%n", topic, threadCount, messageSize, keyEnable);
final Logger log = ClientLogger.getLog();
- final Message msg = buildMessage(messageSize, topic);
-
final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount);
final StatsBenchmarkProducer statsBenchmark = new StatsBenchmarkProducer();
@@ -117,10 +118,37 @@ public class Producer {
public void run() {
while (true) {
try {
+ final Message msg;
+ try {
+ msg = buildMessage(messageSize, topic);
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ return;
+ }
final long beginTimestamp = System.currentTimeMillis();
if (keyEnable) {
msg.setKeys(String.valueOf(beginTimestamp / 1000));
}
+ if (propertySize > 0) {
+ if (msg.getProperties() != null) {
+ msg.getProperties().clear();
+ }
+ int i = 0;
+ int startValue = (new Random(System.currentTimeMillis())).nextInt(100);
+ int size = 0;
+ while (true) {
+ String prop1 = "prop" + i, prop1V = "hello" + startValue;
+ String prop2 = "prop" + (i + 1), prop2V = String.valueOf(startValue);
+ msg.putUserProperty(prop1, prop1V);
+ msg.putUserProperty(prop2, prop2V);
+ size += prop1.length() + prop2.length() + prop1V.length() + prop2V.length();
+ if (size > propertySize) {
+ break;
+ }
+ i += 2;
+ startValue += 2;
+ }
+ }
producer.send(msg);
statsBenchmark.getSendRequestSuccessCount().incrementAndGet();
statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet();
@@ -214,7 +242,7 @@ class StatsBenchmarkProducer {
private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);
public Long[] createSnapshot() {
- Long[] snap = new Long[] {
+ Long[] snap = new Long[]{
System.currentTimeMillis(),
this.sendRequestSuccessCount.get(),
this.sendRequestFailedCount.get(),
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java b/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java
new file mode 100644
index 0000000..9a3b813
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/filter/SqlConsumer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.example.filter;
+
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.MessageSelector;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.message.MessageExt;
+
+import java.util.List;
+
+public class SqlConsumer {
+
+ public static void main(String[] args) {
+ DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_4");
+
+ try {
+ consumer.subscribe("TopicTest",
+ MessageSelector.bySql("(TAGS is not null and TAGS in ('TagA', 'TagB'))" +
+ "and (a is not null and a between 0 3)"));
+ } catch (MQClientException e) {
+ e.printStackTrace();
+ return;
+ }
+
+ consumer.registerMessageListener(new MessageListenerConcurrently() {
+
+ @Override
+ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
+ ConsumeConcurrentlyContext context) {
+ System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n");
+ return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+ }
+ });
+
+ try {
+ consumer.start();
+ } catch (MQClientException e) {
+ e.printStackTrace();
+ return;
+ }
+ System.out.printf("Consumer Started.%n");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/example/src/main/java/org/apache/rocketmq/example/filter/SqlProducer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/filter/SqlProducer.java b/example/src/main/java/org/apache/rocketmq/example/filter/SqlProducer.java
new file mode 100644
index 0000000..3f3a0e6
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/filter/SqlProducer.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.example.filter;
+
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+
+public class SqlProducer {
+
+ public static void main(String[] args) {
+ DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+ try {
+ producer.start();
+ } catch (MQClientException e) {
+ e.printStackTrace();
+ return;
+ }
+
+ for (int i = 0; i < 10; i++) {
+ try {
+ String tag;
+ int div = i % 3;
+ if (div == 0) {
+ tag = "TagA";
+ } else if (div == 1) {
+ tag = "TagB";
+ } else {
+ tag = "TagC";
+ }
+ Message msg = new Message("TopicTest",
+ tag,
+ ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)
+ );
+ msg.putUserProperty("a", String.valueOf(i));
+
+ SendResult sendResult = producer.send(msg);
+ System.out.printf("%s%n", sendResult);
+ } catch (Exception e) {
+ e.printStackTrace();
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e1) {
+ e1.printStackTrace();
+ }
+ }
+ }
+ producer.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/pom.xml
----------------------------------------------------------------------
diff --git a/filter/pom.xml b/filter/pom.xml
new file mode 100644
index 0000000..7978f05
--- /dev/null
+++ b/filter/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>rocketmq-all</artifactId>
+ <groupId>org.apache.rocketmq</groupId>
+ <version>4.1.0-incubating-SNAPSHOT</version>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+ <packaging>jar</packaging>
+ <artifactId>rocketmq-filter</artifactId>
+ <name>rocketmq-filter ${project.version}</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-srvutil</artifactId>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/FilterFactory.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/FilterFactory.java b/filter/src/main/java/org/apache/rocketmq/filter/FilterFactory.java
new file mode 100644
index 0000000..a318548
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/FilterFactory.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Filter factory: support other filter to register.
+ */
+public class FilterFactory {
+
+ public static final FilterFactory INSTANCE = new FilterFactory();
+
+ protected static final Map<String, FilterSpi> FILTER_SPI_HOLDER = new HashMap<String, FilterSpi>(4);
+
+ static {
+ FilterFactory.INSTANCE.register(new SqlFilter());
+ }
+
+ /**
+ * Register a filter.
+ * <br>
+ * Note:
+ * <li>1. Filter registered will be used in broker server, so take care of it's reliability and performance.</li>
+ *
+ * @param filterSpi
+ */
+ public void register(FilterSpi filterSpi) {
+ if (FILTER_SPI_HOLDER.containsKey(filterSpi.ofType())) {
+ throw new IllegalArgumentException(String.format("Filter spi type(%s) already exist!", filterSpi.ofType()));
+ }
+
+ FILTER_SPI_HOLDER.put(filterSpi.ofType(), filterSpi);
+ }
+
+ /**
+ * Un register a filter.
+ *
+ * @param type
+ * @return
+ */
+ public FilterSpi unRegister(String type) {
+ return FILTER_SPI_HOLDER.remove(type);
+ }
+
+ /**
+ * Get a filter registered, null if none exist.
+ *
+ * @param type
+ * @return
+ */
+ public FilterSpi get(String type) {
+ return FILTER_SPI_HOLDER.get(type);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/FilterSpi.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/FilterSpi.java b/filter/src/main/java/org/apache/rocketmq/filter/FilterSpi.java
new file mode 100644
index 0000000..fcc39fa
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/FilterSpi.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter;
+
+import org.apache.rocketmq.filter.expression.Expression;
+import org.apache.rocketmq.filter.expression.MQFilterException;
+
+/**
+ * Filter spi interface.
+ */
+public interface FilterSpi {
+
+ /**
+ * Compile.
+ *
+ * @param expr
+ * @return
+ * @throws org.apache.rocketmq.filter.expression.MQFilterException
+ */
+ Expression compile(final String expr) throws MQFilterException;
+
+ /**
+ * Which type.
+ *
+ * @return
+ */
+ String ofType();
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/SqlFilter.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/SqlFilter.java b/filter/src/main/java/org/apache/rocketmq/filter/SqlFilter.java
new file mode 100644
index 0000000..0c1ffb8
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/SqlFilter.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter;
+
+import org.apache.rocketmq.common.filter.ExpressionType;
+import org.apache.rocketmq.filter.expression.Expression;
+import org.apache.rocketmq.filter.expression.MQFilterException;
+import org.apache.rocketmq.filter.parser.SelectorParser;
+
+/**
+ * SQL92 Filter, just a wrapper of {@link org.apache.rocketmq.filter.parser.SelectorParser}.
+ * <p/>
+ * <p>
+ * Do not use this filter directly.Use {@link FilterFactory#get} to select a filter.
+ * </p>
+ */
+public class SqlFilter implements FilterSpi {
+
+ @Override
+ public Expression compile(final String expr) throws MQFilterException {
+ return SelectorParser.parse(expr);
+ }
+
+ @Override
+ public String ofType() {
+ return ExpressionType.SQL92;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/constant/UnaryType.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/constant/UnaryType.java b/filter/src/main/java/org/apache/rocketmq/filter/constant/UnaryType.java
new file mode 100644
index 0000000..d2d04cd
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/constant/UnaryType.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.constant;
+
+public enum UnaryType {
+ NEGATE,
+ IN,
+ NOT,
+ BOOLEANCAST,
+ LIKE
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/BinaryExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/BinaryExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/BinaryExpression.java
new file mode 100644
index 0000000..0f172e3
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/BinaryExpression.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+/**
+ * An expression which performs an operation on two expression values.
+ * <p>
+ * This class was taken from ActiveMQ org.apache.activemq.filter.BinaryExpression,
+ * </p>
+ */
+public abstract class BinaryExpression implements Expression {
+ protected Expression left;
+ protected Expression right;
+
+ public BinaryExpression(Expression left, Expression right) {
+ this.left = left;
+ this.right = right;
+ }
+
+ public Expression getLeft() {
+ return left;
+ }
+
+ public Expression getRight() {
+ return right;
+ }
+
+ /**
+ * @see Object#toString()
+ */
+ public String toString() {
+ return "(" + left.toString() + " " + getExpressionSymbol() + " " + right.toString() + ")";
+ }
+
+ /**
+ * @see Object#hashCode()
+ */
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ /**
+ * @see Object#equals(Object)
+ */
+ public boolean equals(Object o) {
+
+ if (o == null || !this.getClass().equals(o.getClass())) {
+ return false;
+ }
+ return toString().equals(o.toString());
+
+ }
+
+ /**
+ * Returns the symbol that represents this binary expression. For example, addition is
+ * represented by "+"
+ *
+ * @return
+ */
+ public abstract String getExpressionSymbol();
+
+ /**
+ * @param expression
+ */
+ public void setRight(Expression expression) {
+ right = expression;
+ }
+
+ /**
+ * @param expression
+ */
+ public void setLeft(Expression expression) {
+ left = expression;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/BooleanExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/BooleanExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/BooleanExpression.java
new file mode 100644
index 0000000..bb54632
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/BooleanExpression.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+/**
+ * A BooleanExpression is an expression that always
+ * produces a Boolean result.
+ * <p>
+ * This class was taken from ActiveMQ org.apache.activemq.filter.BooleanExpression,
+ * but the parameter is changed to an interface.
+ * </p>
+ *
+ * @see org.apache.rocketmq.filter.expression.EvaluationContext
+ */
+public interface BooleanExpression extends Expression {
+
+ /**
+ * @param context
+ * @return true if the expression evaluates to Boolean.TRUE.
+ * @throws Exception
+ */
+ boolean matches(EvaluationContext context) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/ComparisonExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/ComparisonExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/ComparisonExpression.java
new file mode 100644
index 0000000..8b82e57
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/ComparisonExpression.java
@@ -0,0 +1,413 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+import java.util.List;
+
+/**
+ * A filter performing a comparison of two objects
+ * <p>
+ * This class was taken from ActiveMQ org.apache.activemq.filter.ComparisonExpression,
+ * but:
+ * 1. Remove LIKE expression, and related methods;
+ * 2. Extract a new method __compare which has int return value;
+ * 3. When create between expression, check whether left value is less or equal than right value;
+ * 4. For string type value(can not convert to number), only equal or unequal comparison are supported.
+ * </p>
+ */
+public abstract class ComparisonExpression extends BinaryExpression implements BooleanExpression {
+
+ public static final ThreadLocal<Boolean> CONVERT_STRING_EXPRESSIONS = new ThreadLocal<Boolean>();
+
+ boolean convertStringExpressions = false;
+
+ /**
+ * @param left
+ * @param right
+ */
+ public ComparisonExpression(Expression left, Expression right) {
+ super(left, right);
+ convertStringExpressions = CONVERT_STRING_EXPRESSIONS.get() != null;
+ }
+
+ public static BooleanExpression createBetween(Expression value, Expression left, Expression right) {
+ // check
+ if (left instanceof ConstantExpression && right instanceof ConstantExpression) {
+ Object lv = ((ConstantExpression) left).getValue();
+ Object rv = ((ConstantExpression) right).getValue();
+ if (lv == null || rv == null) {
+ throw new RuntimeException("Illegal values of between, values can not be null!");
+ }
+ if (lv instanceof Comparable && rv instanceof Comparable) {
+ int ret = __compare((Comparable) rv, (Comparable) lv, true);
+ if (ret < 0)
+ throw new RuntimeException(
+ String.format("Illegal values of between, left value(%s) must less than or equal to right value(%s)", lv, rv)
+ );
+ }
+ }
+
+ return LogicExpression.createAND(createGreaterThanEqual(value, left), createLessThanEqual(value, right));
+ }
+
+ public static BooleanExpression createNotBetween(Expression value, Expression left, Expression right) {
+ return LogicExpression.createOR(createLessThan(value, left), createGreaterThan(value, right));
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public static BooleanExpression createInFilter(Expression left, List elements) {
+
+ if (!(left instanceof PropertyExpression)) {
+ throw new RuntimeException("Expected a property for In expression, got: " + left);
+ }
+ return UnaryExpression.createInExpression((PropertyExpression) left, elements, false);
+
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public static BooleanExpression createNotInFilter(Expression left, List elements) {
+
+ if (!(left instanceof PropertyExpression)) {
+ throw new RuntimeException("Expected a property for In expression, got: " + left);
+ }
+ return UnaryExpression.createInExpression((PropertyExpression) left, elements, true);
+
+ }
+
+ public static BooleanExpression createIsNull(Expression left) {
+ return doCreateEqual(left, ConstantExpression.NULL);
+ }
+
+ public static BooleanExpression createIsNotNull(Expression left) {
+ return UnaryExpression.createNOT(doCreateEqual(left, ConstantExpression.NULL));
+ }
+
+ public static BooleanExpression createNotEqual(Expression left, Expression right) {
+ return UnaryExpression.createNOT(createEqual(left, right));
+ }
+
+ public static BooleanExpression createEqual(Expression left, Expression right) {
+ checkEqualOperand(left);
+ checkEqualOperand(right);
+ checkEqualOperandCompatability(left, right);
+ return doCreateEqual(left, right);
+ }
+
+ @SuppressWarnings({"rawtypes"})
+ private static BooleanExpression doCreateEqual(Expression left, Expression right) {
+ return new ComparisonExpression(left, right) {
+
+ public Object evaluate(EvaluationContext context) throws Exception {
+ Object lv = left.evaluate(context);
+ Object rv = right.evaluate(context);
+
+ // If one of the values is null
+ if (lv == null ^ rv == null) {
+ if (lv == null) {
+ return null;
+ }
+ return Boolean.FALSE;
+ }
+ if (lv == rv || lv.equals(rv)) {
+ return Boolean.TRUE;
+ }
+ if (lv instanceof Comparable && rv instanceof Comparable) {
+ return compare((Comparable) lv, (Comparable) rv);
+ }
+ return Boolean.FALSE;
+ }
+
+ protected boolean asBoolean(int answer) {
+ return answer == 0;
+ }
+
+ public String getExpressionSymbol() {
+ return "==";
+ }
+ };
+ }
+
+ public static BooleanExpression createGreaterThan(final Expression left, final Expression right) {
+ checkLessThanOperand(left);
+ checkLessThanOperand(right);
+ return new ComparisonExpression(left, right) {
+ protected boolean asBoolean(int answer) {
+ return answer > 0;
+ }
+
+ public String getExpressionSymbol() {
+ return ">";
+ }
+ };
+ }
+
+ public static BooleanExpression createGreaterThanEqual(final Expression left, final Expression right) {
+ checkLessThanOperand(left);
+ checkLessThanOperand(right);
+ return new ComparisonExpression(left, right) {
+ protected boolean asBoolean(int answer) {
+ return answer >= 0;
+ }
+
+ public String getExpressionSymbol() {
+ return ">=";
+ }
+ };
+ }
+
+ public static BooleanExpression createLessThan(final Expression left, final Expression right) {
+ checkLessThanOperand(left);
+ checkLessThanOperand(right);
+ return new ComparisonExpression(left, right) {
+
+ protected boolean asBoolean(int answer) {
+ return answer < 0;
+ }
+
+ public String getExpressionSymbol() {
+ return "<";
+ }
+
+ };
+ }
+
+ public static BooleanExpression createLessThanEqual(final Expression left, final Expression right) {
+ checkLessThanOperand(left);
+ checkLessThanOperand(right);
+ return new ComparisonExpression(left, right) {
+
+ protected boolean asBoolean(int answer) {
+ return answer <= 0;
+ }
+
+ public String getExpressionSymbol() {
+ return "<=";
+ }
+ };
+ }
+
+ /**
+ * Only Numeric expressions can be used in >, >=, < or <= expressions.s
+ *
+ * @param expr
+ */
+ public static void checkLessThanOperand(Expression expr) {
+ if (expr instanceof ConstantExpression) {
+ Object value = ((ConstantExpression) expr).getValue();
+ if (value instanceof Number) {
+ return;
+ }
+
+ // Else it's boolean or a String..
+ throw new RuntimeException("Value '" + expr + "' cannot be compared.");
+ }
+ if (expr instanceof BooleanExpression) {
+ throw new RuntimeException("Value '" + expr + "' cannot be compared.");
+ }
+ }
+
+ /**
+ * Validates that the expression can be used in == or <> expression. Cannot
+ * not be NULL TRUE or FALSE litterals.
+ *
+ * @param expr
+ */
+ public static void checkEqualOperand(Expression expr) {
+ if (expr instanceof ConstantExpression) {
+ Object value = ((ConstantExpression) expr).getValue();
+ if (value == null) {
+ throw new RuntimeException("'" + expr + "' cannot be compared.");
+ }
+ }
+ }
+
+ /**
+ * @param left
+ * @param right
+ */
+ private static void checkEqualOperandCompatability(Expression left, Expression right) {
+ if (left instanceof ConstantExpression && right instanceof ConstantExpression) {
+ if (left instanceof BooleanExpression && !(right instanceof BooleanExpression)) {
+ throw new RuntimeException("'" + left + "' cannot be compared with '" + right + "'");
+ }
+ }
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public Object evaluate(EvaluationContext context) throws Exception {
+ Comparable<Comparable> lv = (Comparable) left.evaluate(context);
+ if (lv == null) {
+ return null;
+ }
+ Comparable rv = (Comparable) right.evaluate(context);
+ if (rv == null) {
+ return null;
+ }
+ if (getExpressionSymbol().equals(">=") || getExpressionSymbol().equals(">")
+ || getExpressionSymbol().equals("<") || getExpressionSymbol().equals("<=")) {
+ Class<? extends Comparable> lc = lv.getClass();
+ Class<? extends Comparable> rc = rv.getClass();
+ if (lc == rc && lc == String.class) {
+ // Compare String is illegal
+ // first try to convert to double
+ try {
+ Comparable lvC = Double.valueOf((String) (Comparable) lv);
+ Comparable rvC = Double.valueOf((String) rv);
+
+ return compare(lvC, rvC);
+ } catch (Exception e) {
+ throw new RuntimeException("It's illegal to compare string by '>=', '>', '<', '<='. lv=" + lv + ", rv=" + rv, e);
+ }
+ }
+ }
+ return compare(lv, rv);
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ protected static int __compare(Comparable lv, Comparable rv, boolean convertStringExpressions) {
+ Class<? extends Comparable> lc = lv.getClass();
+ Class<? extends Comparable> rc = rv.getClass();
+ // If the the objects are not of the same type,
+ // try to convert up to allow the comparison.
+ if (lc != rc) {
+ try {
+ if (lc == Boolean.class) {
+ if (convertStringExpressions && rc == String.class) {
+ lv = Boolean.valueOf((String) lv).booleanValue();
+ } else {
+ return -1;
+ }
+ } else if (lc == Byte.class) {
+ if (rc == Short.class) {
+ lv = Short.valueOf(((Number) lv).shortValue());
+ } else if (rc == Integer.class) {
+ lv = Integer.valueOf(((Number) lv).intValue());
+ } else if (rc == Long.class) {
+ lv = Long.valueOf(((Number) lv).longValue());
+ } else if (rc == Float.class) {
+ lv = new Float(((Number) lv).floatValue());
+ } else if (rc == Double.class) {
+ lv = new Double(((Number) lv).doubleValue());
+ } else if (convertStringExpressions && rc == String.class) {
+ rv = Byte.valueOf((String) rv);
+ } else {
+ return -1;
+ }
+ } else if (lc == Short.class) {
+ if (rc == Integer.class) {
+ lv = Integer.valueOf(((Number) lv).intValue());
+ } else if (rc == Long.class) {
+ lv = Long.valueOf(((Number) lv).longValue());
+ } else if (rc == Float.class) {
+ lv = new Float(((Number) lv).floatValue());
+ } else if (rc == Double.class) {
+ lv = new Double(((Number) lv).doubleValue());
+ } else if (convertStringExpressions && rc == String.class) {
+ rv = Short.valueOf((String) rv);
+ } else {
+ return -1;
+ }
+ } else if (lc == Integer.class) {
+ if (rc == Long.class) {
+ lv = Long.valueOf(((Number) lv).longValue());
+ } else if (rc == Float.class) {
+ lv = new Float(((Number) lv).floatValue());
+ } else if (rc == Double.class) {
+ lv = new Double(((Number) lv).doubleValue());
+ } else if (convertStringExpressions && rc == String.class) {
+ rv = Integer.valueOf((String) rv);
+ } else {
+ return -1;
+ }
+ } else if (lc == Long.class) {
+ if (rc == Integer.class) {
+ rv = Long.valueOf(((Number) rv).longValue());
+ } else if (rc == Float.class) {
+ lv = new Float(((Number) lv).floatValue());
+ } else if (rc == Double.class) {
+ lv = new Double(((Number) lv).doubleValue());
+ } else if (convertStringExpressions && rc == String.class) {
+ rv = Long.valueOf((String) rv);
+ } else {
+ return -1;
+ }
+ } else if (lc == Float.class) {
+ if (rc == Integer.class) {
+ rv = new Float(((Number) rv).floatValue());
+ } else if (rc == Long.class) {
+ rv = new Float(((Number) rv).floatValue());
+ } else if (rc == Double.class) {
+ lv = new Double(((Number) lv).doubleValue());
+ } else if (convertStringExpressions && rc == String.class) {
+ rv = Float.valueOf((String) rv);
+ } else {
+ return -1;
+ }
+ } else if (lc == Double.class) {
+ if (rc == Integer.class) {
+ rv = new Double(((Number) rv).doubleValue());
+ } else if (rc == Long.class) {
+ rv = new Double(((Number) rv).doubleValue());
+ } else if (rc == Float.class) {
+ rv = new Float(((Number) rv).doubleValue());
+ } else if (convertStringExpressions && rc == String.class) {
+ rv = Double.valueOf((String) rv);
+ } else {
+ return -1;
+ }
+ } else if (convertStringExpressions && lc == String.class) {
+ if (rc == Boolean.class) {
+ lv = Boolean.valueOf((String) lv);
+ } else if (rc == Byte.class) {
+ lv = Byte.valueOf((String) lv);
+ } else if (rc == Short.class) {
+ lv = Short.valueOf((String) lv);
+ } else if (rc == Integer.class) {
+ lv = Integer.valueOf((String) lv);
+ } else if (rc == Long.class) {
+ lv = Long.valueOf((String) lv);
+ } else if (rc == Float.class) {
+ lv = Float.valueOf((String) lv);
+ } else if (rc == Double.class) {
+ lv = Double.valueOf((String) lv);
+ } else {
+ return -1;
+ }
+ } else {
+ return -1;
+ }
+ } catch (NumberFormatException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return lv.compareTo(rv);
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ protected Boolean compare(Comparable lv, Comparable rv) {
+ return asBoolean(__compare(lv, rv, convertStringExpressions)) ? Boolean.TRUE : Boolean.FALSE;
+ }
+
+ protected abstract boolean asBoolean(int answer);
+
+ public boolean matches(EvaluationContext context) throws Exception {
+ Object object = evaluate(context);
+ return object != null && object == Boolean.TRUE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/ConstantExpression.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/ConstantExpression.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/ConstantExpression.java
new file mode 100644
index 0000000..ca70f51
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/ConstantExpression.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+/**
+ * Represents a constant expression
+ * <p>
+ * This class was taken from ActiveMQ org.apache.activemq.filter.ConstantExpression,
+ * but:
+ * 1. For long type constant, the range bound by java Long type;
+ * 2. For float type constant, the range bound by java Double type;
+ * 3. Remove Hex and Octal expression;
+ * 4. Add now expression to support to get current time.
+ * </p>
+ */
+public class ConstantExpression implements Expression {
+
+ static class BooleanConstantExpression extends ConstantExpression implements BooleanExpression {
+ public BooleanConstantExpression(Object value) {
+ super(value);
+ }
+
+ public boolean matches(EvaluationContext context) throws Exception {
+ Object object = evaluate(context);
+ return object != null && object == Boolean.TRUE;
+ }
+ }
+
+ public static final BooleanConstantExpression NULL = new BooleanConstantExpression(null);
+ public static final BooleanConstantExpression TRUE = new BooleanConstantExpression(Boolean.TRUE);
+ public static final BooleanConstantExpression FALSE = new BooleanConstantExpression(Boolean.FALSE);
+
+ private Object value;
+
+ public ConstantExpression(Object value) {
+ this.value = value;
+ }
+
+ public static ConstantExpression createFromDecimal(String text) {
+
+ // Strip off the 'l' or 'L' if needed.
+ if (text.endsWith("l") || text.endsWith("L")) {
+ text = text.substring(0, text.length() - 1);
+ }
+
+ // only support Long.MIN_VALUE ~ Long.MAX_VALUE
+ Number value = new Long(text);
+// try {
+// value = new Long(text);
+// } catch (NumberFormatException e) {
+// // The number may be too big to fit in a long.
+// value = new BigDecimal(text);
+// }
+
+ long l = value.longValue();
+ if (Integer.MIN_VALUE <= l && l <= Integer.MAX_VALUE) {
+ value = Integer.valueOf(value.intValue());
+ }
+ return new ConstantExpression(value);
+ }
+
+ public static ConstantExpression createFloat(String text) {
+ Double value = new Double(text);
+ if (value > Double.MAX_VALUE) {
+ throw new RuntimeException(text + " is greater than " + Double.MAX_VALUE);
+ }
+ if (value < Double.MIN_VALUE) {
+ throw new RuntimeException(text + " is less than " + Double.MIN_VALUE);
+ }
+ return new ConstantExpression(value);
+ }
+
+ public static ConstantExpression createNow() {
+ return new NowExpression();
+ }
+
+ public Object evaluate(EvaluationContext context) throws Exception {
+ return value;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+
+ /**
+ * @see Object#toString()
+ */
+ public String toString() {
+ Object value = getValue();
+ if (value == null) {
+ return "NULL";
+ }
+ if (value instanceof Boolean) {
+ return ((Boolean) value).booleanValue() ? "TRUE" : "FALSE";
+ }
+ if (value instanceof String) {
+ return encodeString((String) value);
+ }
+ return value.toString();
+ }
+
+ /**
+ * @see Object#hashCode()
+ */
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ /**
+ * @see Object#equals(Object)
+ */
+ public boolean equals(Object o) {
+
+ if (o == null || !this.getClass().equals(o.getClass())) {
+ return false;
+ }
+ return toString().equals(o.toString());
+
+ }
+
+ /**
+ * Encodes the value of string so that it looks like it would look like when
+ * it was provided in a selector.
+ *
+ * @return
+ */
+ public static String encodeString(String s) {
+ StringBuffer b = new StringBuffer();
+ b.append('\'');
+ for (int i = 0; i < s.length(); i++) {
+ char c = s.charAt(i);
+ if (c == '\'') {
+ b.append(c);
+ }
+ b.append(c);
+ }
+ b.append('\'');
+ return b.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/EmptyEvaluationContext.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/EmptyEvaluationContext.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/EmptyEvaluationContext.java
new file mode 100644
index 0000000..52af2d0
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/EmptyEvaluationContext.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+import java.util.Map;
+
+/**
+ * Empty context.
+ */
+public class EmptyEvaluationContext implements EvaluationContext {
+ @Override
+ public Object get(String name) {
+ return null;
+ }
+
+ @Override
+ public Map<String, Object> keyValues() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/58f1574b/filter/src/main/java/org/apache/rocketmq/filter/expression/EvaluationContext.java
----------------------------------------------------------------------
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/expression/EvaluationContext.java b/filter/src/main/java/org/apache/rocketmq/filter/expression/EvaluationContext.java
new file mode 100644
index 0000000..094ef53
--- /dev/null
+++ b/filter/src/main/java/org/apache/rocketmq/filter/expression/EvaluationContext.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.filter.expression;
+
+import java.util.Map;
+
+/**
+ * Context of evaluate expression.
+ *
+ * Compare to org.apache.activemq.filter.MessageEvaluationContext, this is just an interface.
+ */
+public interface EvaluationContext {
+
+ /**
+ * Get value by name from context
+ *
+ * @param name
+ * @return
+ */
+ Object get(String name);
+
+ /**
+ * Context variables.
+ *
+ * @return
+ */
+ Map<String, Object> keyValues();
+}
[48/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-219] Add
batch example, closes apache/incubator-rocketmq#112
Posted by do...@apache.org.
[ROCKETMQ-219] Add batch example, closes apache/incubator-rocketmq#112
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/f45a1bcd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/f45a1bcd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/f45a1bcd
Branch: refs/heads/release-4.1.0-incubating
Commit: f45a1bcd2b42423050e6bd28c5fb92f5c47e263e
Parents: 703ac00
Author: dongeforever <zh...@yeah.net>
Authored: Thu Jun 8 11:13:24 2017 +0800
Committer: yukon <yu...@apache.org>
Committed: Thu Jun 8 11:20:40 2017 +0800
----------------------------------------------------------------------
.../example/batch/SimpleBatchProducer.java | 42 +++++++++
.../example/batch/SplitBatchProducer.java | 97 ++++++++++++++++++++
2 files changed, 139 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/f45a1bcd/example/src/main/java/org/apache/rocketmq/example/batch/SimpleBatchProducer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/batch/SimpleBatchProducer.java b/example/src/main/java/org/apache/rocketmq/example/batch/SimpleBatchProducer.java
new file mode 100644
index 0000000..a8609e7
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/batch/SimpleBatchProducer.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.example.batch;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.common.message.Message;
+
+public class SimpleBatchProducer {
+
+
+ public static void main(String[] args) throws Exception {
+ DefaultMQProducer producer = new DefaultMQProducer("BatchProducerGroupName");
+ producer.start();
+
+ //If you just send messages of no more than 1MiB at a time, it is easy to use batch
+ //Messages of the same batch should have: same topic, same waitStoreMsgOK and no schedule support
+ String topic = "BatchTest";
+ List<Message> messages = new ArrayList<>();
+ messages.add(new Message(topic, "Tag", "OrderID001", "Hello world 0".getBytes()));
+ messages.add(new Message(topic, "Tag", "OrderID002", "Hello world 1".getBytes()));
+ messages.add(new Message(topic, "Tag", "OrderID003", "Hello world 2".getBytes()));
+
+ producer.send(messages);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/f45a1bcd/example/src/main/java/org/apache/rocketmq/example/batch/SplitBatchProducer.java
----------------------------------------------------------------------
diff --git a/example/src/main/java/org/apache/rocketmq/example/batch/SplitBatchProducer.java b/example/src/main/java/org/apache/rocketmq/example/batch/SplitBatchProducer.java
new file mode 100644
index 0000000..8809a11
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/batch/SplitBatchProducer.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.example.batch;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.common.message.Message;
+
+public class SplitBatchProducer {
+
+ public static void main(String[] args) throws Exception {
+
+ DefaultMQProducer producer = new DefaultMQProducer("BatchProducerGroupName");
+ producer.start();
+
+ //large batch
+ String topic = "BatchTest";
+ List<Message> messages = new ArrayList<>(100 * 1000);
+ for (int i = 0; i < 100 * 1000; i++) {
+ messages.add(new Message(topic, "Tag", "OrderID" + i, ("Hello world " + i).getBytes()));
+ }
+
+ //split the large batch into small ones:
+ ListSplitter splitter = new ListSplitter(messages);
+ while (splitter.hasNext()) {
+ List<Message> listItem = splitter.next();
+ producer.send(listItem);
+ }
+ }
+
+}
+
+
+class ListSplitter implements Iterator<List<Message>> {
+ private int sizeLimit = 1000 * 1000;
+ private final List<Message> messages;
+ private int currIndex;
+ public ListSplitter(List<Message> messages) {
+ this.messages = messages;
+ }
+ @Override public boolean hasNext() {
+ return currIndex < messages.size();
+ }
+ @Override public List<Message> next() {
+ int nextIndex = currIndex;
+ int totalSize = 0;
+ for (; nextIndex < messages.size(); nextIndex++) {
+ Message message = messages.get(nextIndex);
+ int tmpSize = message.getTopic().length() + message.getBody().length;
+ Map<String, String> properties = message.getProperties();
+ for (Map.Entry<String, String> entry : properties.entrySet()) {
+ tmpSize += entry.getKey().length() + entry.getValue().length();
+ }
+ tmpSize = tmpSize + 20; //for log overhead
+ if (tmpSize > sizeLimit) {
+ //it is unexpected that single message exceeds the sizeLimit
+ //here just let it go, otherwise it will block the splitting process
+ if (nextIndex - currIndex == 0) {
+ //if the next sublist has no element, add this one and then break, otherwise just break
+ nextIndex++;
+ }
+ break;
+ }
+ if (tmpSize + totalSize > sizeLimit) {
+ break;
+ } else {
+ totalSize += tmpSize;
+ }
+
+ }
+ List<Message> subList = messages.subList(currIndex, nextIndex);
+ currIndex = nextIndex;
+ return subList;
+ }
+
+ @Override public void remove() {
+ throw new UnsupportedOperationException("Not allowed to remove");
+ }
+}
[13/50] [abbrv] incubator-rocketmq git commit: Add javadoc to
NettyRemotingAbstract class and several other trivial clean up.
Posted by do...@apache.org.
Add javadoc to NettyRemotingAbstract class and several other trivial clean up.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/6609c866
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/6609c866
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/6609c866
Branch: refs/heads/release-4.1.0-incubating
Commit: 6609c86650917ebfb5bd12a4bd8b1bcf9c477759
Parents: 6a9628b
Author: Zhanhui Li <li...@apache.org>
Authored: Tue Apr 25 22:58:14 2017 +0800
Committer: Zhanhui Li <li...@apache.org>
Committed: Tue Apr 25 22:58:14 2017 +0800
----------------------------------------------------------------------
.../remoting/netty/NettyRemotingAbstract.java | 93 ++++++++++++++++++--
.../remoting/netty/NettyRemotingClient.java | 19 ++--
.../remoting/netty/NettyRemotingServer.java | 22 ++---
3 files changed, 106 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6609c866/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
----------------------------------------------------------------------
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
index 83eeb02..cddab3d 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
@@ -48,32 +48,84 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public abstract class NettyRemotingAbstract {
+
+ /**
+ * Remoting logger instance.
+ */
private static final Logger PLOG = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
+ /**
+ * Semaphore to limit maximum number of on-going one-way requests, which protects system memory footprint.
+ */
protected final Semaphore semaphoreOneway;
+ /**
+ * Semaphore to limit maximum number of on-going asynchronous requests, which protects system memory footprint.
+ */
protected final Semaphore semaphoreAsync;
+ /**
+ * This map caches all on-going requests.
+ */
protected final ConcurrentHashMap<Integer /* opaque */, ResponseFuture> responseTable =
new ConcurrentHashMap<Integer, ResponseFuture>(256);
+ /**
+ * This container holds all processors per request code, aka, for each incoming request, we may look up the
+ * responding processor in this map to handle the request.
+ */
protected final HashMap<Integer/* request code */, Pair<NettyRequestProcessor, ExecutorService>> processorTable =
new HashMap<Integer, Pair<NettyRequestProcessor, ExecutorService>>(64);
- protected final NettyEventExecuter nettyEventExecuter = new NettyEventExecuter();
+ /**
+ * Executor to feed netty events to user defined {@link ChannelEventListener}.
+ */
+ protected final NettyEventExecutor nettyEventExecutor = new NettyEventExecutor();
+
+ /**
+ * The default request processor to use in case there is no exact match in {@link #processorTable} per request code.
+ */
protected Pair<NettyRequestProcessor, ExecutorService> defaultRequestProcessor;
+ /**
+ * Constructor, specifying capacity of one-way and asynchronous semaphores.
+ * @param permitsOneway Number of permits for one-way requests.
+ * @param permitsAsync Number of permits for asynchronous requests.
+ */
public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) {
this.semaphoreOneway = new Semaphore(permitsOneway, true);
this.semaphoreAsync = new Semaphore(permitsAsync, true);
}
+ /**
+ * Custom channel event listener.
+ * @return custom channel event listener if defined; null otherwise.
+ */
public abstract ChannelEventListener getChannelEventListener();
+ /**
+ * Put a netty event to the executor.
+ * @param event Netty event instance.
+ */
public void putNettyEvent(final NettyEvent event) {
- this.nettyEventExecuter.putNettyEvent(event);
+ this.nettyEventExecutor.putNettyEvent(event);
}
+ /**
+ * Entry of incoming command processing.
+ *
+ * <p>
+ * <strong>Note:</strong>
+ * The incoming remoting command may be
+ * <ul>
+ * <li>An inquiry request from a remote peer component;</li>
+ * <li>A response to a previous request issued by this very participant.</li>
+ * </ul>
+ * </p>
+ * @param ctx Channel handler context.
+ * @param msg incoming remoting command.
+ * @throws Exception if there were any error while processing the incoming command.
+ */
public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception {
final RemotingCommand cmd = msg;
if (cmd != null) {
@@ -90,6 +142,11 @@ public abstract class NettyRemotingAbstract {
}
}
+ /**
+ * Process incoming request command issued by remote peer.
+ * @param ctx channel handler context.
+ * @param cmd request command.
+ */
public void processRequestCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) {
final Pair<NettyRequestProcessor, ExecutorService> matched = this.processorTable.get(cmd.getCode());
final Pair<NettyRequestProcessor, ExecutorService> pair = null == matched ? this.defaultRequestProcessor : matched;
@@ -175,6 +232,11 @@ public abstract class NettyRemotingAbstract {
}
}
+ /**
+ * Process response from remote peer to the previous issued requests.
+ * @param ctx channel handler context.
+ * @param cmd response command instance.
+ */
public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cmd) {
final int opaque = cmd.getOpaque();
final ResponseFuture responseFuture = responseTable.get(opaque);
@@ -196,7 +258,10 @@ public abstract class NettyRemotingAbstract {
}
}
- //execute callback in callback executor. If callback executor is null, run directly in current thread
+ /**
+ * Execute callback in callback executor. If callback executor is null, run directly in current thread
+ * @param responseFuture
+ */
private void executeInvokeCallback(final ResponseFuture responseFuture) {
boolean runInThisThread = false;
ExecutorService executor = this.getCallbackExecutor();
@@ -229,10 +294,24 @@ public abstract class NettyRemotingAbstract {
}
}
+ /**
+ * Custom RPC hook.
+ * @return RPC hook if specified; null otherwise.
+ */
public abstract RPCHook getRPCHook();
- abstract public ExecutorService getCallbackExecutor();
-
+ /**
+ * This method specifies thread pool to use while invoking callback methods.
+ * @return Dedicated thread pool instance if specified; or null if the callback is supposed to be executed in the
+ * netty event-loop thread.
+ */
+ public abstract ExecutorService getCallbackExecutor();
+
+ /**
+ * <p>
+ * This method is periodically invoked to scan and expire deprecated request.
+ * </p>
+ */
public void scanResponseTable() {
final List<ResponseFuture> rfList = new LinkedList<ResponseFuture>();
Iterator<Entry<Integer, ResponseFuture>> it = this.responseTable.entrySet().iterator();
@@ -386,7 +465,7 @@ public abstract class NettyRemotingAbstract {
}
}
- class NettyEventExecuter extends ServiceThread {
+ class NettyEventExecutor extends ServiceThread {
private final LinkedBlockingQueue<NettyEvent> eventQueue = new LinkedBlockingQueue<NettyEvent>();
private final int maxSize = 10000;
@@ -436,7 +515,7 @@ public abstract class NettyRemotingAbstract {
@Override
public String getServiceName() {
- return NettyEventExecuter.class.getSimpleName();
+ return NettyEventExecutor.class.getSimpleName();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6609c866/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
----------------------------------------------------------------------
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
index 26088aa..52ca47e 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
@@ -172,7 +172,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
}, 1000 * 3, 1000);
if (this.channelEventListener != null) {
- this.nettyEventExecuter.start();
+ this.nettyEventExecutor.start();
}
}
@@ -189,8 +189,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
this.eventLoopGroupWorker.shutdownGracefully();
- if (this.nettyEventExecuter != null) {
- this.nettyEventExecuter.shutdown();
+ if (this.nettyEventExecutor != null) {
+ this.nettyEventExecutor.shutdown();
}
if (this.defaultEventExecutorGroup != null) {
@@ -586,7 +586,6 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
@Override
protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception {
processMessageReceived(ctx, msg);
-
}
}
@@ -594,8 +593,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
@Override
public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress,
ChannelPromise promise) throws Exception {
- final String local = localAddress == null ? "UNKNOWN" : localAddress.toString();
- final String remote = remoteAddress == null ? "UNKNOWN" : remoteAddress.toString();
+ final String local = localAddress == null ? "UNKNOWN" : RemotingHelper.parseSocketAddressAddr(localAddress);
+ final String remote = remoteAddress == null ? "UNKNOWN" : RemotingHelper.parseSocketAddressAddr(remoteAddress);
log.info("NETTY CLIENT PIPELINE: CONNECT {} => {}", local, remote);
super.connect(ctx, remoteAddress, localAddress, promise);
@@ -613,7 +612,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
super.disconnect(ctx, promise);
if (NettyRemotingClient.this.channelEventListener != null) {
- NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress.toString(), ctx.channel()));
+ NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress, ctx.channel()));
}
}
@@ -625,7 +624,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
super.close(ctx, promise);
if (NettyRemotingClient.this.channelEventListener != null) {
- NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress.toString(), ctx.channel()));
+ NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress, ctx.channel()));
}
}
@@ -639,7 +638,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
closeChannel(ctx.channel());
if (NettyRemotingClient.this.channelEventListener != null) {
NettyRemotingClient.this
- .putNettyEvent(new NettyEvent(NettyEventType.IDLE, remoteAddress.toString(), ctx.channel()));
+ .putNettyEvent(new NettyEvent(NettyEventType.IDLE, remoteAddress, ctx.channel()));
}
}
}
@@ -654,7 +653,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
log.warn("NETTY CLIENT PIPELINE: exceptionCaught exception.", cause);
closeChannel(ctx.channel());
if (NettyRemotingClient.this.channelEventListener != null) {
- NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress.toString(), ctx.channel()));
+ NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress, ctx.channel()));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/6609c866/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
----------------------------------------------------------------------
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
index 6a6df37..d8d9b65 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
@@ -160,7 +160,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
new NettyEncoder(),
new NettyDecoder(),
new IdleStateHandler(0, 0, nettyServerConfig.getServerChannelMaxIdleTimeSeconds()),
- new NettyConnetManageHandler(),
+ new NettyConnectManageHandler(),
new NettyServerHandler());
}
});
@@ -178,7 +178,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}
if (this.channelEventListener != null) {
- this.nettyEventExecuter.start();
+ this.nettyEventExecutor.start();
}
this.timer.scheduleAtFixedRate(new TimerTask() {
@@ -205,8 +205,8 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
this.eventLoopGroupSelector.shutdownGracefully();
- if (this.nettyEventExecuter != null) {
- this.nettyEventExecuter.shutdown();
+ if (this.nettyEventExecutor != null) {
+ this.nettyEventExecutor.shutdown();
}
if (this.defaultEventExecutorGroup != null) {
@@ -297,7 +297,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}
}
- class NettyConnetManageHandler extends ChannelDuplexHandler {
+ class NettyConnectManageHandler extends ChannelDuplexHandler {
@Override
public void channelRegistered(ChannelHandlerContext ctx) throws Exception {
final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
@@ -319,7 +319,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
super.channelActive(ctx);
if (NettyRemotingServer.this.channelEventListener != null) {
- NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CONNECT, remoteAddress.toString(), ctx.channel()));
+ NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CONNECT, remoteAddress, ctx.channel()));
}
}
@@ -330,21 +330,21 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
super.channelInactive(ctx);
if (NettyRemotingServer.this.channelEventListener != null) {
- NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress.toString(), ctx.channel()));
+ NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress, ctx.channel()));
}
}
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
if (evt instanceof IdleStateEvent) {
- IdleStateEvent evnet = (IdleStateEvent) evt;
- if (evnet.state().equals(IdleState.ALL_IDLE)) {
+ IdleStateEvent event = (IdleStateEvent) evt;
+ if (event.state().equals(IdleState.ALL_IDLE)) {
final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
log.warn("NETTY SERVER PIPELINE: IDLE exception [{}]", remoteAddress);
RemotingUtil.closeChannel(ctx.channel());
if (NettyRemotingServer.this.channelEventListener != null) {
NettyRemotingServer.this
- .putNettyEvent(new NettyEvent(NettyEventType.IDLE, remoteAddress.toString(), ctx.channel()));
+ .putNettyEvent(new NettyEvent(NettyEventType.IDLE, remoteAddress, ctx.channel()));
}
}
}
@@ -359,7 +359,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
log.warn("NETTY SERVER PIPELINE: exceptionCaught exception.", cause);
if (NettyRemotingServer.this.channelEventListener != null) {
- NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress.toString(), ctx.channel()));
+ NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress, ctx.channel()));
}
RemotingUtil.closeChannel(ctx.channel());
[38/50] [abbrv] incubator-rocketmq git commit: Merge remote-tracking
branch 'wip/ROCKETMQ-206' into develop
Posted by do...@apache.org.
Merge remote-tracking branch 'wip/ROCKETMQ-206' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/e57f9ac4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/e57f9ac4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/e57f9ac4
Branch: refs/heads/release-4.1.0-incubating
Commit: e57f9ac433bdf4ec640089ccaf580954e93f50dc
Parents: 04c8925 aced0de
Author: dongeforever <zh...@yeah.net>
Authored: Sat May 27 12:39:25 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Sat May 27 12:39:25 2017 +0800
----------------------------------------------------------------------
.../client/consumer/store/LocalFileOffsetStore.java | 14 ++++++++++++--
.../apache/rocketmq/example/benchmark/Consumer.java | 3 ++-
.../org/apache/rocketmq/example/filter/Consumer.java | 3 ++-
.../rocketmq/namesrv/kvconfig/KVConfigManager.java | 7 ++++++-
4 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[27/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-175] Consumer may miss messages because of inconsistent sub… closes apache/incubator-rocketmq#92
Posted by do...@apache.org.
[ROCKETMQ-175] Consumer may miss messages because of inconsistent sub… closes apache/incubator-rocketmq#92
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/82803889
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/82803889
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/82803889
Branch: refs/heads/release-4.1.0-incubating
Commit: 8280388917c466d030ffb774a2474ca8e4144811
Parents: 42826c4
Author: vsair <li...@gmail.com>
Authored: Fri May 26 15:13:29 2017 +0800
Committer: dongeforever <zh...@yeah.net>
Committed: Fri May 26 15:13:29 2017 +0800
----------------------------------------------------------------------
.../rocketmq/client/impl/consumer/RebalancePushImpl.java | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/82803889/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
index 1730c99..509c9a4 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
@@ -30,6 +30,7 @@ import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
public class RebalancePushImpl extends RebalanceImpl {
private final static long UNLOCK_DELAY_TIME_MILLS = Long.parseLong(System.getProperty("rocketmq.client.unlockDelayTimeMills", "20000"));
@@ -47,6 +48,16 @@ public class RebalancePushImpl extends RebalanceImpl {
@Override
public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {
+ /**
+ * When rebalance result changed, should update subscription's version to notify broker.
+ * Fix: inconsistency subscription may lead to consumer miss messages.
+ */
+ SubscriptionData subscriptionData = this.subscriptionInner.get(topic);
+ long newVersion = System.currentTimeMillis();
+ log.info("{} Rebalance changed, also update version: {}, {}", topic, subscriptionData.getSubVersion(), newVersion);
+ subscriptionData.setSubVersion(newVersion);
+ // notify broker
+ this.getmQClientFactory().sendHeartbeatToAllBrokerWithLock();
}
@Override
[19/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-191] Fix
socket options
Posted by do...@apache.org.
[ROCKETMQ-191] Fix socket options
Project: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/commit/80aac138
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/tree/80aac138
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/diff/80aac138
Branch: refs/heads/release-4.1.0-incubating
Commit: 80aac138d905561c7a63c8e15fdfe60e958a3032
Parents: f5a2ee0
Author: Li Zhanhui <li...@apache.org>
Authored: Wed May 10 10:44:34 2017 +0800
Committer: Li Zhanhui <li...@apache.org>
Committed: Wed May 10 10:44:34 2017 +0800
----------------------------------------------------------------------
.../remoting/netty/NettyRemotingServer.java | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/80aac138/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
----------------------------------------------------------------------
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
index d8d9b65..a9a55ab 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
@@ -26,7 +26,9 @@ import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.channel.epoll.Epoll;
import io.netty.channel.epoll.EpollEventLoopGroup;
+import io.netty.channel.epoll.EpollServerSocketChannel;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
@@ -105,8 +107,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}
});
- if (RemotingUtil.isLinuxPlatform() //
- && nettyServerConfig.isUseEpollNativeSelector()) {
+ if (useEpoll()) {
this.eventLoopGroupSelector = new EpollEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactory() {
private AtomicInteger threadIndex = new AtomicInteger(0);
private int threadTotal = nettyServerConfig.getServerSelectorThreads();
@@ -129,6 +130,12 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}
}
+ private boolean useEpoll() {
+ return RemotingUtil.isLinuxPlatform()
+ && nettyServerConfig.isUseEpollNativeSelector()
+ && Epoll.isAvailable();
+ }
+
@Override
public void start() {
this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(
@@ -144,13 +151,14 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
});
ServerBootstrap childHandler =
- this.serverBootstrap.group(this.eventLoopGroupBoss, this.eventLoopGroupSelector).channel(NioServerSocketChannel.class)
+ this.serverBootstrap.group(this.eventLoopGroupBoss, this.eventLoopGroupSelector)
+ .channel(useEpoll() ? EpollServerSocketChannel.class : NioServerSocketChannel.class)
.option(ChannelOption.SO_BACKLOG, 1024)
.option(ChannelOption.SO_REUSEADDR, true)
.option(ChannelOption.SO_KEEPALIVE, false)
.childOption(ChannelOption.TCP_NODELAY, true)
- .option(ChannelOption.SO_SNDBUF, nettyServerConfig.getServerSocketSndBufSize())
- .option(ChannelOption.SO_RCVBUF, nettyServerConfig.getServerSocketRcvBufSize())
+ .childOption(ChannelOption.SO_SNDBUF, nettyServerConfig.getServerSocketSndBufSize())
+ .childOption(ChannelOption.SO_RCVBUF, nettyServerConfig.getServerSocketRcvBufSize())
.localAddress(new InetSocketAddress(this.nettyServerConfig.getListenPort()))
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
[21/50] [abbrv] incubator-rocketmq git commit: [ROCKETMQ-186]
Implement the OpenMessaging specification 0.1.0-alpha version
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/FutureState.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/FutureState.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/FutureState.java
new file mode 100644
index 0000000..84b6c2d
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/FutureState.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.openmessaging.rocketmq.promise;
+
+public enum FutureState {
+ /**
+ * the task is doing
+ **/
+ DOING(0),
+ /**
+ * the task is done
+ **/
+ DONE(1),
+ /**
+ * ths task is cancelled
+ **/
+ CANCELLED(2);
+
+ public final int value;
+
+ private FutureState(int value) {
+ this.value = value;
+ }
+
+ public boolean isCancelledState() {
+ return this == CANCELLED;
+ }
+
+ public boolean isDoneState() {
+ return this == DONE;
+ }
+
+ public boolean isDoingState() {
+ return this == DOING;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java
new file mode 100644
index 0000000..104d3d9
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.utils;
+
+import io.openmessaging.KeyValue;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.slf4j.Logger;
+
+public final class BeanUtils {
+ final static Logger log = ClientLogger.getLog();
+
+ /**
+ * Maps primitive {@code Class}es to their corresponding wrapper {@code Class}.
+ */
+ private static Map<Class<?>, Class<?>> primitiveWrapperMap = new HashMap<Class<?>, Class<?>>();
+
+ static {
+ primitiveWrapperMap.put(Boolean.TYPE, Boolean.class);
+ primitiveWrapperMap.put(Byte.TYPE, Byte.class);
+ primitiveWrapperMap.put(Character.TYPE, Character.class);
+ primitiveWrapperMap.put(Short.TYPE, Short.class);
+ primitiveWrapperMap.put(Integer.TYPE, Integer.class);
+ primitiveWrapperMap.put(Long.TYPE, Long.class);
+ primitiveWrapperMap.put(Double.TYPE, Double.class);
+ primitiveWrapperMap.put(Float.TYPE, Float.class);
+ primitiveWrapperMap.put(Void.TYPE, Void.TYPE);
+ }
+
+ private static Map<Class<?>, Class<?>> wrapperMap = new HashMap<Class<?>, Class<?>>();
+
+ static {
+ for (final Class<?> primitiveClass : primitiveWrapperMap.keySet()) {
+ final Class<?> wrapperClass = primitiveWrapperMap.get(primitiveClass);
+ if (!primitiveClass.equals(wrapperClass)) {
+ wrapperMap.put(wrapperClass, primitiveClass);
+ }
+ }
+ wrapperMap.put(String.class, String.class);
+ }
+
+ /**
+ * <p>Populate the JavaBeans properties of the specified bean, based on
+ * the specified name/value pairs. This method uses Java reflection APIs
+ * to identify corresponding "property setter" method names, and deals
+ * with setter arguments of type <Code>String</Code>, <Code>boolean</Code>,
+ * <Code>int</Code>, <Code>long</Code>, <Code>float</Code>, and
+ * <Code>double</Code>.</p>
+ *
+ * <p>The particular setter method to be called for each property is
+ * determined using the usual JavaBeans introspection mechanisms. Thus,
+ * you may identify custom setter methods using a BeanInfo class that is
+ * associated with the class of the bean itself. If no such BeanInfo
+ * class is available, the standard method name conversion ("set" plus
+ * the capitalized name of the property in question) is used.</p>
+ *
+ * <p><strong>NOTE</strong>: It is contrary to the JavaBeans Specification
+ * to have more than one setter method (with different argument
+ * signatures) for the same property.</p>
+ *
+ * @param clazz JavaBean class whose properties are being populated
+ * @param properties Map keyed by property name, with the corresponding (String or String[]) value(s) to be set
+ * @param <T> Class type
+ * @return Class instance
+ */
+ public static <T> T populate(final Properties properties, final Class<T> clazz) {
+ T obj = null;
+ try {
+ obj = clazz.newInstance();
+ return populate(properties, obj);
+ } catch (Throwable e) {
+ log.warn("Error occurs !", e);
+ }
+ return obj;
+ }
+
+ public static <T> T populate(final KeyValue properties, final Class<T> clazz) {
+ T obj = null;
+ try {
+ obj = clazz.newInstance();
+ return populate(properties, obj);
+ } catch (Throwable e) {
+ log.warn("Error occurs !", e);
+ }
+ return obj;
+ }
+
+ public static Class<?> getMethodClass(Class<?> clazz, String methodName) {
+ Method[] methods = clazz.getMethods();
+ for (Method method : methods) {
+ if (method.getName().equalsIgnoreCase(methodName)) {
+ return method.getParameterTypes()[0];
+ }
+ }
+ return null;
+ }
+
+ public static void setProperties(Class<?> clazz, Object obj, String methodName,
+ Object value) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ Class<?> parameterClass = getMethodClass(clazz, methodName);
+ Method setterMethod = clazz.getMethod(methodName, parameterClass);
+ if (parameterClass == Boolean.TYPE) {
+ setterMethod.invoke(obj, Boolean.valueOf(value.toString()));
+ } else if (parameterClass == Integer.TYPE) {
+ setterMethod.invoke(obj, Integer.valueOf(value.toString()));
+ } else if (parameterClass == Double.TYPE) {
+ setterMethod.invoke(obj, Double.valueOf(value.toString()));
+ } else if (parameterClass == Float.TYPE) {
+ setterMethod.invoke(obj, Float.valueOf(value.toString()));
+ } else if (parameterClass == Long.TYPE) {
+ setterMethod.invoke(obj, Long.valueOf(value.toString()));
+ } else
+ setterMethod.invoke(obj, value);
+ }
+
+ public static <T> T populate(final Properties properties, final T obj) {
+ Class<?> clazz = obj.getClass();
+ try {
+
+ Set<Map.Entry<Object, Object>> entries = properties.entrySet();
+ for (Map.Entry<Object, Object> entry : entries) {
+ String entryKey = entry.getKey().toString();
+ String[] keyGroup = entryKey.split("\\.");
+ for (int i = 0; i < keyGroup.length; i++) {
+ keyGroup[i] = keyGroup[i].toLowerCase();
+ keyGroup[i] = StringUtils.capitalize(keyGroup[i]);
+ }
+ String beanFieldNameWithCapitalization = StringUtils.join(keyGroup);
+ try {
+ setProperties(clazz, obj, "set" + beanFieldNameWithCapitalization, entry.getValue());
+ } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) {
+ //ignored...
+ }
+ }
+ } catch (RuntimeException e) {
+ log.warn("Error occurs !", e);
+ }
+ return obj;
+ }
+
+ public static <T> T populate(final KeyValue properties, final T obj) {
+ Class<?> clazz = obj.getClass();
+ try {
+
+ final Set<String> keySet = properties.keySet();
+ for (String key : keySet) {
+ String[] keyGroup = key.split("\\.");
+ for (int i = 0; i < keyGroup.length; i++) {
+ keyGroup[i] = keyGroup[i].toLowerCase();
+ keyGroup[i] = StringUtils.capitalize(keyGroup[i]);
+ }
+ String beanFieldNameWithCapitalization = StringUtils.join(keyGroup);
+ try {
+ setProperties(clazz, obj, "set" + beanFieldNameWithCapitalization, properties.getString(key));
+ } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) {
+ //ignored...
+ }
+ }
+ } catch (RuntimeException e) {
+ log.warn("Error occurs !", e);
+ }
+ return obj;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java
new file mode 100644
index 0000000..60c8408
--- /dev/null
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.utils;
+
+import io.openmessaging.BytesMessage;
+import io.openmessaging.KeyValue;
+import io.openmessaging.MessageHeader;
+import io.openmessaging.OMS;
+import io.openmessaging.SendResult;
+import io.openmessaging.rocketmq.domain.BytesMessageImpl;
+import io.openmessaging.rocketmq.domain.NonStandardKeys;
+import io.openmessaging.rocketmq.domain.SendResultImpl;
+import java.lang.reflect.Field;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.message.MessageAccessor;
+
+public class OMSUtil {
+
+ /**
+ * Builds a OMS client instance name.
+ *
+ * @return a unique instance name
+ */
+ public static String buildInstanceName() {
+ return Integer.toString(UtilAll.getPid()) + "%OpenMessaging" + "%" + System.nanoTime();
+ }
+
+ public static org.apache.rocketmq.common.message.Message msgConvert(BytesMessage omsMessage) {
+ org.apache.rocketmq.common.message.Message rmqMessage = new org.apache.rocketmq.common.message.Message();
+ rmqMessage.setBody(omsMessage.getBody());
+
+ KeyValue headers = omsMessage.headers();
+ KeyValue properties = omsMessage.properties();
+
+ //All destinations in RocketMQ use Topic
+ if (headers.containsKey(MessageHeader.TOPIC)) {
+ rmqMessage.setTopic(headers.getString(MessageHeader.TOPIC));
+ rmqMessage.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, "TOPIC");
+ } else {
+ rmqMessage.setTopic(headers.getString(MessageHeader.QUEUE));
+ rmqMessage.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, "QUEUE");
+ }
+
+ for (String key : properties.keySet()) {
+ MessageAccessor.putProperty(rmqMessage, key, properties.getString(key));
+ }
+
+ //Headers has a high priority
+ for (String key : headers.keySet()) {
+ MessageAccessor.putProperty(rmqMessage, key, headers.getString(key));
+ }
+
+ return rmqMessage;
+ }
+
+ public static BytesMessage msgConvert(org.apache.rocketmq.common.message.MessageExt rmqMsg) {
+ BytesMessage omsMsg = new BytesMessageImpl();
+ omsMsg.setBody(rmqMsg.getBody());
+
+ KeyValue headers = omsMsg.headers();
+ KeyValue properties = omsMsg.properties();
+
+ final Set<Map.Entry<String, String>> entries = rmqMsg.getProperties().entrySet();
+
+ for (final Map.Entry<String, String> entry : entries) {
+ if (isOMSHeader(entry.getKey())) {
+ headers.put(entry.getKey(), entry.getValue());
+ } else {
+ properties.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ omsMsg.putHeaders(MessageHeader.MESSAGE_ID, rmqMsg.getMsgId());
+ if (!rmqMsg.getProperties().containsKey(NonStandardKeys.MESSAGE_DESTINATION) ||
+ rmqMsg.getProperties().get(NonStandardKeys.MESSAGE_DESTINATION).equals("TOPIC")) {
+ omsMsg.putHeaders(MessageHeader.TOPIC, rmqMsg.getTopic());
+ } else {
+ omsMsg.putHeaders(MessageHeader.QUEUE, rmqMsg.getTopic());
+ }
+ omsMsg.putHeaders(MessageHeader.SEARCH_KEY, rmqMsg.getKeys());
+ omsMsg.putHeaders(MessageHeader.BORN_HOST, String.valueOf(rmqMsg.getBornHost()));
+ omsMsg.putHeaders(MessageHeader.BORN_TIMESTAMP, rmqMsg.getBornTimestamp());
+ omsMsg.putHeaders(MessageHeader.STORE_HOST, String.valueOf(rmqMsg.getStoreHost()));
+ omsMsg.putHeaders(MessageHeader.STORE_TIMESTAMP, rmqMsg.getStoreTimestamp());
+ return omsMsg;
+ }
+
+ public static boolean isOMSHeader(String value) {
+ for (Field field : MessageHeader.class.getDeclaredFields()) {
+ try {
+ if (field.get(MessageHeader.class).equals(value)) {
+ return true;
+ }
+ } catch (IllegalAccessException e) {
+ return false;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Convert a RocketMQ SEND_OK SendResult instance to a OMS SendResult.
+ */
+ public static SendResult sendResultConvert(org.apache.rocketmq.client.producer.SendResult rmqResult) {
+ assert rmqResult.getSendStatus().equals(SendStatus.SEND_OK);
+ return new SendResultImpl(rmqResult.getMsgId(), OMS.newKeyValue());
+ }
+
+ public static KeyValue buildKeyValue(KeyValue... keyValues) {
+ KeyValue keyValue = OMS.newKeyValue();
+ for (KeyValue properties : keyValues) {
+ for (String key : properties.keySet()) {
+ keyValue.put(key, properties.getString(key));
+ }
+ }
+ return keyValue;
+ }
+
+ /**
+ * Returns an iterator that cycles indefinitely over the elements of {@code Iterable}.
+ */
+ public static <T> Iterator<T> cycle(final Iterable<T> iterable) {
+ return new Iterator<T>() {
+ Iterator<T> iterator = new Iterator<T>() {
+ @Override
+ public synchronized boolean hasNext() {
+ return false;
+ }
+
+ @Override
+ public synchronized T next() {
+ throw new NoSuchElementException();
+ }
+
+ @Override
+ public synchronized void remove() {
+ //Ignore
+ }
+ };
+
+ @Override
+ public synchronized boolean hasNext() {
+ return iterator.hasNext() || iterable.iterator().hasNext();
+ }
+
+ @Override
+ public synchronized T next() {
+ if (!iterator.hasNext()) {
+ iterator = iterable.iterator();
+ if (!iterator.hasNext()) {
+ throw new NoSuchElementException();
+ }
+ }
+ return iterator.next();
+ }
+
+ @Override
+ public synchronized void remove() {
+ iterator.remove();
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/LocalMessageCacheTest.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/LocalMessageCacheTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/LocalMessageCacheTest.java
new file mode 100644
index 0000000..ae4d3ed
--- /dev/null
+++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/LocalMessageCacheTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.consumer;
+
+import io.openmessaging.rocketmq.config.ClientConfig;
+import io.openmessaging.rocketmq.domain.ConsumeRequest;
+import io.openmessaging.rocketmq.domain.NonStandardKeys;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class LocalMessageCacheTest {
+ private LocalMessageCache localMessageCache;
+ @Mock
+ private DefaultMQPullConsumer rocketmqPullConsume;
+ @Mock
+ private ConsumeRequest consumeRequest;
+
+ @Before
+ public void init() {
+ ClientConfig clientConfig = new ClientConfig();
+ clientConfig.setRmqPullMessageBatchNums(512);
+ clientConfig.setRmqPullMessageCacheCapacity(1024);
+ localMessageCache = new LocalMessageCache(rocketmqPullConsume, clientConfig);
+ }
+
+ @Test
+ public void testNextPullBatchNums() throws Exception {
+ assertThat(localMessageCache.nextPullBatchNums()).isEqualTo(512);
+ for (int i = 0; i < 513; i++) {
+ localMessageCache.submitConsumeRequest(consumeRequest);
+ }
+ assertThat(localMessageCache.nextPullBatchNums()).isEqualTo(511);
+ }
+
+ @Test
+ public void testNextPullOffset() throws Exception {
+ MessageQueue messageQueue = new MessageQueue();
+ when(rocketmqPullConsume.fetchConsumeOffset(any(MessageQueue.class), anyBoolean()))
+ .thenReturn(123L);
+ assertThat(localMessageCache.nextPullOffset(new MessageQueue())).isEqualTo(123L);
+ }
+
+ @Test
+ public void testUpdatePullOffset() throws Exception {
+ MessageQueue messageQueue = new MessageQueue();
+ localMessageCache.updatePullOffset(messageQueue, 124L);
+ assertThat(localMessageCache.nextPullOffset(messageQueue)).isEqualTo(124L);
+ }
+
+ @Test
+ public void testSubmitConsumeRequest() throws Exception {
+ byte [] body = new byte[]{'1', '2', '3'};
+ MessageExt consumedMsg = new MessageExt();
+ consumedMsg.setMsgId("NewMsgId");
+ consumedMsg.setBody(body);
+ consumedMsg.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, "TOPIC");
+ consumedMsg.setTopic("HELLO_QUEUE");
+
+ when(consumeRequest.getMessageExt()).thenReturn(consumedMsg);
+ localMessageCache.submitConsumeRequest(consumeRequest);
+ assertThat(localMessageCache.poll()).isEqualTo(consumedMsg);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java
new file mode 100644
index 0000000..277a5c6
--- /dev/null
+++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.consumer;
+
+import io.openmessaging.BytesMessage;
+import io.openmessaging.Message;
+import io.openmessaging.MessageHeader;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.MessagingAccessPointFactory;
+import io.openmessaging.OMS;
+import io.openmessaging.PropertyKeys;
+import io.openmessaging.PullConsumer;
+import io.openmessaging.rocketmq.config.ClientConfig;
+import io.openmessaging.rocketmq.domain.NonStandardKeys;
+import java.lang.reflect.Field;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PullConsumerImplTest {
+ private PullConsumer consumer;
+ private String queueName = "HELLO_QUEUE";
+
+ @Mock
+ private DefaultMQPullConsumer rocketmqPullConsumer;
+ private LocalMessageCache localMessageCache =
+ spy(new LocalMessageCache(rocketmqPullConsumer, new ClientConfig()));
+
+ @Before
+ public void init() throws NoSuchFieldException, IllegalAccessException {
+ final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory
+ .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace");
+ consumer = messagingAccessPoint.createPullConsumer(queueName,
+ OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, "TestGroup"));
+
+ Field field = PullConsumerImpl.class.getDeclaredField("rocketmqPullConsumer");
+ field.setAccessible(true);
+ field.set(consumer, rocketmqPullConsumer); //Replace
+
+ field = PullConsumerImpl.class.getDeclaredField("localMessageCache");
+ field.setAccessible(true);
+ field.set(consumer, localMessageCache);
+
+ messagingAccessPoint.startup();
+ consumer.startup();
+ }
+
+ @Test
+ public void testPoll() {
+ final byte[] testBody = new byte[] {'a', 'b'};
+ MessageExt consumedMsg = new MessageExt();
+ consumedMsg.setMsgId("NewMsgId");
+ consumedMsg.setBody(testBody);
+ consumedMsg.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, "TOPIC");
+ consumedMsg.setTopic(queueName);
+
+ when(localMessageCache.poll()).thenReturn(consumedMsg);
+
+ Message message = consumer.poll();
+ assertThat(message.headers().getString(MessageHeader.MESSAGE_ID)).isEqualTo("NewMsgId");
+ assertThat(((BytesMessage) message).getBody()).isEqualTo(testBody);
+ }
+
+ @Test
+ public void testPoll_WithTimeout() {
+ //There is a default timeout value, @see ClientConfig#omsOperationTimeout.
+ Message message = consumer.poll();
+ assertThat(message).isNull();
+
+ message = consumer.poll(OMS.newKeyValue().put(PropertyKeys.OPERATION_TIMEOUT, 100));
+ assertThat(message).isNull();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java
new file mode 100644
index 0000000..882e57e
--- /dev/null
+++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.consumer;
+
+import io.openmessaging.BytesMessage;
+import io.openmessaging.Message;
+import io.openmessaging.MessageHeader;
+import io.openmessaging.MessageListener;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.MessagingAccessPointFactory;
+import io.openmessaging.OMS;
+import io.openmessaging.PushConsumer;
+import io.openmessaging.ReceivedMessageContext;
+import io.openmessaging.rocketmq.domain.NonStandardKeys;
+import java.lang.reflect.Field;
+import java.util.Collections;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PushConsumerImplTest {
+ private PushConsumer consumer;
+
+ @Mock
+ private DefaultMQPushConsumer rocketmqPushConsumer;
+
+ @Before
+ public void init() throws NoSuchFieldException, IllegalAccessException {
+ final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory
+ .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace");
+ consumer = messagingAccessPoint.createPushConsumer(
+ OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, "TestGroup"));
+
+ Field field = PushConsumerImpl.class.getDeclaredField("rocketmqPushConsumer");
+ field.setAccessible(true);
+ DefaultMQPushConsumer innerConsumer = (DefaultMQPushConsumer) field.get(consumer);
+ field.set(consumer, rocketmqPushConsumer); //Replace
+
+ when(rocketmqPushConsumer.getMessageListener()).thenReturn(innerConsumer.getMessageListener());
+ messagingAccessPoint.startup();
+ consumer.startup();
+ }
+
+ @Test
+ public void testConsumeMessage() {
+ final byte[] testBody = new byte[] {'a', 'b'};
+
+ MessageExt consumedMsg = new MessageExt();
+ consumedMsg.setMsgId("NewMsgId");
+ consumedMsg.setBody(testBody);
+ consumedMsg.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, "TOPIC");
+ consumedMsg.setTopic("HELLO_QUEUE");
+ consumer.attachQueue("HELLO_QUEUE", new MessageListener() {
+ @Override
+ public void onMessage(final Message message, final ReceivedMessageContext context) {
+ assertThat(message.headers().getString(MessageHeader.MESSAGE_ID)).isEqualTo("NewMsgId");
+ assertThat(((BytesMessage) message).getBody()).isEqualTo(testBody);
+ context.ack();
+ }
+ });
+ ((MessageListenerConcurrently) rocketmqPushConsumer
+ .getMessageListener()).consumeMessage(Collections.singletonList(consumedMsg), null);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/ProducerImplTest.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/ProducerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/ProducerImplTest.java
new file mode 100644
index 0000000..1db80c3
--- /dev/null
+++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/ProducerImplTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.producer;
+
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.MessagingAccessPointFactory;
+import io.openmessaging.Producer;
+import io.openmessaging.exception.OMSRuntimeException;
+import java.lang.reflect.Field;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ProducerImplTest {
+ private Producer producer;
+
+ @Mock
+ private DefaultMQProducer rocketmqProducer;
+
+ @Before
+ public void init() throws NoSuchFieldException, IllegalAccessException {
+ final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory
+ .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace");
+ producer = messagingAccessPoint.createProducer();
+
+ Field field = AbstractOMSProducer.class.getDeclaredField("rocketmqProducer");
+ field.setAccessible(true);
+ field.set(producer, rocketmqProducer);
+
+ messagingAccessPoint.startup();
+ producer.startup();
+ }
+
+ @Test
+ public void testSend_OK() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
+ SendResult sendResult = new SendResult();
+ sendResult.setMsgId("TestMsgID");
+ sendResult.setSendStatus(SendStatus.SEND_OK);
+ when(rocketmqProducer.send(any(Message.class), anyLong())).thenReturn(sendResult);
+ io.openmessaging.SendResult omsResult =
+ producer.send(producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'}));
+
+ assertThat(omsResult.messageId()).isEqualTo("TestMsgID");
+ }
+
+ @Test
+ public void testSend_Not_OK() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
+ SendResult sendResult = new SendResult();
+ sendResult.setSendStatus(SendStatus.FLUSH_DISK_TIMEOUT);
+
+ when(rocketmqProducer.send(any(Message.class), anyLong())).thenReturn(sendResult);
+ try {
+ producer.send(producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'}));
+ failBecauseExceptionWasNotThrown(OMSRuntimeException.class);
+ } catch (Exception e) {
+ assertThat(e).hasMessageContaining("Send message to RocketMQ broker failed.");
+ }
+ }
+
+ @Test
+ public void testSend_WithException() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
+ when(rocketmqProducer.send(any(Message.class), anyLong())).thenThrow(MQClientException.class);
+ try {
+ producer.send(producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'}));
+ failBecauseExceptionWasNotThrown(OMSRuntimeException.class);
+ } catch (Exception e) {
+ assertThat(e).hasMessageContaining("Send message to RocketMQ broker failed.");
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/SequenceProducerImplTest.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/SequenceProducerImplTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/SequenceProducerImplTest.java
new file mode 100644
index 0000000..823fe01
--- /dev/null
+++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/producer/SequenceProducerImplTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.producer;
+
+import io.openmessaging.BytesMessage;
+import io.openmessaging.MessageHeader;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.MessagingAccessPointFactory;
+import io.openmessaging.SequenceProducer;
+import java.lang.reflect.Field;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class SequenceProducerImplTest {
+
+ private SequenceProducer producer;
+
+ @Mock
+ private DefaultMQProducer rocketmqProducer;
+
+ @Before
+ public void init() throws NoSuchFieldException, IllegalAccessException {
+ final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory
+ .getMessagingAccessPoint("openmessaging:rocketmq://IP1:9876,IP2:9876/namespace");
+ producer = messagingAccessPoint.createSequenceProducer();
+
+ Field field = AbstractOMSProducer.class.getDeclaredField("rocketmqProducer");
+ field.setAccessible(true);
+ field.set(producer, rocketmqProducer);
+
+ messagingAccessPoint.startup();
+ producer.startup();
+ }
+
+ @Test
+ public void testSend_WithCommit() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
+ SendResult sendResult = new SendResult();
+ sendResult.setMsgId("TestMsgID");
+ sendResult.setSendStatus(SendStatus.SEND_OK);
+ when(rocketmqProducer.send(ArgumentMatchers.<Message>anyList())).thenReturn(sendResult);
+ when(rocketmqProducer.getMaxMessageSize()).thenReturn(1024);
+ final BytesMessage message = producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'});
+ producer.send(message);
+ producer.commit();
+ assertThat(message.headers().getString(MessageHeader.MESSAGE_ID)).isEqualTo("TestMsgID");
+ }
+
+ @Test
+ public void testRollback() {
+ when(rocketmqProducer.getMaxMessageSize()).thenReturn(1024);
+ final BytesMessage message = producer.createBytesMessageToTopic("HELLO_TOPIC", new byte[] {'a'});
+ producer.send(message);
+ producer.rollback();
+ producer.commit(); //Commit nothing.
+ assertThat(message.headers().getString(MessageHeader.MESSAGE_ID)).isEqualTo(null);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/test/java/io/openmessaging/rocketmq/promise/DefaultPromiseTest.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/promise/DefaultPromiseTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/promise/DefaultPromiseTest.java
new file mode 100644
index 0000000..2240ff2
--- /dev/null
+++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/promise/DefaultPromiseTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.promise;
+
+import io.openmessaging.Promise;
+import io.openmessaging.PromiseListener;
+import io.openmessaging.exception.OMSRuntimeException;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
+
+public class DefaultPromiseTest {
+ private Promise<String> promise;
+
+ @Before
+ public void init() {
+ promise = new DefaultPromise<>();
+ }
+
+ @Test
+ public void testIsCancelled() throws Exception {
+ assertThat(promise.isCancelled()).isEqualTo(false);
+ }
+
+ @Test
+ public void testIsDone() throws Exception {
+ assertThat(promise.isDone()).isEqualTo(false);
+ promise.set("Done");
+ assertThat(promise.isDone()).isEqualTo(true);
+ }
+
+ @Test
+ public void testGet() throws Exception {
+ promise.set("Done");
+ assertThat(promise.get()).isEqualTo("Done");
+ }
+
+ @Test
+ public void testGet_WithTimeout() throws Exception {
+ try {
+ promise.get(100);
+ failBecauseExceptionWasNotThrown(OMSRuntimeException.class);
+ } catch (OMSRuntimeException e) {
+ assertThat(e).hasMessageContaining("Get request result is timeout or interrupted");
+ }
+ }
+
+ @Test
+ public void testAddListener() throws Exception {
+ promise.addListener(new PromiseListener<String>() {
+ @Override
+ public void operationCompleted(final Promise<String> promise) {
+ assertThat(promise.get()).isEqualTo("Done");
+ }
+
+ @Override
+ public void operationFailed(final Promise<String> promise) {
+
+ }
+ });
+ promise.set("Done");
+ }
+
+ @Test
+ public void testAddListener_ListenerAfterSet() throws Exception {
+ promise.set("Done");
+ promise.addListener(new PromiseListener<String>() {
+ @Override
+ public void operationCompleted(final Promise<String> promise) {
+ assertThat(promise.get()).isEqualTo("Done");
+ }
+
+ @Override
+ public void operationFailed(final Promise<String> promise) {
+
+ }
+ });
+ }
+
+ @Test
+ public void testAddListener_WithException_ListenerAfterSet() throws Exception {
+ final Throwable exception = new OMSRuntimeException("-1", "Test Error");
+ promise.setFailure(exception);
+ promise.addListener(new PromiseListener<String>() {
+ @Override
+ public void operationCompleted(final Promise<String> promise) {
+ }
+
+ @Override
+ public void operationFailed(final Promise<String> promise) {
+ assertThat(promise.getThrowable()).isEqualTo(exception);
+ }
+ });
+ }
+
+ @Test
+ public void testAddListener_WithException() throws Exception {
+ final Throwable exception = new OMSRuntimeException("-1", "Test Error");
+ promise.addListener(new PromiseListener<String>() {
+ @Override
+ public void operationCompleted(final Promise<String> promise) {
+ }
+
+ @Override
+ public void operationFailed(final Promise<String> promise) {
+ assertThat(promise.getThrowable()).isEqualTo(exception);
+ }
+ });
+ promise.setFailure(exception);
+ }
+
+ @Test
+ public void getThrowable() throws Exception {
+ assertThat(promise.getThrowable()).isNull();
+ Throwable exception = new OMSRuntimeException("-1", "Test Error");
+ promise.setFailure(exception);
+ assertThat(promise.getThrowable()).isEqualTo(exception);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/openmessaging/src/test/java/io/openmessaging/rocketmq/utils/BeanUtilsTest.java
----------------------------------------------------------------------
diff --git a/openmessaging/src/test/java/io/openmessaging/rocketmq/utils/BeanUtilsTest.java b/openmessaging/src/test/java/io/openmessaging/rocketmq/utils/BeanUtilsTest.java
new file mode 100644
index 0000000..71ca11c
--- /dev/null
+++ b/openmessaging/src/test/java/io/openmessaging/rocketmq/utils/BeanUtilsTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.rocketmq.utils;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.OMS;
+import io.openmessaging.rocketmq.config.ClientConfig;
+import io.openmessaging.rocketmq.domain.NonStandardKeys;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class BeanUtilsTest {
+ private KeyValue properties = OMS.newKeyValue();
+
+ public static class CustomizedConfig extends ClientConfig {
+ final static String STRING_TEST = "string.test";
+ String stringTest = "foobar";
+
+ final static String DOUBLE_TEST = "double.test";
+ double doubleTest = 123.0;
+
+ final static String LONG_TEST = "long.test";
+ long longTest = 123L;
+
+ String getStringTest() {
+ return stringTest;
+ }
+
+ public void setStringTest(String stringTest) {
+ this.stringTest = stringTest;
+ }
+
+ double getDoubleTest() {
+ return doubleTest;
+ }
+
+ public void setDoubleTest(final double doubleTest) {
+ this.doubleTest = doubleTest;
+ }
+
+ long getLongTest() {
+ return longTest;
+ }
+
+ public void setLongTest(final long longTest) {
+ this.longTest = longTest;
+ }
+
+ CustomizedConfig() {
+ }
+ }
+
+ @Before
+ public void init() {
+ properties.put(NonStandardKeys.MAX_REDELIVERY_TIMES, 120);
+ properties.put(CustomizedConfig.STRING_TEST, "kaka");
+ properties.put(NonStandardKeys.CONSUMER_GROUP, "Default_Consumer_Group");
+ properties.put(NonStandardKeys.MESSAGE_CONSUME_TIMEOUT, 101);
+
+ properties.put(CustomizedConfig.LONG_TEST, 1234567890L);
+ properties.put(CustomizedConfig.DOUBLE_TEST, 10.234);
+ }
+
+ @Test
+ public void testPopulate() {
+ CustomizedConfig config = BeanUtils.populate(properties, CustomizedConfig.class);
+
+ //RemotingConfig config = BeanUtils.populate(properties, RemotingConfig.class);
+ Assert.assertEquals(config.getRmqMaxRedeliveryTimes(), 120);
+ Assert.assertEquals(config.getStringTest(), "kaka");
+ Assert.assertEquals(config.getRmqConsumerGroup(), "Default_Consumer_Group");
+ Assert.assertEquals(config.getRmqMessageConsumeTimeout(), 101);
+ Assert.assertEquals(config.getLongTest(), 1234567890L);
+ Assert.assertEquals(config.getDoubleTest(), 10.234, 0.000001);
+ }
+
+ @Test
+ public void testPopulate_ExistObj() {
+ CustomizedConfig config = new CustomizedConfig();
+ config.setOmsConsumerId("NewConsumerId");
+
+ Assert.assertEquals(config.getOmsConsumerId(), "NewConsumerId");
+
+ config = BeanUtils.populate(properties, config);
+
+ //RemotingConfig config = BeanUtils.populate(properties, RemotingConfig.class);
+ Assert.assertEquals(config.getRmqMaxRedeliveryTimes(), 120);
+ Assert.assertEquals(config.getStringTest(), "kaka");
+ Assert.assertEquals(config.getRmqConsumerGroup(), "Default_Consumer_Group");
+ Assert.assertEquals(config.getRmqMessageConsumeTimeout(), 101);
+ Assert.assertEquals(config.getLongTest(), 1234567890L);
+ Assert.assertEquals(config.getDoubleTest(), 10.234, 0.000001);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rocketmq/blob/1d966b50/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 05ead63..25e4c84 100644
--- a/pom.xml
+++ b/pom.xml
@@ -181,6 +181,7 @@
<module>filter</module>
<module>test</module>
<module>distribution</module>
+ <module>openmessaging</module>
</modules>
<build>
@@ -617,6 +618,11 @@
<artifactId>guava</artifactId>
<version>19.0</version>
</dependency>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-api</artifactId>
+ <version>0.1.0-alpha</version>
+ </dependency>
</dependencies>
</dependencyManagement>
</project>